#1210 frontend: memory optimize /packages/ and /builds/ routes
Merged 4 years ago by praiskup. Opened 4 years ago by praiskup.
Unknown source optimize-packages-builds  into  master

@@ -338,7 +338,7 @@

              copr_dir = copr.main_dir

          query = query.filter(models.Build.copr_dir_id==copr_dir.id)

          query = query.options(selectinload('build_chroots'), selectinload('package'))

-         return list(query)

+         return query

  

      @classmethod

      def join_group(cls, query):

@@ -43,16 +43,32 @@

              .filter(models.Build.package_id.in_(pkg_ids)) \

              .with_entities(func.max(models.Build.id)) \

              .group_by(models.Build.package_id)

+ 

+         # map package.id => package object in packages array

          packages_map = {package.id: package for package in packages}

-         builds = (

-             models.Build.query.filter(models.Build.id.in_(builds_ids))

-                               .options(selectinload('build_chroots')))

+ 

+         builds = (models.Build.query.filter(models.Build.id.in_(builds_ids))

+                   .options(selectinload('build_chroots'))

+                   .yield_per(1000))

+ 

          for build in builds:

+             class SmallBuild():

+                 pass

+ 

              if not build.package_id:

                  continue

-             packages_map[build.package_id].latest_build = build

  

-         return list(packages)

+             small_build_object = SmallBuild()

+             for param in ['state', 'status', 'pkg_version',

+                           'submitted_on']:

+                 # we don't want to keep all the attributes here in memory, and

+                 # also we don't need any further info about assigned

+                 # build_chroot(s).  So we only pick the info we need, and throw

+                 # the expensive objects away.

+                 setattr(small_build_object, param, getattr(build, param))

+             packages_map[build.package_id].latest_build = small_build_object

+ 

+         return packages

  

      @classmethod

      def get_list_by_copr(cls, copr_id, package_name):

@@ -55,9 +55,10 @@

      flashes = flask.session.pop('_flashes', [])

      dirname = flask.request.args.get('dirname')

      builds_query = builds_logic.BuildsLogic.get_copr_builds_list(copr, dirname)

+     builds = builds_query.yield_per(1000)

      response = flask.Response(stream_with_context(helpers.stream_template("coprs/detail/builds.html",

                                copr=copr,

-                               builds=list(builds_query),

+                               builds=builds,

                                current_dirname=dirname,

                                flashes=flashes)))

  

To have properly generated "stream" feeling from streamed jinja, we need
to have streamed sqlalchemy result set as well. So yield_per helped a
lot in case of /builds/.

With packages, it's not that trivial. But the set of returned packages
is much smaller than set of returned builds (and build_chroots!). So
let's only return we are interested in (package + latest_build status,
and some small things).

rebased onto fcb2fc3

4 years ago

Commit 8c87939 fixes this pull-request

Pull-Request has been merged by praiskup

4 years ago