#1157 frontend: cache the recent task queries
Merged 4 years ago by praiskup. Opened 4 years ago by praiskup.
Unknown source caching-recent-builds  into  master

@@ -78,6 +78,7 @@

  BuildRequires: python3-dateutil

  BuildRequires: python3-decorator

  BuildRequires: python3-flask

+ BuildRequires: python3-flask-cache

  BuildRequires: python3-flask-openid

  BuildRequires: python3-flask-restful

  BuildRequires: python3-flask-script
@@ -131,6 +132,7 @@

  Requires: python3-copr-common > 0.4

  Requires: python3-dateutil

  Requires: python3-flask

+ Requires: python3-flask-cache

  Requires: python3-flask-openid

  Requires: python3-flask-restful

  Requires: python3-flask-script

@@ -5,6 +5,7 @@

  

  from flask_sqlalchemy import SQLAlchemy

  from contextlib import contextmanager

+ from flask_cache import Cache

  from flask_openid import OpenID

  from flask_whooshee import Whooshee

  from openid_teams.teams import TeamsResponse
@@ -57,6 +58,15 @@

  rcp = RedisConnectionProvider(config=app.config)

  app.session_interface = RedisSessionInterface(rcp.get_connection())

  

+ cache_rcp = RedisConnectionProvider(config=app.config, db=1)

+ cache = Cache(app, config={

+     'CACHE_TYPE': 'redis',

+     'CACHE_REDIS_DB': 1,    # we use 0 for sessions

+     'CACHE_KEY_PREFIX': 'copr_cache_',

+     'CACHE_REDIS_HOST': cache_rcp.host,

+     'CACHE_REDIS_PORT': cache_rcp.port,

+ })

+ 

  from coprs.views import admin_ns

  from coprs.views.admin_ns import admin_general

  from coprs.views import api_ns

@@ -312,12 +312,13 @@

  

  

  class RedisConnectionProvider(object):

-     def __init__(self, config):

+     def __init__(self, config, db=0):

          self.host = config.get("REDIS_HOST", "127.0.0.1")

          self.port = int(config.get("REDIS_PORT", "6379"))

+         self.db = db

  

      def get_connection(self):

-         return StrictRedis(host=self.host, port=self.port)

+         return StrictRedis(host=self.host, port=self.port, db=self.db)

  

  

  def get_redis_connection():

@@ -19,6 +19,7 @@

  

  from copr_common.enums import FailTypeEnum, StatusEnum

  from coprs import app

+ from coprs import cache

  from coprs import db

  from coprs import models

  from coprs import helpers
@@ -66,7 +67,8 @@

          return result

  

      @classmethod

-     def get_recent_tasks(cls, user=None, limit=100, period_days=2):

+     @cache.memoize(timeout=2*60)

+     def get_recent_task_ids(cls, user=None, limit=100, period_days=2):

          query_args = (

              models.BuildChroot.build_id,

              func.max(models.BuildChroot.ended_on).label('max_ended_on'),
@@ -94,6 +96,12 @@

          subquery = subquery.order_by(desc('max_ended_on')).limit(limit).subquery()

  

          query = models.Build.query.join(subquery, subquery.c.build_id == models.Build.id)

+         return [i.id for i in query.all()]

+ 

+     @classmethod

+     def get_recent_tasks(cls, *args, **kwargs):

+         task_ids = cls.get_recent_task_ids(*args, **kwargs)

+         query = models.Build.query.filter(models.Build.id.in_(task_ids))

          return list(query.all())

  

      @classmethod

So as part of this commit, initialize the flask_cache mechanism, using
the already running redis database (we use it for session management).
The 'cache' object logically belongs to init.py, though it needs to
be placed on a pretty weird place (that's because there are
bi-directional dependencies among our modules, this is a problem but to
be solved in a different PR).

Also, instead of caching get_recent_tasks() directly, I created
get_recent_task_ids() which is safe for caching. That's because the
contents is just a list of integers, not sqlalchemy objects.

The problem of caching sqlalchemy objects is that they allow
lazy-loading later in templates, and if loaded from cache - when the
object isn't bound to any session - the lazy load attempt would fail.
Therefore we cache just the ides, and before passing the objects down to
the template we perform another (but way cheaper) query.

rebased onto a57f7e3

4 years ago

Thanks, I'm merging sooner than 24h+ so I can wrap micro release with this.

Pull-Request has been merged by praiskup

4 years ago