#15 Caching Refactor
Merged 3 years ago by frantisekz. Opened 3 years ago by frantisekz.

file modified
+5 -5
@@ -51,8 +51,6 @@ 

  # Is this an OpenShift deployment?

  openshift_production = os.getenv('OPENSHIFT_PROD')

  

- cache = Cache(app, config={'CACHE_TYPE': 'simple'})

- 

  # Load default config, then override that with a config file

  if os.getenv('DEV') == 'true':

      default_config_obj = 'oraculum.config.DevelopmentConfig'
@@ -128,11 +126,13 @@ 

  

  ## database

  db = SQLAlchemy(app)

- #

- # register blueprints

+ 

+ from .utils import cache_utils

+ CACHE = cache_utils.Cached(max_cache_age=app.config['MAX_DB_AGE'])

+ 

  

  CORS(app, supports_credentials=True)

- #CORS(app)

+ 

  oidc = OpenIDConnect(app)

  

  login_manager = LoginManager()

file modified
+10 -7
@@ -24,8 +24,10 @@ 

  from alembic import command as al_command

  from alembic.migration import MigrationContext

  

- from oraculum import app, db, controllers

- from oraculum.utils import db_utils

+ from oraculum import db, CACHE

+ from oraculum.controllers.main import register_cache_providers

+ from oraculum.data_providers import PROVIDERS

+ from oraculum.models.db_cache import CachedData

  

  def get_alembic_config():

      # the location of the alembic ini file and alembic scripts changes when
@@ -67,11 +69,12 @@ 

  

  def sync():

      print("Refreshing DB Cache")

-     app.config['MAX_DB_AGE'] = 0

-     app.config['FORCE_CACHED_DATA'] = False

-     db_utils.refresh_data("get_actions", controllers.main.get_actions())

-     db_utils.refresh_data("api_v1_landing_page", controllers.main.api_v1_landing_page())

-     db_utils.refresh_data("api_v1_libkarma", controllers.main.api_v1_libkarma('all'))

+     db.session.query(CachedData).delete()

+     db.session.commit()

+ 

+     register_cache_providers()

+     for r in CACHE._refreshers:

+         CACHE._refresh(r)

  

  def main():

      parser = ArgumentParser()

file modified
+29 -38
@@ -30,9 +30,20 @@ 

      login_user, logout_user, current_user

  from flask import request, url_for, jsonify, redirect

  

- from oraculum import app, login_manager, oidc

+ from oraculum import app, login_manager, oidc, CACHE

  from oraculum.data_providers import PROVIDERS

- from oraculum.utils import fedocal, schedule, blockerbugs, meetbot, db_utils, libkarma

+ from oraculum.utils import fedocal, schedule, blockerbugs, meetbot, cache_utils, libkarma

+ 

+ @app.before_first_request

+ def register_cache_providers():

+     """

+     Add providers to be cached here

+     """

+     app.logger.debug('Registering cache refreshers')

+     CACHE.register('landing_page', api_v1_landing_page)

+     CACHE.register('bodhi', api_v1_all_bodhi_updates)

+     for p_name, p_module in PROVIDERS.items():

+         CACHE.register(p_name, p_module.get_actions)

  

  # FIXME: move somewhere else?

  class User(UserMixin):
@@ -42,41 +53,29 @@ 

  

  @app.route('/api/v1/libkarma/<release>', methods=['GET'])

  def route_api_v1_libkarma(release):

-     data = api_v1_libkarma(release)

-     if data:

-         return jsonify(data)

-     return jsonify({'error': 'Release %s not found' % release}), 404

- 

- def api_v1_libkarma(release):

-     landing_page = api_v1_landing_page()

+     landing_page = CACHE.get('landing_page')

      current_releases = [str(i) for i in [(landing_page["stable"] - 1), landing_page["stable"], landing_page["devel"]]]

      if release in current_releases:

-         return get_updates_for_release(release)

+         return jsonify(CACHE.get('bodhi')["F" + release])

      if release.lower() == 'all':

-         updates = {}

-         for single_release in current_releases:

-             app.logger.debug("Appending updates for release: %s" % single_release)

-             updates["F" + single_release] = get_updates_for_release(single_release)

-         return updates

-     return None

- 

- def get_updates_for_release(release):

-     cached = db_utils.get_db_data("api_v1_libkarma_F" + release)

-     if cached:

-         return cached

-     karma = libkarma.get_updates(release, app.logger)

-     db_utils.refresh_data("api_v1_libkarma_F" + release, karma)

-     return karma

+         return jsonify(CACHE.get('bodhi'))

+     return jsonify({'error': 'Release %s not found' % release}), 404

+ 

+ def api_v1_all_bodhi_updates():

+     landing_page = CACHE.get('landing_page')

+     current_releases = [str(i) for i in [(landing_page["stable"] - 1), landing_page["stable"], landing_page["devel"]]]

+     updates = {}

+     for single_release in current_releases:

+         app.logger.debug("Appending updates for release: %s" % single_release)

+         updates["F" + single_release] = libkarma.get_updates(single_release, app.logger)

+     return updates

  

  @app.route('/api/v1/landing_page')

  def route_api_v1_landing_page():

-     return jsonify(api_v1_landing_page())

+     return jsonify(CACHE.get('landing_page'))

  

  def api_v1_landing_page():

-     cached = db_utils.get_db_data("api_v1_landing_page")

-     if cached:

-         return cached

-     resp = {

+     return {

          'meetings': fedocal.get_qa_meetings(),

          'last_qa_meeting': meetbot.get_last_qa_meeting(),

          'schedule': schedule.get_schedule(),
@@ -84,19 +83,11 @@ 

          'stable': schedule.current_stable(),

          'devel': schedule.current_devel(),

      }

-     db_utils.refresh_data("api_v1_landing_page", resp)

-     return resp

  

  def get_actions(provider=None, tags=None):

      actions = []

      for p_name, p_module in PROVIDERS.items():

-         cached = db_utils.get_db_data(p_name)

-         if cached:

-             actions.extend(cached)

-         else:

-             p_actions = p_module.get_actions()

-             actions.extend(p_actions)

-             db_utils.refresh_data(p_name, p_actions)

+         actions.extend(CACHE.get(p_name))

  

      if provider:

          actions = [a for a in actions if a['provider'] == provider]

@@ -20,6 +20,8 @@ 

  # Authors:

  #   Frantisek Zatloukal <fzatlouk@redhat.com>

  

+ import json

+ 

  from oraculum import db

  from datetime import datetime

  
@@ -33,3 +35,13 @@ 

          self.provider = provider

          self.time_created = datetime.utcnow()

          self.raw_text = raw_text

+         self._parsed_raw_text = None

+ 

+     @property

+     def data(self):

+         try:

+             if self._parsed_raw_text is None:

+                 self._parsed_raw_text = json.loads(self.raw_text)

+         except AttributeError:

+                 self._parsed_raw_text = json.loads(self.raw_text)

+         return self._parsed_raw_text

@@ -0,0 +1,106 @@ 

+ #

+ # cache_utils.py - Database helper functions landing page cache

+ #

+ # Copyright 2020, Red Hat, Inc

+ #

+ # This program is free software; you can redistribute it and/or modify

+ # it under the terms of the GNU General Public License as published by

+ # the Free Software Foundation; either version 2 of the License, or

+ # (at your option) any later version.

+ #

+ # This program is distributed in the hope that it will be useful,

+ # but WITHOUT ANY WARRANTY; without even the implied warranty of

+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ # GNU General Public License for more details.

+ #

+ # You should have received a copy of the GNU General Public License along

+ # with this program; if not, write to the Free Software Foundation, Inc.,

+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ #

+ # Authors:

+ #   Frantisek Zatloukal <fzatlouk@redhat.com>

+ #   Josef Skladanka <jskladan@redhat.com>

+ 

+ import datetime

+ import json

+ 

+ from oraculum import app, db

+ from oraculum.models.db_cache import CachedData

+ 

+ class CachedObject():

+     def __init__(self, time_created, data):

+         self.time_created = time_created

+         self.data = data

+ 

+ class Cached():

+     def __init__(self, max_cache_age):

+         self._refreshers = {}

+         self._local_cache = {}

+         self._max_cache_age = datetime.timedelta(seconds=max_cache_age)

+ 

+     # getting the values

+     @property

+     def max_cache_age(self):

+         return self._max_cache_age

+ 

+     # setting the values

+     @max_cache_age.setter

+     def max_cache_age(self, value):

+         self._max_cache_age = datetime.timedelta(seconds=value)

+ 

+     def register(self, what, refresh_method):

+         self._refreshers[what] = refresh_method

+ 

+     def get(self, what):

+         if what not in self._refreshers:

+             raise Exception

+ 

+         if self._new_enough(self._local_cache.get(what)):

+             app.logger.debug("_local_cache cache hit on %s" % what)

+             return self._local_cache[what].data

+ 

+         app.logger.debug("_local_cache miss on %s" % what)

+ 

+         from_db = CachedData.query.filter_by(provider=what).first()

+         if not from_db or not self._new_enough(from_db):

+             app.logger.debug("db cache miss on %s" % what)

+             self._refresh(what)

+         else:

+             app.logger.debug("db cache hit on %s" % what)

+             self._local_cache[what] = CachedObject(from_db.time_created, from_db.data)

+ 

+         return self._local_cache[what].data

+ 

+     def _new_enough(self, cached_object):

+         if cached_object is None:

+             return False

+         if app.config['FORCE_CACHED_DATA']:

+             return True

+         if datetime.datetime.utcnow() > cached_object.time_created + self.max_cache_age:

+             return False

+         return True

+ 

+     def _refresh(self, what):

+         row = CachedData.query.filter_by(provider=what).first()

+ 

+         data = self._refreshers[what]()

+ 

+         if row:

+             row.time_created = datetime.datetime.utcnow()

+             row.raw_text = json.dumps(data)

+         else:

+             row = CachedData(what, json.dumps(data))

+             db.session.add(row)

+ 

+         db.session.commit()

+         self._local_cache[what] = CachedObject(row.time_created, data)

+ 

+ if __name__ == "__main__":

+     CACHE = Cached(max_cache_age=300)

+     import ipdb; ipdb.set_trace()

+     """

+     CACHE.register('bodhi', libkarma.get_all_updates)

+ 

+     print("- Getting bodhi")

+     test = CACHE.get('bodhi')

+     """

@@ -1,63 +0,0 @@ 

- #

- # db_utils.py - Database helper functions landing page cache

- #

- # Copyright 2020, Red Hat, Inc

- #

- # This program is free software; you can redistribute it and/or modify

- # it under the terms of the GNU General Public License as published by

- # the Free Software Foundation; either version 2 of the License, or

- # (at your option) any later version.

- #

- # This program is distributed in the hope that it will be useful,

- # but WITHOUT ANY WARRANTY; without even the implied warranty of

- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

- # GNU General Public License for more details.

- #

- # You should have received a copy of the GNU General Public License along

- # with this program; if not, write to the Free Software Foundation, Inc.,

- # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

- #

- # Authors:

- #   Frantisek Zatloukal <fzatlouk@redhat.com>

- import json

- 

- from datetime import datetime, timedelta

- from oraculum import app, db

- from oraculum.models.db_cache import CachedData

- 

- def is_new_enough(db_time):

-     """

-     Checks if given db_time is new enough according to MAX_DB_AGE

-     Skips check if FORCE_CACHED_DATA is set to True

-     """

-     if app.config['FORCE_CACHED_DATA']:

-         return True

-     if not db_time:

-         return False

-     if (datetime.utcnow() - timedelta(seconds=app.config['MAX_DB_AGE'])) >= db_time:

-         return False

-     return True

- 

- def refresh_data(provider, data):

-     """

-     Refreshes given data for given provider in the db

-     """

-     row = CachedData.query.filter_by(provider=provider).first()

-     if row:

-         row.time_created = datetime.utcnow()

-         row.raw_text = json.dumps(data)

-     else:

-         db.session.add(CachedData(provider, json.dumps(data)))

-     db.session.commit()

- 

- def get_db_data(provider):

-     """

-     Returns raw_text for API serving

-     Tries to fetch raw_text from db if new enough, returns false on cache miss

-     """

-     row = CachedData.query.filter_by(provider=provider).first()

-     if row and is_new_enough(row.time_created):

-         app.logger.debug("DB Cache hit for provider: %s" % provider)

-         return json.loads(row.raw_text)

-     app.logger.debug("DB Cache miss for provider: %s" % provider)

-     return False

no initial comment

1 new commit added

  • sync cleanup
3 years ago

2 new commits added

  • sync cleanup
  • Caching Refactor
3 years ago

2 new commits added

  • sync cleanup
  • Caching Refactor
3 years ago

1 new commit added

  • Cleaning
3 years ago

1 new commit added

  • Fixup
3 years ago

ACK, with changes discussed in person.

rebased onto 2b53679

3 years ago

Pull-Request has been merged by frantisekz

3 years ago