#111 Store counters in the database instead of Redis
Opened 2 months ago by frantisekz. Modified 2 months ago

@@ -0,0 +1,34 @@ 

+ """DB counters

+ 

+ Revision ID: 708ec1762eed

+ Revises: 2ac7e20e2b30

+ Create Date: 2020-11-15 12:26:33.890125

+ 

+ """

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ # revision identifiers, used by Alembic.

+ revision = '708ec1762eed'

+ down_revision = '2ac7e20e2b30'

+ branch_labels = None

+ depends_on = None

+ 

+ 

+ def upgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.create_table('counters',

+     sa.Column('id', sa.Integer(), nullable=False),

+     sa.Column('what', sa.Text(), nullable=True),

+     sa.Column('counter', sa.Integer(), nullable=True),

+     sa.PrimaryKeyConstraint('id'),

+     sa.UniqueConstraint('what')

+     )

+     # ### end Alembic commands ###

+ 

+ 

+ def downgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.drop_table('counters')

+     # ### end Alembic commands ###

@@ -0,0 +1,33 @@ 

+ #

+ # counters.py - Database model for Oraculum Counters

+ #

+ # Copyright 2020, Red Hat, Inc

+ #

+ # This program is free software; you can redistribute it and/or modify

+ # it under the terms of the GNU General Public License as published by

+ # the Free Software Foundation; either version 2 of the License, or

+ # (at your option) any later version.

+ #

+ # This program is distributed in the hope that it will be useful,

+ # but WITHOUT ANY WARRANTY; without even the implied warranty of

+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ # GNU General Public License for more details.

+ #

+ # You should have received a copy of the GNU General Public License along

+ # with this program; if not, write to the Free Software Foundation, Inc.,

+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ #

+ # Authors:

+ #   Frantisek Zatloukal <fzatlouk@redhat.com>

+ 

+ from oraculum import db

+ 

+ 

+ class Counters(db.Model):

+     id = db.Column(db.Integer, primary_key=True)

+     what = db.Column(db.Text, unique=True)

+     counter = db.Column(db.Integer)

+ 

+     def __init__(self, what, counter=0):

+         self.counter = counter

+         self.what = what

file modified
+1 -1
@@ -1,5 +1,5 @@ 

  #

- # landing_page.py - Database model for landing page

+ # db_cache.py - Database model for Oraculum Cache Data

  #

  # Copyright 2020, Red Hat, Inc

  #

@@ -26,9 +26,10 @@ 

  import traceback

  

  import oraculum

- from oraculum import app, celery_app

+ from oraculum import app, celery_app, db

  from oraculum.action_providers import ACTION_PROVIDERS

  from oraculum.models.dashboard_users import DashboardUserData

+ from oraculum.models.counters import Counters

  from oraculum.utils.watchdog_utils import push_to_watchdog, process_queue

  

  @celery_app.task
@@ -42,9 +43,17 @@ 

      priority = priorities.get(priority, priorities['medium'])

  

      what = oraculum.CACHE._construct_what(*args, **kwargs)

+     counter = Counters.query.with_for_update().filter_by(what=what).first()

+ 

+     if counter:

+         counter.counter = counter.counter + 1

+     else:

+         counter = (Counters(what, 1))

+         db.session.add(counter)

+     db.session.commit()

+ 

+     counter = counter.counter

  

-     redis_conn = celery_app.broker_connection().default_channel.client

-     counter = redis_conn.incr(what)

      if counter == 1:

          app.logger.debug("PLANNING - celery_refresh on %s" % what)

          push_to_watchdog("sync_planned", what)
@@ -66,8 +75,9 @@ 

          #raise

      finally:

          # Cleanup counter to allow another scheduling

-         redis_conn = celery_app.broker_connection().default_channel.client

-         counter = redis_conn.getset(what, 0)

+         counter = Counters.query.with_for_update().filter_by(what=what).first()

+         counter.counter = 0

+         db.session.commit()

  

      app.logger.debug("END - celery_refresh on %s" % what)

  

@@ -22,11 +22,13 @@ 

  

  import re

  import datetime

+ from typing import Counter

  

  from sqlalchemy import desc

  

  from oraculum.models.watchdog import WatchDogData, WatchDogStartPoint

  from oraculum.models.db_cache import CachedData

+ from oraculum.models.counters import Counters

  from oraculum.utils.mailing_utils import send_mail

  

  from oraculum import app, celery_app, db
@@ -55,9 +57,6 @@ 

      from oraculum.utils.celery_utils import get_users_for_sync

      from oraculum import CACHE

  

-     if app.config["ACTIVE_WATCHDOG"]:

-         redis_conn = celery_app.broker_connection().default_channel.client

- 

      # Prepare users and packages who meet criteria for sync

      cached_users = get_users_for_sync()

      cached_packages = set()
@@ -71,6 +70,15 @@ 

      # Prepare data from db for watchdog emailing

      data_too_old = CachedData.query.filter(CachedData.time_created <= cache_cutoff).all()

  

+     # Orphaned counters

+     if app.config["ACTIVE_WATCHDOG"]:

+         counters = Counters.query.filter(Counters.counter != 0).all()

+         for counter in counters:

+             cached_counterpart = CachedData.query.filter(CachedData.provider == counter.what).first()

+             if not cached_counterpart:

+                 counter.counter = 0

+                 db.session.commit()

+ 

      # Find out what db line id to start with

      start_point = WatchDogStartPoint.query.first()

  
@@ -149,7 +157,10 @@ 

  

          # Reset counter if ACTIVE_WATCHDOG

          if app.config["ACTIVE_WATCHDOG"]:

-             counter = redis_conn.getset(old_data.provider, 0)

+             counter = Counters.query.with_for_update().filter_by(what=old_data.provider).first()

+             if counter:

+                 counter.counter = 0

+                 db.session.commit()

  

      try:

          if not app.config["SEND_ERROR_EMAILS"]:

Counter manipulation should be atomic.

Perf difference of sync planning (kalev, 392 packages):

  • Redis counters (current master): 18672.807ms
  • DB counters (this PR): 33152.847ms

Build succeeded.

1 new commit added

  • More WatchDog guards for in-db counters
2 months ago

Build succeeded.

1 new commit added

  • Remove now not needed try/except block
2 months ago

Build succeeded.

Build succeeded.