#154 Packages endpoint
Merged 2 years ago by frantisekz. Opened 2 years ago by frantisekz.

@@ -0,0 +1,34 @@ 

+ """Packages endpoint

+ 

+ Revision ID: 270f52aeacca

+ Revises: 8da50b403664

+ Create Date: 2021-05-05 11:07:11.390492

+ 

+ """

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ # revision identifiers, used by Alembic.

+ revision = '270f52aeacca'

+ down_revision = '8da50b403664'

+ branch_labels = None

+ depends_on = None

+ 

+ 

+ def upgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.create_table('dashboard_user_package_data',

+     sa.Column('id', sa.Integer(), nullable=False),

+     sa.Column('package_name', sa.Text(), nullable=True),

+     sa.Column('last_accessed', sa.DateTime(), nullable=True),

+     sa.PrimaryKeyConstraint('id'),

+     sa.UniqueConstraint('package_name')

+     )

+     # ### end Alembic commands ###

+ 

+ 

+ def downgrade():

+     # ### commands auto generated by Alembic - please adjust! ###

+     op.drop_table('dashboard_user_package_data')

+     # ### end Alembic commands ###

file modified
+1 -1
@@ -76,7 +76,7 @@ 

  

      for user in users:

          print("Recreating static info cache for: %s" % user)

-         CACHE._refresh("packager-dashboard_user_data_static", user)

+         CACHE._refresh("packager-dashboard_data_static", user)

  

  def recreate_cache():

      print("Refreshing DB Cache")

file modified
+2 -2
@@ -28,7 +28,7 @@ 

  

  from oraculum.controllers.landing_page import api_v1_landing_page

  from oraculum.controllers.libkarma import all_bodhi_updates

- from oraculum.controllers.packager_dashboard import dashboard_user_data_static, handle_orphan_user

+ from oraculum.controllers.packager_dashboard import dashboard_data_static, handle_orphan_user

  

  

  @app.before_first_request
@@ -66,7 +66,7 @@ 

      CACHE.register('package_calendars', calendars.fetch_calendars)

      CACHE.register('package_versions_generic', versions.prepare_package_versions_dataset)

  

-     CACHE.register('packager-dashboard_user_data_static', dashboard_user_data_static)

+     CACHE.register('packager-dashboard_data_static', dashboard_data_static)

      CACHE.register('packager-dashboard__all_package_bugs', bugzilla.get_all_package_bugs)

      CACHE.register('packager-dashboard_bugs', bugzilla.get_package_bugs)

      CACHE.register('packager-dashboard_bugs_private', bugzilla.get_package_bugs_private)

@@ -55,6 +55,19 @@ 

  def data_caching_condition():

      return jsonify({"visits_required_every_n_days": app.config["ACTIVITY_REQUIRED"]})

  

+ @app.route('/api/v1/packager_dashboard/package/<package>', methods=['GET'])

+ def route_dashboard_package_data(package):

+     package = dashboard_helpers.clean_fas_username(package)

+     static_info = {'status': 200, 'data': dashboard_data_static(item=package, kind="package"),

+                    'last_synced': datetime.utcnow().isoformat()}

+ 

+     dashboard_helpers.update_package_access_time(package)

+ 

+     prs = dashboard_data_prs(package, kind="package")

+     bzs = dashboard_data_bzs(package, kind="package", authenticated=is_packager())

+ 

+     return jsonify({'static_info': static_info, 'prs': prs, 'bzs': bzs})

+ 

  @app.route('/api/v1/packager_dashboard/<user>', methods=['GET'])

  def route_dashboard_user_data(user):

      if user == 'orphan':
@@ -63,36 +76,30 @@ 

      # Cleanup the username to follow FAS naming restrictions

      user = dashboard_helpers.clean_fas_username(user)

  

-     packages_promise = CACHE.async_get('packager-dashboard_user_data_static', 'high', user)

-     last_synced = CACHE.get_refreshed_time('packager-dashboard_user_data_static', user)

+     packages_promise = CACHE.async_get('packager-dashboard_data_static', 'high', user)

+     last_synced = CACHE.get_refreshed_time('packager-dashboard_data_static', user)

      if packages_promise == cache_utils.RefresherNotRegistered:

          static_info = {'status': 404, 'data': None, 'last_synced': None}

      elif (packages_promise == cache_utils.AsyncRefreshInProgress or

           (last_synced and (last_synced + timedelta(seconds=app.config["STATIC_INFO_ROT_AFTER"]) <= datetime.utcnow()))):

-         celery_utils.plan_celery_refresh('high', 'packager-dashboard_user_data_static', user)

+         celery_utils.plan_celery_refresh('high', 'packager-dashboard_data_static', user)

          static_info = {'status': 204, 'data': None, 'last_synced': last_synced}

      else:

          static_info = {'status': 200, 'data': packages_promise, 'last_synced': last_synced.isoformat()}

          if len(static_info["data"]["packages"]) > 0:

              dashboard_helpers.update_user_access_time(user)

  

-     prs = dashboard_user_data_prs(user)

-     try:

-         fas_groups = current_user.fas_groups or []

-     except AttributeError:

-         # Hit when user isn't logged in

-         fas_groups = []

- 

-     if "packager" in fas_groups:

-         bzs = dashboard_user_data_bzs(user, authenticated=True)

-     else:

-         bzs = dashboard_user_data_bzs(user, authenticated=False)

+     prs = dashboard_data_prs(user, kind="user")

+     bzs = dashboard_data_bzs(user, kind="user", authenticated=is_packager())

  

      return jsonify({'static_info': static_info, 'prs': prs, 'bzs': bzs})

  

  

- def dashboard_user_data_static(user):

-     packages = pagure.get_packages(user, CACHE.get('packages_owners_json'), CACHE.get('pagure_groups'))

+ def dashboard_data_static(item, kind="user"):

+     if kind == "user":

+         packages = pagure.get_packages(item, CACHE.get('packages_owners_json'), CACHE.get('pagure_groups'))

+     else:

+         packages = {"combined": [item], "group": [], "primary": [item]}

      # Just throw out empty lists and dicts for users without any packages

      if len(packages["combined"]) == 0:

          return {
@@ -120,26 +127,32 @@ 

          'package_versions': versions.get_packages_versions(packages["combined"])

      }

  

- 

- def dashboard_user_data_prs(user):

-     data = CACHE.async_get('packager-dashboard_user_data_static', 'low', user)

-     if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:

-         return {"status": 204, "data": None}

+ def dashboard_data_prs(item, kind="user"):

+     if kind == "user":

+         data = CACHE.async_get('packager-dashboard_data_static', 'low', item)

+         if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:

+             return {"status": 204, "data": None}

+         packages = data["packages"]

+     else:

+         packages = [item]

  

      status = 200

-     data, load_status = CACHE.async_get_batch('packager-dashboard_pull_requests', data["packages"], 'low')

+     data, load_status = CACHE.async_get_batch('packager-dashboard_pull_requests', packages, 'low')

      if load_status in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:

          status = 202

      return {"status": status, "data": data}

  

  

- def dashboard_user_data_bzs(user, authenticated=False):

-     data = CACHE.async_get('packager-dashboard_user_data_static', 'low', user)

-     if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:

-         return {"status": 204, "data": None}

+ def dashboard_data_bzs(item, kind="user", authenticated=False):

+     if kind == "user":

+         data = CACHE.async_get('packager-dashboard_data_static', 'low', item)

+         if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:

+             return {"status": 204, "data": None}

+         packages = data["packages"]

+     else:

+         packages = [item]

  

      status = 200

-     packages = data["packages"]

  

      data, load_status = CACHE.async_get_batch('packager-dashboard_bugs', packages, 'low')

      if load_status in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]:
@@ -160,6 +173,16 @@ 

  

      return {"status": status, "data": data}

  

+ 

+ def is_packager():

+     try:

+         fas_groups = current_user.fas_groups or []

+     except AttributeError:

+         # Hit when user isn't logged in

+         fas_groups = []

+ 

+     return "packager" in fas_groups

+ 

  @app.route('/api/v1/package_versions/<package>', methods=['GET'])

  def route_package_versions(package):

      return(jsonify(versions.get_package_versions(package)))

@@ -0,0 +1,34 @@ 

+ #

+ # dashboard_user_packages.py - Database model for Packager Dashboard user package info

+ #

+ # Copyright 2020, Red Hat, Inc

+ #

+ # This program is free software; you can redistribute it and/or modify

+ # it under the terms of the GNU General Public License as published by

+ # the Free Software Foundation; either version 2 of the License, or

+ # (at your option) any later version.

+ #

+ # This program is distributed in the hope that it will be useful,

+ # but WITHOUT ANY WARRANTY; without even the implied warranty of

+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ # GNU General Public License for more details.

+ #

+ # You should have received a copy of the GNU General Public License along

+ # with this program; if not, write to the Free Software Foundation, Inc.,

+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

+ #

+ # Authors:

+ #   Frantisek Zatloukal <fzatlouk@redhat.com>

+ 

+ import datetime

+ 

+ from oraculum import db

+ 

+ class DashboardUserPackageData(db.Model):

+     id = db.Column(db.Integer, primary_key=True)

+     package_name = db.Column(db.Text, unique=True)

+     last_accessed = db.Column(db.DateTime, unique=False)

+ 

+     def __init__(self, package_name, last_accessed=None):

+         self.package_name = package_name

+         self.last_accessed = last_accessed or datetime.datetime.utcnow()

file modified
+32 -17
@@ -31,6 +31,7 @@ 

  from oraculum import app, celery_app

  from oraculum.action_providers import ACTION_PROVIDERS

  from oraculum.models.dashboard_users import DashboardUserData

+ from oraculum.models.dashboard_user_packages import DashboardUserPackageData

  from oraculum.utils.watchdog_utils import push_to_watchdog, process_queue

  

  @celery_app.task
@@ -136,34 +137,46 @@ 

      """

      Returns list of usernames to be included in sync if conditions specified in settings.py are met

      """

-     now = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED'])

-     row = DashboardUserData.query.filter(DashboardUserData.last_accessed >= now)

+     cache_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED'])

+     row = DashboardUserData.query.filter(DashboardUserData.last_accessed >= cache_cutoff)

      return [user.username for user in row]

  

+ def get_packages_for_sync():

+     """

+     Returns list of packages to be included in sync if conditions specified in settings.py are met

+     """

+     cache_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED'])

+     row = DashboardUserPackageData.query.filter(DashboardUserPackageData.last_accessed >= cache_cutoff)

+     return [package.package_name for package in row]

+ 

  

  @celery_app.task

  def celery_sync_static_user_data():

      users = get_users_for_sync()

      for user in users:

-         plan_celery_refresh("medium", 'packager-dashboard_user_data_static', user)

+         plan_celery_refresh("medium", 'packager-dashboard_data_static', user)

  

  @celery_app.task

  def celery_sync_package_bugs():

      users = get_users_for_sync()

+     packages_for_sync = get_packages_for_sync()

  

      # Bump counter to decide which sync mode we should use (full/partial)

      redis_conn = celery_app.broker_connection().default_channel.client

      counter = redis_conn.incr("bz_sync_mode_counter")

  

      tasks_pool = []

+     packages = set(packages_for_sync)

  

      for user in users:

-         data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user)

-         for package in data["packages"]:

-             task = plan_celery_refresh("low", 'packager-dashboard__all_package_bugs', package)

-             # when task is None, the refresh is already planned, and will run in the future anyway

-             if task is not None:

-                 tasks_pool.append(task)

+         data = oraculum.controllers.packager_dashboard.dashboard_data_static(user)

+         packages.update(data["packages"])

+ 

+     for package in packages:

+         task = plan_celery_refresh("low", 'packager-dashboard__all_package_bugs', package)

+         # when task is None, the refresh is already planned, and will run in the future anyway

+         if task is not None:

+             tasks_pool.append(task)

  

      if not tasks_pool:

          return
@@ -182,16 +195,18 @@ 

                  sleep(app.config["BZ_SYNC_SLEEP"])

                  break

  

-     for user in users:

-         data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user)

-         for package in data["packages"]:

-             plan_celery_refresh("low", 'packager-dashboard_bugs', package)

-             plan_celery_refresh("low", 'packager-dashboard_bugs_private', package)

+     for package in packages:

+         plan_celery_refresh("low", 'packager-dashboard_bugs', package)

+         plan_celery_refresh("low", 'packager-dashboard_bugs_private', package)

  

  @celery_app.task

  def celery_sync_package_prs():

      users = get_users_for_sync()

+     packages = set(get_packages_for_sync())

+ 

      for user in users:

-         data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user)

-         for package in data["packages"]:

-             plan_celery_refresh("low", 'packager-dashboard_pull_requests', package)

+         data = oraculum.controllers.packager_dashboard.dashboard_data_static(user)

+         packages.update(data["packages"])

+ 

+     for package in packages:

+         plan_celery_refresh("low", 'packager-dashboard_pull_requests', package)

@@ -35,6 +35,19 @@ 

  

  from oraculum import app, db, CACHE

  from oraculum.models.dashboard_users import DashboardUserData

+ from oraculum.models.dashboard_user_packages import DashboardUserPackageData

+ 

+ def update_package_access_time(package):

+     """

+     Updates user last_accessed with current timestamp

+     """

+     row = DashboardUserPackageData.query.filter_by(package_name=package).first()

+     if not row:

+         row = DashboardUserPackageData(package, None)

+         db.session.add(row)

+     else:

+         row.last_accessed = datetime.datetime.utcnow()

+     db.session.commit()

  

  def update_user_access_time(user):

      """

@@ -62,7 +62,7 @@ 

      cached_users = get_users_for_sync()

      cached_packages = set()

      for cached_user in cached_users:

-         packages = CACHE.get("packager-dashboard_user_data_static", cached_user)["packages"]

+         packages = CACHE.get("packager-dashboard_data_static", cached_user)["packages"]

          cached_packages.update(packages)

  

      # Calculate time for "how old is too old"
@@ -136,7 +136,7 @@ 

  

      for old_data in data_too_old:

          # Filter data we don't cache (inactive users and their packages)

-         if "packager-dashboard_user_data_static" in old_data.provider:

+         if "packager-dashboard_data_static" in old_data.provider:

              # Regexp down gets args from "what"

              # Eg. packager-dashboard_bugs [('fedora-easy-karma',), {}] > 'fedora-easy-karma'

              if re.search(r"\[\('(.*)',\)", old_data.provider).groups()[0] not in cached_users:

no initial comment

Build succeeded.

1 new commit added

  • Better
2 years ago

2 new commits added

  • Better
  • [WIP] Packages endpoint
2 years ago

Build succeeded.

1 new commit added

  • Even more better
2 years ago

Build succeeded.

1 new commit added

  • Even more better
2 years ago

Build succeeded.

rebased onto 441518d

2 years ago

Build succeeded.

Pull-Request has been merged by frantisekz

2 years ago