From 441518daa4d4437524cec76f625d031c7bfe062b Mon Sep 17 00:00:00 2001 From: FrantiĊĦek Zatloukal Date: May 17 2021 10:57:32 +0000 Subject: Packages endpoint --- diff --git a/alembic/versions/270f52aeacca_packages_endpoint.py b/alembic/versions/270f52aeacca_packages_endpoint.py new file mode 100644 index 0000000..2064c59 --- /dev/null +++ b/alembic/versions/270f52aeacca_packages_endpoint.py @@ -0,0 +1,34 @@ +"""Packages endpoint + +Revision ID: 270f52aeacca +Revises: 8da50b403664 +Create Date: 2021-05-05 11:07:11.390492 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '270f52aeacca' +down_revision = '8da50b403664' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('dashboard_user_package_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('package_name', sa.Text(), nullable=True), + sa.Column('last_accessed', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('package_name') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('dashboard_user_package_data') + # ### end Alembic commands ### diff --git a/oraculum/cli.py b/oraculum/cli.py index baffcb1..ec30a7b 100644 --- a/oraculum/cli.py +++ b/oraculum/cli.py @@ -76,7 +76,7 @@ def sync_users(): for user in users: print("Recreating static info cache for: %s" % user) - CACHE._refresh("packager-dashboard_user_data_static", user) + CACHE._refresh("packager-dashboard_data_static", user) def recreate_cache(): print("Refreshing DB Cache") diff --git a/oraculum/controllers/main.py b/oraculum/controllers/main.py index ce6a624..d20db3b 100644 --- a/oraculum/controllers/main.py +++ b/oraculum/controllers/main.py @@ -28,7 +28,7 @@ from oraculum.action_providers import ACTION_PROVIDERS from oraculum.controllers.landing_page import api_v1_landing_page from oraculum.controllers.libkarma import all_bodhi_updates -from oraculum.controllers.packager_dashboard import dashboard_user_data_static, handle_orphan_user +from oraculum.controllers.packager_dashboard import dashboard_data_static, handle_orphan_user @app.before_first_request @@ -66,7 +66,7 @@ def register_cache_providers(): CACHE.register('package_calendars', calendars.fetch_calendars) CACHE.register('package_versions_generic', versions.prepare_package_versions_dataset) - CACHE.register('packager-dashboard_user_data_static', dashboard_user_data_static) + CACHE.register('packager-dashboard_data_static', dashboard_data_static) CACHE.register('packager-dashboard__all_package_bugs', bugzilla.get_all_package_bugs) CACHE.register('packager-dashboard_bugs', bugzilla.get_package_bugs) CACHE.register('packager-dashboard_bugs_private', bugzilla.get_package_bugs_private) diff --git a/oraculum/controllers/packager_dashboard.py b/oraculum/controllers/packager_dashboard.py index 099eb38..335254b 100644 --- a/oraculum/controllers/packager_dashboard.py +++ b/oraculum/controllers/packager_dashboard.py @@ -55,6 +55,19 @@ def handle_orphan_user(): def data_caching_condition(): return jsonify({"visits_required_every_n_days": app.config["ACTIVITY_REQUIRED"]}) +@app.route('/api/v1/packager_dashboard/package/', methods=['GET']) +def route_dashboard_package_data(package): + package = dashboard_helpers.clean_fas_username(package) + static_info = {'status': 200, 'data': dashboard_data_static(item=package, kind="package"), + 'last_synced': datetime.utcnow().isoformat()} + + dashboard_helpers.update_package_access_time(package) + + prs = dashboard_data_prs(package, kind="package") + bzs = dashboard_data_bzs(package, kind="package", authenticated=is_packager()) + + return jsonify({'static_info': static_info, 'prs': prs, 'bzs': bzs}) + @app.route('/api/v1/packager_dashboard/', methods=['GET']) def route_dashboard_user_data(user): if user == 'orphan': @@ -63,36 +76,30 @@ def route_dashboard_user_data(user): # Cleanup the username to follow FAS naming restrictions user = dashboard_helpers.clean_fas_username(user) - packages_promise = CACHE.async_get('packager-dashboard_user_data_static', 'high', user) - last_synced = CACHE.get_refreshed_time('packager-dashboard_user_data_static', user) + packages_promise = CACHE.async_get('packager-dashboard_data_static', 'high', user) + last_synced = CACHE.get_refreshed_time('packager-dashboard_data_static', user) if packages_promise == cache_utils.RefresherNotRegistered: static_info = {'status': 404, 'data': None, 'last_synced': None} elif (packages_promise == cache_utils.AsyncRefreshInProgress or (last_synced and (last_synced + timedelta(seconds=app.config["STATIC_INFO_ROT_AFTER"]) <= datetime.utcnow()))): - celery_utils.plan_celery_refresh('high', 'packager-dashboard_user_data_static', user) + celery_utils.plan_celery_refresh('high', 'packager-dashboard_data_static', user) static_info = {'status': 204, 'data': None, 'last_synced': last_synced} else: static_info = {'status': 200, 'data': packages_promise, 'last_synced': last_synced.isoformat()} if len(static_info["data"]["packages"]) > 0: dashboard_helpers.update_user_access_time(user) - prs = dashboard_user_data_prs(user) - try: - fas_groups = current_user.fas_groups or [] - except AttributeError: - # Hit when user isn't logged in - fas_groups = [] - - if "packager" in fas_groups: - bzs = dashboard_user_data_bzs(user, authenticated=True) - else: - bzs = dashboard_user_data_bzs(user, authenticated=False) + prs = dashboard_data_prs(user, kind="user") + bzs = dashboard_data_bzs(user, kind="user", authenticated=is_packager()) return jsonify({'static_info': static_info, 'prs': prs, 'bzs': bzs}) -def dashboard_user_data_static(user): - packages = pagure.get_packages(user, CACHE.get('packages_owners_json'), CACHE.get('pagure_groups')) +def dashboard_data_static(item, kind="user"): + if kind == "user": + packages = pagure.get_packages(item, CACHE.get('packages_owners_json'), CACHE.get('pagure_groups')) + else: + packages = {"combined": [item], "group": [], "primary": [item]} # Just throw out empty lists and dicts for users without any packages if len(packages["combined"]) == 0: return { @@ -120,26 +127,32 @@ def dashboard_user_data_static(user): 'package_versions': versions.get_packages_versions(packages["combined"]) } - -def dashboard_user_data_prs(user): - data = CACHE.async_get('packager-dashboard_user_data_static', 'low', user) - if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: - return {"status": 204, "data": None} +def dashboard_data_prs(item, kind="user"): + if kind == "user": + data = CACHE.async_get('packager-dashboard_data_static', 'low', item) + if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: + return {"status": 204, "data": None} + packages = data["packages"] + else: + packages = [item] status = 200 - data, load_status = CACHE.async_get_batch('packager-dashboard_pull_requests', data["packages"], 'low') + data, load_status = CACHE.async_get_batch('packager-dashboard_pull_requests', packages, 'low') if load_status in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: status = 202 return {"status": status, "data": data} -def dashboard_user_data_bzs(user, authenticated=False): - data = CACHE.async_get('packager-dashboard_user_data_static', 'low', user) - if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: - return {"status": 204, "data": None} +def dashboard_data_bzs(item, kind="user", authenticated=False): + if kind == "user": + data = CACHE.async_get('packager-dashboard_data_static', 'low', item) + if data in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: + return {"status": 204, "data": None} + packages = data["packages"] + else: + packages = [item] status = 200 - packages = data["packages"] data, load_status = CACHE.async_get_batch('packager-dashboard_bugs', packages, 'low') if load_status in [cache_utils.RefresherNotRegistered, cache_utils.AsyncRefreshInProgress]: @@ -160,6 +173,16 @@ def dashboard_user_data_bzs(user, authenticated=False): return {"status": status, "data": data} + +def is_packager(): + try: + fas_groups = current_user.fas_groups or [] + except AttributeError: + # Hit when user isn't logged in + fas_groups = [] + + return "packager" in fas_groups + @app.route('/api/v1/package_versions/', methods=['GET']) def route_package_versions(package): return(jsonify(versions.get_package_versions(package))) diff --git a/oraculum/models/dashboard_user_packages.py b/oraculum/models/dashboard_user_packages.py new file mode 100644 index 0000000..9ca6403 --- /dev/null +++ b/oraculum/models/dashboard_user_packages.py @@ -0,0 +1,34 @@ +# +# dashboard_user_packages.py - Database model for Packager Dashboard user package info +# +# Copyright 2020, Red Hat, Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Frantisek Zatloukal + +import datetime + +from oraculum import db + +class DashboardUserPackageData(db.Model): + id = db.Column(db.Integer, primary_key=True) + package_name = db.Column(db.Text, unique=True) + last_accessed = db.Column(db.DateTime, unique=False) + + def __init__(self, package_name, last_accessed=None): + self.package_name = package_name + self.last_accessed = last_accessed or datetime.datetime.utcnow() diff --git a/oraculum/utils/celery_utils.py b/oraculum/utils/celery_utils.py index e23c7d8..95b4349 100644 --- a/oraculum/utils/celery_utils.py +++ b/oraculum/utils/celery_utils.py @@ -31,6 +31,7 @@ import oraculum from oraculum import app, celery_app from oraculum.action_providers import ACTION_PROVIDERS from oraculum.models.dashboard_users import DashboardUserData +from oraculum.models.dashboard_user_packages import DashboardUserPackageData from oraculum.utils.watchdog_utils import push_to_watchdog, process_queue @celery_app.task @@ -136,34 +137,46 @@ def get_users_for_sync(): """ Returns list of usernames to be included in sync if conditions specified in settings.py are met """ - now = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED']) - row = DashboardUserData.query.filter(DashboardUserData.last_accessed >= now) + cache_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED']) + row = DashboardUserData.query.filter(DashboardUserData.last_accessed >= cache_cutoff) return [user.username for user in row] +def get_packages_for_sync(): + """ + Returns list of packages to be included in sync if conditions specified in settings.py are met + """ + cache_cutoff = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED']) + row = DashboardUserPackageData.query.filter(DashboardUserPackageData.last_accessed >= cache_cutoff) + return [package.package_name for package in row] + @celery_app.task def celery_sync_static_user_data(): users = get_users_for_sync() for user in users: - plan_celery_refresh("medium", 'packager-dashboard_user_data_static', user) + plan_celery_refresh("medium", 'packager-dashboard_data_static', user) @celery_app.task def celery_sync_package_bugs(): users = get_users_for_sync() + packages_for_sync = get_packages_for_sync() # Bump counter to decide which sync mode we should use (full/partial) redis_conn = celery_app.broker_connection().default_channel.client counter = redis_conn.incr("bz_sync_mode_counter") tasks_pool = [] + packages = set(packages_for_sync) for user in users: - data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user) - for package in data["packages"]: - task = plan_celery_refresh("low", 'packager-dashboard__all_package_bugs', package) - # when task is None, the refresh is already planned, and will run in the future anyway - if task is not None: - tasks_pool.append(task) + data = oraculum.controllers.packager_dashboard.dashboard_data_static(user) + packages.update(data["packages"]) + + for package in packages: + task = plan_celery_refresh("low", 'packager-dashboard__all_package_bugs', package) + # when task is None, the refresh is already planned, and will run in the future anyway + if task is not None: + tasks_pool.append(task) if not tasks_pool: return @@ -182,16 +195,18 @@ def celery_sync_package_bugs(): sleep(app.config["BZ_SYNC_SLEEP"]) break - for user in users: - data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user) - for package in data["packages"]: - plan_celery_refresh("low", 'packager-dashboard_bugs', package) - plan_celery_refresh("low", 'packager-dashboard_bugs_private', package) + for package in packages: + plan_celery_refresh("low", 'packager-dashboard_bugs', package) + plan_celery_refresh("low", 'packager-dashboard_bugs_private', package) @celery_app.task def celery_sync_package_prs(): users = get_users_for_sync() + packages = set(get_packages_for_sync()) + for user in users: - data = oraculum.controllers.packager_dashboard.dashboard_user_data_static(user) - for package in data["packages"]: - plan_celery_refresh("low", 'packager-dashboard_pull_requests', package) + data = oraculum.controllers.packager_dashboard.dashboard_data_static(user) + packages.update(data["packages"]) + + for package in packages: + plan_celery_refresh("low", 'packager-dashboard_pull_requests', package) diff --git a/oraculum/utils/dashboard_helpers.py b/oraculum/utils/dashboard_helpers.py index eae4fce..8398582 100644 --- a/oraculum/utils/dashboard_helpers.py +++ b/oraculum/utils/dashboard_helpers.py @@ -35,6 +35,19 @@ from json import JSONDecodeError from oraculum import app, db, CACHE from oraculum.models.dashboard_users import DashboardUserData +from oraculum.models.dashboard_user_packages import DashboardUserPackageData + +def update_package_access_time(package): + """ + Updates user last_accessed with current timestamp + """ + row = DashboardUserPackageData.query.filter_by(package_name=package).first() + if not row: + row = DashboardUserPackageData(package, None) + db.session.add(row) + else: + row.last_accessed = datetime.datetime.utcnow() + db.session.commit() def update_user_access_time(user): """ diff --git a/oraculum/utils/watchdog_utils.py b/oraculum/utils/watchdog_utils.py index 967d219..7c9ebb0 100644 --- a/oraculum/utils/watchdog_utils.py +++ b/oraculum/utils/watchdog_utils.py @@ -62,7 +62,7 @@ def process_queue(): cached_users = get_users_for_sync() cached_packages = set() for cached_user in cached_users: - packages = CACHE.get("packager-dashboard_user_data_static", cached_user)["packages"] + packages = CACHE.get("packager-dashboard_data_static", cached_user)["packages"] cached_packages.update(packages) # Calculate time for "how old is too old" @@ -136,7 +136,7 @@ def process_queue(): for old_data in data_too_old: # Filter data we don't cache (inactive users and their packages) - if "packager-dashboard_user_data_static" in old_data.provider: + if "packager-dashboard_data_static" in old_data.provider: # Regexp down gets args from "what" # Eg. packager-dashboard_bugs [('fedora-easy-karma',), {}] > 'fedora-easy-karma' if re.search(r"\[\('(.*)',\)", old_data.provider).groups()[0] not in cached_users: