From 94b2a55d57c32b3ae2ff5d7da2ec7e6f280305cd Mon Sep 17 00:00:00 2001 From: Lukas Brabec Date: Oct 12 2022 10:05:35 +0000 Subject: add sync button and logs to admin view --- diff --git a/alembic/versions/358443d0fb4c_add_lock_table.py b/alembic/versions/358443d0fb4c_add_lock_table.py new file mode 100644 index 0000000..4590c95 --- /dev/null +++ b/alembic/versions/358443d0fb4c_add_lock_table.py @@ -0,0 +1,32 @@ +"""add Lock table + +Revision ID: 358443d0fb4c +Revises: f8c29e9793f4 +Create Date: 2022-08-01 20:41:59.824132 + +""" + +# revision identifiers, used by Alembic. +revision = '358443d0fb4c' +down_revision = 'f8c29e9793f4' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('lock', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=1024), nullable=True), + sa.Column('locked', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_lock')), + sa.UniqueConstraint('name', name=op.f('uq_lock_name')) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('lock') + # ### end Alembic commands ### diff --git a/blockerbugs/cli.py b/blockerbugs/cli.py index c7870b7..4a0d0b7 100644 --- a/blockerbugs/cli.py +++ b/blockerbugs/cli.py @@ -13,6 +13,7 @@ from blockerbugs.util.update_sync import UpdateSync from blockerbugs.util.bz_interface import BlockerBugs import blockerbugs.util.discussion_sync as discussion_sync import blockerbugs.util.pagure_bot as pagure_bot +from blockerbugs.util.named_lock import NamedLock, LockLocked from blockerbugs.models.bug import Bug from blockerbugs.util import testdata from alembic.config import Config as al_Config @@ -174,38 +175,53 @@ def check_blockers(): def sync_bugs(args): - docheck = args.check - fullsync = args.full - # FIXME: as a workaround for very slow standard sync times, perform a full - # sync always. If no downsides are found, completely remove the "--full" - # cmdline option (and related function arguments) and make full sync the - # only approach. - # Related: https://pagure.io/fedora-qa/blockerbugs/issue/184 - fullsync = True - - current_milestone = Milestone.query.filter_by(active=True).first() - if not current_milestone: - sys.stderr.write('BugSync ERROR: No active releases found!') + try: + with NamedLock('sync'): + docheck = args.check + fullsync = args.full + # FIXME: as a workaround for very slow standard sync times, perform a full + # sync always. If no downsides are found, completely remove the "--full" + # cmdline option (and related function arguments) and make full sync the + # only approach. + # Related: https://pagure.io/fedora-qa/blockerbugs/issue/184 + fullsync = True + + current_milestone = Milestone.query.filter_by(active=True).first() + if not current_milestone: + sys.stderr.write('BugSync ERROR: No active releases found!') + sys.exit(1) + + sync = BugSync(db) + + sync.update_active(full_sync=fullsync) + + if docheck: + print("Checking bug sync status") + check_blockers() + except LockLocked as e: + app.logger.error("Sync in progress") sys.exit(1) - sync = BugSync(db) - - sync.update_active(full_sync=fullsync) - - if docheck: - print("Checking bug sync status") - check_blockers() def sync_updates(args): - active_releases = Release.query.filter_by(active=True).all() - update_sync = UpdateSync(db) - for release in active_releases: - update_sync.sync_updates(release) - + try: + with NamedLock('sync'): + active_releases = Release.query.filter_by(active=True).all() + update_sync = UpdateSync(db) + for release in active_releases: + update_sync.sync_updates(release) + except LockLocked as e: + app.logger.error("Sync in progress") + sys.exit(1) def sync_discussions(args): - discussion_sync.sync_discussions() + try: + with NamedLock('sync'): + discussion_sync.sync_discussions() + except LockLocked as e: + app.logger.error("Sync in progress") + sys.exit(1) def sync(args): diff --git a/blockerbugs/controllers/admin/__init__.py b/blockerbugs/controllers/admin/__init__.py index af539a7..6fff8f3 100644 --- a/blockerbugs/controllers/admin/__init__.py +++ b/blockerbugs/controllers/admin/__init__.py @@ -22,21 +22,54 @@ import logging +from threading import Thread from flask import flash, request, redirect, url_for from flask_admin.babel import gettext import flask_admin from blockerbugs import app, db +from blockerbugs.cli import sync_bugs, sync_discussions, sync_updates from blockerbugs.models.release import Release from blockerbugs.models.userinfo import UserInfo from blockerbugs.models.milestone import Milestone from blockerbugs.controllers.admin.auth import FasAuthModelView +from blockerbugs.controllers.forms import SyncForm from blockerbugs.controllers.users import check_admin_rights +class DummyArgs(): + def __init__(self, check=False, full=True): + self.check=check + self.full=full + + class AdminIndexViewSqla(flask_admin.AdminIndexView): def _handle_view(self, name, **kwargs): return check_admin_rights() + @flask_admin.expose('/', methods=['GET', 'POST']) + def index(self): + syncform = SyncForm(sync_bugs=True, sync_discussions=True, sync_updates=True) + + if syncform.validate_on_submit(): + app.logger.debug('Sync requested from admin UI') + to_sync = [] + if syncform.sync_bugs.data: + app.logger.debug('Syncing bugs') + to_sync.append(sync_bugs) + if syncform.sync_discussions.data: + app.logger.debug('Syncing discussions') + to_sync.append(sync_discussions) + if syncform.sync_updates.data: + app.logger.debug('Syncing updates') + to_sync.append(sync_updates) + + sync_thread = Thread(target=lambda a: [f(a) for f in to_sync], args=[DummyArgs()]) + sync_thread.start() + + return redirect(url_for('admin.index')) # Post/Redirect/Get to avoid form resubmission + + return self.render('admin/index.html', syncform=syncform) + admin = flask_admin.Admin(app, 'Blocker Bug Tracking Admin', base_template='admin_layout.html', diff --git a/blockerbugs/controllers/api/api.py b/blockerbugs/controllers/api/api.py index 5aed174..0db1aac 100644 --- a/blockerbugs/controllers/api/api.py +++ b/blockerbugs/controllers/api/api.py @@ -20,17 +20,24 @@ """RESTful API handlers""" +from cgitb import handler from typing import Any -from flask import Blueprint, request +from flask import Blueprint, request, Response, g, abort +from flask_fas_openid import fas_login_required +import io +import logging +import time from blockerbugs import app from blockerbugs.controllers.main import update_to_milestones +from blockerbugs.controllers.users import check_admin_rights from blockerbugs.models.milestone import Milestone from blockerbugs.models.update import Update from blockerbugs.models.release import Release from blockerbugs.models.bug import Bug from blockerbugs.util import pagure_bot +from blockerbugs.util.named_lock import NamedLock from . import errors from .utils import get_or_404, JsonResponse, SVGResponse, check_signature @@ -228,3 +235,44 @@ def bug_image(bug_id): info_all.append(milestone_info + ", ".join(bugtypes)) return SVGResponse(_svg_response_text(info_all)) + + +@api_v0.route('/lock/') +def lock_locked(lock_name): + return JsonResponse({'lock_name': lock_name, 'locked': NamedLock(lock_name).locked}) + + +@api_v0.route('/lock_force_unlock/') +@fas_login_required +def lock_force_unlock(lock_name): + if not app.config['FAS_ADMIN_GROUP'] in g.fas_user.groups: + abort(401) + + lock = NamedLock(lock_name) + lock.__exit__(None, None, None) + return JsonResponse({'lock_name': lock_name, 'msg': 'forced unlock'}) + + +@api_v0.route('/logs') +@fas_login_required +def log_stream(): + if not app.config['FAS_ADMIN_GROUP'] in g.fas_user.groups: + abort(401) + + if not app.config['FILE_LOGGING'] or not app.config['LOGFILE']: + return Response("FILE_LOGGING disabled", mimetype="text/plain") + else: + def stream(): + # use logging.WatchedFileHandler's method reopenIfNeeded() to avoid + # reading from wrong logfile due to rotation of logs + # https://github.com/python/cpython/blob/3.10/Lib/logging/handlers.py#L490 + handler = logging.handlers.WatchedFileHandler(app.config['LOGFILE'], mode="r") + while True: + handler.reopenIfNeeded() + line = handler.stream.readline() + if not line: + time.sleep(0.1) + continue + yield line + + return Response(stream(), mimetype="text/plain", content_type="text/event-stream") \ No newline at end of file diff --git a/blockerbugs/controllers/forms.py b/blockerbugs/controllers/forms.py index b2ad949..1dad497 100644 --- a/blockerbugs/controllers/forms.py +++ b/blockerbugs/controllers/forms.py @@ -24,6 +24,8 @@ from wtforms import StringField, SelectField, BooleanField, IntegerField from wtforms import TextAreaField, ValidationError from wtforms.validators import DataRequired +from blockerbugs.cli import sync_discussions + def one_proposal(form, field): if not (form.freeze_exception.data or form.blocker.data): @@ -37,3 +39,14 @@ class BugProposeForm(FlaskForm): blocker = BooleanField(u'Blocker', [one_proposal]) freeze_exception = BooleanField(u'Freeze Exception', [one_proposal]) justification = TextAreaField(u'Justification', [DataRequired()]) + + +def at_least_one(form, field): + if not any([form.sync_bugs.data, form.sync_discussions.data, form.sync_updates.data]): + raise ValidationError('You must select at least one sync option') + + +class SyncForm(FlaskForm): + sync_bugs = BooleanField('Sync bugs', [at_least_one]) + sync_discussions = BooleanField('Sync discussions', [at_least_one]) + sync_updates = BooleanField('Sync updates', [at_least_one]) \ No newline at end of file diff --git a/blockerbugs/models/lock.py b/blockerbugs/models/lock.py new file mode 100644 index 0000000..b317c6b --- /dev/null +++ b/blockerbugs/models/lock.py @@ -0,0 +1,25 @@ +# Copyright 2022, Red Hat, Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Lukas Brabec + +from blockerbugs import db, BaseModel + +class Lock(BaseModel): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(1024), unique=True) + locked = db.Column(db.Boolean) diff --git a/blockerbugs/static/css/admin_layout.css b/blockerbugs/static/css/admin_layout.css index bb985ba..f68bdf7 100644 --- a/blockerbugs/static/css/admin_layout.css +++ b/blockerbugs/static/css/admin_layout.css @@ -2,3 +2,16 @@ .select2-container { width: 545px !important; } + +#logs { + height: 500px; + overflow-y: scroll; + font-family: monospace; + font-size: 9pt; + border: 1px solid #ccc; + overflow-x: scroll; + white-space: pre; + color: #fff; + background-color: #000; + padding: 0.5em; +} \ No newline at end of file diff --git a/blockerbugs/templates/admin/index.html b/blockerbugs/templates/admin/index.html new file mode 100644 index 0000000..51b1914 --- /dev/null +++ b/blockerbugs/templates/admin/index.html @@ -0,0 +1,126 @@ +{% macro form_error(element) %} {% if element.errors %} + + {% for error in element.errors %} + {{ error }} + {% endfor %} + +{% endif %} {%- endmacro %} +{% extends 'admin/master.html' %} + +{% block body %} + +
Sync DB
+ + + +
+ {{ syncform.csrf_token }} +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+
+ +
+
+ +
+
+
+ +
Log
+
+
+ +
+ +
+
+ raw +
+
+
+
+ + + +{% endblock %} \ No newline at end of file diff --git a/blockerbugs/util/named_lock.py b/blockerbugs/util/named_lock.py new file mode 100644 index 0000000..2d47249 --- /dev/null +++ b/blockerbugs/util/named_lock.py @@ -0,0 +1,59 @@ +# Copyright 2022, Red Hat, Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: +# Lukas Brabec + +from blockerbugs import db +from blockerbugs.models.lock import Lock + + +class LockLocked(Exception): + pass + + +class NamedLock(): + def __init__(self, name): + self.name = name + + def __enter__(self): + lock = Lock.query.with_for_update(of=Lock).filter_by(name=self.name).first() + if not lock: + lock = Lock(name=self.name, locked=True) + else: + if lock.locked: + raise LockLocked + lock.locked = True + + db.session.add(lock) + db.session.commit() + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type == LockLocked: + return + lock = Lock.query.with_for_update(of=Lock).filter_by(name=self.name).first() + if lock: + lock.locked = False + db.session.add(lock) + db.session.commit() + + @property + def locked(self): + lock = Lock.query.filter_by(name=self.name).first() + if not lock: + return False + else: + return lock.locked