From 6095c512ac5021920b1e3a0eedd62b0b9cee0b3a Mon Sep 17 00:00:00 2001 From: Tomas Hrnciar Date: Oct 31 2019 14:16:34 +0000 Subject: frontend: python3-flask-script-removal - other commands get_admins.py - done fail_build.py - done rawhide_to_release.py - done backend_rawhide_to_release.py - done update_graphs.py - done vacuum_graphs.py - done notify_outdated_chroots.py - done delete_outdated_chroots.py - done clean_expired_projects.py - done clean_old_builds.py - done delete_orphans.py - done --- diff --git a/frontend/coprs_frontend/commands/backend_rawhide_to_release.py b/frontend/coprs_frontend/commands/backend_rawhide_to_release.py deleted file mode 100644 index 266853a..0000000 --- a/frontend/coprs_frontend/commands/backend_rawhide_to_release.py +++ /dev/null @@ -1,31 +0,0 @@ -from coprs import db -from coprs.logic import coprs_logic, packages_logic, actions_logic -from commands.rawhide_to_release import RawhideToReleaseCommand - - -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand): - - "Copy backend data of the latest successful rawhide builds into a new chroot" - - def run(self, rawhide_chroot, dest_chroot): - for copr in coprs_logic.CoprsLogic.get_all(): - if not self.has_rawhide(copr): - continue - - data = {"copr": copr.name, - "user": copr.owner_name, - "rawhide_chroot": rawhide_chroot, - "dest_chroot": dest_chroot, - "builds": []} - - for package in packages_logic.PackagesLogic.get_all(copr.id): - last_build = package.last_build(successful=True) - if last_build: - data["builds"].append(last_build.result_dir) - - if len(data["builds"]): - actions_logic.ActionsLogic.send_rawhide_to_release(data) - print("Created copy action from {}/{} to {}/{}" - .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) - - db.session.commit() diff --git a/frontend/coprs_frontend/commands/clean_expired_projects.py b/frontend/coprs_frontend/commands/clean_expired_projects.py index cf4ce3c..e910945 100644 --- a/frontend/coprs_frontend/commands/clean_expired_projects.py +++ b/frontend/coprs_frontend/commands/clean_expired_projects.py @@ -1,15 +1,14 @@ -from flask_script import Command +import click from coprs import db_session_scope from coprs.logic.complex_logic import ComplexLogic -class CleanExpiredProjectsCommand(Command): +@click.command() +def clean_expired_projects(): """ Clean all the expired temporary projects. This command is meant to be executed by cron. """ - # pylint: disable=method-hidden - def run(self): - with db_session_scope(): - ComplexLogic.delete_expired_projects() + with db_session_scope(): + ComplexLogic.delete_expired_projects() diff --git a/frontend/coprs_frontend/commands/clean_old_builds.py b/frontend/coprs_frontend/commands/clean_old_builds.py index c312642..cc9dc14 100644 --- a/frontend/coprs_frontend/commands/clean_old_builds.py +++ b/frontend/coprs_frontend/commands/clean_old_builds.py @@ -1,15 +1,13 @@ -from flask_script import Command +import click from coprs import db_session_scope from coprs.logic.builds_logic import BuildsLogic -class DeleteOldBuilds(Command): +@click.command() +def clean_old_builds(): """ This garbage collects all builds which are "obsoleted" per user configuration, per models.Package.max_builds configuration. """ - - # pylint: disable=method-hidden - def run(self): - with db_session_scope(): - BuildsLogic.clean_old_builds() + with db_session_scope(): + BuildsLogic.clean_old_builds() diff --git a/frontend/coprs_frontend/commands/delete_orphans.py b/frontend/coprs_frontend/commands/delete_orphans.py index 3a24360..073a19c 100644 --- a/frontend/coprs_frontend/commands/delete_orphans.py +++ b/frontend/coprs_frontend/commands/delete_orphans.py @@ -1,13 +1,12 @@ -from flask_script import Command +import click from coprs.logic.builds_logic import BuildsLogic from coprs.logic.packages_logic import PackagesLogic -class DeleteOrphansCommand(Command): +@click.command() +def delete_orphans(): """ Deletes builds and packages associated to deleted coprs. """ - - def run(self): - BuildsLogic.delete_orphaned_builds() - PackagesLogic.delete_orphaned_packages() + BuildsLogic.delete_orphaned_builds() + PackagesLogic.delete_orphaned_packages() diff --git a/frontend/coprs_frontend/commands/delete_outdated_chroots.py b/frontend/coprs_frontend/commands/delete_outdated_chroots.py index 5804508..d1be287 100644 --- a/frontend/coprs_frontend/commands/delete_outdated_chroots.py +++ b/frontend/coprs_frontend/commands/delete_outdated_chroots.py @@ -1,30 +1,31 @@ -from flask_script import Command, Option +import click from coprs import db from coprs.logic import coprs_logic, actions_logic -class DeleteOutdatedChrootsCommand(Command): +@click.command() +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Do not actually remove any data, but rather print information on stdout" +) +def delete_outdated_chroots(dry_run): """ Delete data in all chroots that are considered as outdated. That means, the chroot is EOL and the preservation period is over because admin of the project didn't extend its duration. """ - option_list = [ - Option("--dry-run", action="store_true", - help="Do not actually remove any data, but rather print information on stdout"), - ] - - def run(self, dry_run): - deleter = DryRunDeleter() if dry_run else Deleter() - - chroots = coprs_logic.CoprChrootsLogic \ - .filter_outdated_to_be_deleted(coprs_logic.CoprChrootsLogic.get_multiple()) - for i, chroot in enumerate(chroots, start=1): - # This command will possibly delete a lot of chroots and can be a performance issue when committing - # all at once. We are going to commit every x actions to avoid that. - if i % 1000 == 0: - deleter.commit() - deleter.delete(chroot) - deleter.commit() + + deleter = DryRunDeleter() if dry_run else Deleter() + + chroots = coprs_logic.CoprChrootsLogic \ + .filter_outdated_to_be_deleted(coprs_logic.CoprChrootsLogic.get_multiple()) + for i, chroot in enumerate(chroots, start=1): + # This command will possibly delete a lot of chroots and can be a performance issue when committing + # all at once. We are going to commit every x actions to avoid that. + if i % 1000 == 0: + deleter.commit() + deleter.delete(chroot) + deleter.commit() class Deleter(object): diff --git a/frontend/coprs_frontend/commands/fail_build.py b/frontend/coprs_frontend/commands/fail_build.py index 92531bb..fb8ac04 100644 --- a/frontend/coprs_frontend/commands/fail_build.py +++ b/frontend/coprs_frontend/commands/fail_build.py @@ -1,23 +1,21 @@ import sqlalchemy -from flask_script import Command, Option +import click from coprs import db from coprs.logic import builds_logic -class FailBuildCommand(Command): - +@click.command() +@click.argument("build_id", type=int) +def fail_build(build_id): """ Marks build as failed on all its non-finished chroots """ - option_list = [Option("build_id")] - - def run(self, build_id, **kwargs): - try: - builds_logic.BuildsLogic.mark_as_failed(build_id) - print("Marking non-finished chroots of build {} as failed".format(build_id)) - db.session.commit() + try: + builds_logic.BuildsLogic.mark_as_failed(build_id) + print("Marking non-finished chroots of build {} as failed".format(build_id)) + db.session.commit() - except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: - print("Error: No such build {}".format(build_id)) - return 1 + except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: + print("Error: No such build {}".format(build_id)) + return 1 diff --git a/frontend/coprs_frontend/commands/get_admins.py b/frontend/coprs_frontend/commands/get_admins.py index d4a76e2..f54708a 100644 --- a/frontend/coprs_frontend/commands/get_admins.py +++ b/frontend/coprs_frontend/commands/get_admins.py @@ -1,9 +1,7 @@ -from flask_script import Command +import click from coprs import models - -class GetAdminsCommand(Command): - - def run(self, **kwargs): - for u in models.User.query.filter(models.User.admin == True).all(): - print(u.username) +@click.command() +def get_admins(): + for u in models.User.query.filter(models.User.admin == True).all(): + print(u.username) diff --git a/frontend/coprs_frontend/commands/notify_outdated_chroots.py b/frontend/coprs_frontend/commands/notify_outdated_chroots.py index 5d92ab5..95807a1 100644 --- a/frontend/coprs_frontend/commands/notify_outdated_chroots.py +++ b/frontend/coprs_frontend/commands/notify_outdated_chroots.py @@ -1,74 +1,77 @@ import sys import datetime -from flask_script import Command, Option +import click from coprs import db, app from coprs.logic import coprs_logic from coprs.mail import send_mail, OutdatedChrootMessage -class NotifyOutdatedChrootsCommand(Command): +@click.command() +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Do not actually notify the people, but rather print information on stdout" +) +@click.option( + "--email", "-e", "email_filter", + help="Notify only " +) +@click.option( + "--all/--not-all", + default=False, + help="Notify all (even the recently notified) relevant people" +) +def notify_outdated_chroots(dry_run, email_filter, all): """ Notify all admins of projects with builds in outdated chroots about upcoming deletion. """ - option_list = [ - Option("--dry-run", action="store_true", - help="Do not actually notify the people, but rather print information on stdout"), - Option("-e", "--email", action="append", dest="email_filter", - help="Notify only "), - Option("-a", "--all", action="store_true", - help="Notify all (even the recently notified) relevant people"), - ] - - def run(self, dry_run, email_filter, all): - self.email_filter = email_filter - self.all = all - - if not dry_run: - self.dev_instance_warning() - - notifier = DryRunNotifier() if dry_run else Notifier() - outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple()) - for user, chroots in self.get_user_chroots_map(outdated).items(): - chroots = self.filter_chroots([chroot for chroot in chroots]) - if not chroots: - continue - chroots.sort(key=lambda x: x.copr.full_name) - notifier.notify(user, chroots) - notifier.store_timestamp(chroots) - def get_user_chroots_map(self, chroots): - user_chroot_map = {} - for chroot in chroots: - for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr): - if self.email_filter and admin.mail not in self.email_filter: - continue - if admin not in user_chroot_map: - user_chroot_map[admin] = [] - user_chroot_map[admin].append(chroot) - return user_chroot_map - - def filter_chroots(self, chroots): - if self.all: - return chroots - - filtered = [] - for chroot in chroots: - if not chroot.delete_notify: - filtered.append(chroot) + if not dry_run: + dev_instance_warning(email_filter) + + notifier = DryRunNotifier() if dry_run else Notifier() + outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple()) + for user, chroots in get_user_chroots_map(outdated, email_filter).items(): + chroots = filter_chroots([chroot for chroot in chroots], all) + if not chroots: + continue + chroots.sort(key=lambda x: x.copr.full_name) + notifier.notify(user, chroots) + notifier.store_timestamp(chroots) + +def get_user_chroots_map(chroots, email_filter): + user_chroot_map = {} + for chroot in chroots: + for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr): + if email_filter and admin.mail not in email_filter: continue - - # Skip the chroot if was notified in less than `n` days - now = datetime.datetime.now() - if (now - chroot.delete_notify).days >= 80: - filtered.append(chroot) - - return filtered - - def dev_instance_warning(self): - if app.config["ENV"] != "production" and not self.email_filter: - sys.stderr.write("I will not let you send emails to all Copr users from the dev instance!\n") - sys.stderr.write("Please use this command with -e myself@foo.bar\n") - sys.exit(1) + if admin not in user_chroot_map: + user_chroot_map[admin] = [] + user_chroot_map[admin].append(chroot) + return user_chroot_map + +def filter_chroots(chroots, all): + if all: + return chroots + + filtered = [] + for chroot in chroots: + if not chroot.delete_notify: + filtered.append(chroot) + continue + + # Skip the chroot if was notified in less than `n` days + now = datetime.datetime.now() + if (now - chroot.delete_notify).days >= 80: + filtered.append(chroot) + + return filtered + +def dev_instance_warning(email_filter): + if app.config["ENV"] != "production" and not email_filter: + sys.stderr.write("I will not let you send emails to all Copr users from the dev instance!\n") + sys.stderr.write("Please use this command with -e myself@foo.bar\n") + sys.exit(1) class Notifier(object): diff --git a/frontend/coprs_frontend/commands/update_graphs.py b/frontend/coprs_frontend/commands/update_graphs.py index ef8337a..e59a373 100644 --- a/frontend/coprs_frontend/commands/update_graphs.py +++ b/frontend/coprs_frontend/commands/update_graphs.py @@ -1,13 +1,12 @@ -from flask_script import Command +import click from coprs.logic import builds_logic -class UpdateGraphsDataCommand(Command): +@click.command() +def update_graphs(): """ Generates newest graph data. """ - - def run(self): - builds_logic.BuildsLogic.get_task_graph_data('10min') - builds_logic.BuildsLogic.get_small_graph_data('30min') - builds_logic.BuildsLogic.get_task_graph_data('24h') + builds_logic.BuildsLogic.get_task_graph_data('10min') + builds_logic.BuildsLogic.get_small_graph_data('30min') + builds_logic.BuildsLogic.get_task_graph_data('24h') diff --git a/frontend/coprs_frontend/commands/vacuum_graphs.py b/frontend/coprs_frontend/commands/vacuum_graphs.py index ffa7256..4b66ca0 100644 --- a/frontend/coprs_frontend/commands/vacuum_graphs.py +++ b/frontend/coprs_frontend/commands/vacuum_graphs.py @@ -1,23 +1,21 @@ +import click import time from sqlalchemy import and_, or_ -from flask_script import Command from coprs import db from coprs import models - -class RemoveGraphsDataCommand(Command): - """ - Removes old cached graph data that is no longer used. - """ - - def run(self): - curr_time = int(time.time()) - models.BuildsStatistics.query.filter(or_( - and_(models.BuildsStatistics.time < curr_time - 91 * 86400, - models.BuildsStatistics.stat_type == '24h'), - and_(models.BuildsStatistics.time < curr_time - 87000, - models.BuildsStatistics.stat_type == '30min'), - and_(models.BuildsStatistics.time < curr_time - 87000, - models.BuildsStatistics.stat_type == '10min') - )).delete() - db.session.commit() +@click.command() +def vacuum_graphs(): + """ + Removes old cached graph data that is no longer used. + """ + curr_time = int(time.time()) + models.BuildsStatistics.query.filter(or_( + and_(models.BuildsStatistics.time < curr_time - 91 * 86400, + models.BuildsStatistics.stat_type == '24h'), + and_(models.BuildsStatistics.time < curr_time - 87000, + models.BuildsStatistics.stat_type == '30min'), + and_(models.BuildsStatistics.time < curr_time - 87000, + models.BuildsStatistics.stat_type == '10min') + )).delete() + db.session.commit() diff --git a/frontend/coprs_frontend/manage.py b/frontend/coprs_frontend/manage.py index b8e6d3c..61e0926 100755 --- a/frontend/coprs_frontend/manage.py +++ b/frontend/coprs_frontend/manage.py @@ -22,79 +22,63 @@ import commands.dump_user import commands.update_indexes import commands.update_indexes_quick import commands.update_indexes_required - +import commands.get_admins +import commands.fail_build import commands.rawhide_to_release +import commands.update_graphs +import commands.vacuum_graphs +import commands.notify_outdated_chroots +import commands.delete_outdated_chroots +import commands.clean_expired_projects +import commands.clean_old_builds +import commands.delete_orphans from flask_script import Manager from coprs import app - -commands_old = { - # Other - "get_admins": "GetAdminsCommand", - "fail_build": "FailBuildCommand", - "update_graphs": "UpdateGraphsDataCommand", - "vacuum_graphs": "RemoveGraphsDataCommand", - "notify_outdated_chroots": "NotifyOutdatedChrootsCommand", - "delete_outdated_chroots": "DeleteOutdatedChrootsCommand", - "clean_expired_projects": "CleanExpiredProjectsCommand", - "clean_old_builds": "DeleteOldBuilds", - "delete_orphans": "DeleteOrphansCommand", -} - if os.getuid() == 0: sys.stderr.write("Please don't run this script as a 'root' user, use:\n") sys.stderr.write("$ sudo -u copr-fe {}\n".format( ' '.join([pipes.quote(arg) for arg in sys.argv]))) sys.exit(1) -manager = Manager(app) -for cmdname, clsname in commands_old.items(): - module = importlib.import_module("commands.{0}".format(cmdname)) - cls = getattr(module, clsname) - manager.add_command(cmdname, cls()) - - # General commands - app.cli.add_command(commands.test.test, "test") - - # Database commands - app.cli.add_command(commands.create_sqlite_file.create_sqlite_file_command, "create_sqlite_file") - app.cli.add_command(commands.create_db.create_db, "create_db") - app.cli.add_command(commands.drop_db.drop_db, "drop_db") - - # Chroot commands - app.cli.add_command(commands.create_chroot.create_chroot_command, "create_chroot") - app.cli.add_command(commands.alter_chroot.alter_chroot, "alter_chroot") - app.cli.add_command(commands.display_chroots.display_chroots, "display_chroots") - app.cli.add_command(commands.drop_chroot.drop_chroot, "drop_chroot") - app.cli.add_command(commands.branch_fedora.branch_fedora, "branch_fedora") - app.cli.add_command(commands.comment_chroot.comment_chroot, "comment_chroot") - - # User commands - app.cli.add_command(commands.alter_user.alter_user, "alter_user") - app.cli.add_command(commands.add_user.add_user, "add_user") - app.cli.add_command(commands.dump_user.dump_user, "dump_user") - - # Whooshee indexes - app.cli.add_command(commands.update_indexes.update_indexes, "update_indexes") - app.cli.add_command(commands.update_indexes_quick.update_indexes_quick, "update_indexes_quick") - app.cli.add_command(commands.update_indexes_required.update_indexes_required, "update_indexes_required") - - - # Other - #TODO - app.cli.add_command(commands.rawhide_to_release.rawhide_to_release, "rawhide_to_release") +# General commands +app.cli.add_command(commands.test.test, "test") + +# Database commands +app.cli.add_command(commands.create_sqlite_file.create_sqlite_file_command, "create_sqlite_file") +app.cli.add_command(commands.create_db.create_db, "create_db") +app.cli.add_command(commands.drop_db.drop_db, "drop_db") + +# Chroot commands +app.cli.add_command(commands.create_chroot.create_chroot_command, "create_chroot") +app.cli.add_command(commands.alter_chroot.alter_chroot, "alter_chroot") +app.cli.add_command(commands.display_chroots.display_chroots, "display_chroots") +app.cli.add_command(commands.drop_chroot.drop_chroot, "drop_chroot") +app.cli.add_command(commands.branch_fedora.branch_fedora, "branch_fedora") +app.cli.add_command(commands.comment_chroot.comment_chroot, "comment_chroot") + +# User commands +app.cli.add_command(commands.alter_user.alter_user, "alter_user") +app.cli.add_command(commands.add_user.add_user, "add_user") +app.cli.add_command(commands.dump_user.dump_user, "dump_user") + +# Whooshee indexes +app.cli.add_command(commands.update_indexes.update_indexes, "update_indexes") +app.cli.add_command(commands.update_indexes_quick.update_indexes_quick, "update_indexes_quick") +app.cli.add_command(commands.update_indexes_required.update_indexes_required, "update_indexes_required") + +# Other +app.cli.add_command(commands.get_admins.get_admins, "get_admins") +app.cli.add_command(commands.fail_build.fail_build, "fail_build") +app.cli.add_command(commands.rawhide_to_release.rawhide_to_release, "rawhide_to_release") +app.cli.add_command(commands.update_graphs.update_graphs, "update_graphs") +app.cli.add_command(commands.vacuum_graphs.vacuum_graphs, "vacuum_graphs") +app.cli.add_command(commands.notify_outdated_chroots.notify_outdated_chroots, "notify_outdated_chroots") +app.cli.add_command(commands.delete_outdated_chroots.delete_outdated_chroots, "delete_outdated_chroots") +app.cli.add_command(commands.clean_expired_projects.clean_expired_projects, "clean_expired_projects") +app.cli.add_command(commands.clean_old_builds.clean_old_builds, "clean_old_builds") +app.cli.add_command(commands.delete_orphans.delete_orphans, "delete_orphans") if __name__ == "__main__": - # This is just temporary while migrating to flask script, - # values in arrays are already migrated parameters. - # Else part will be removed once migration is complete. - if sys.argv[1] in [ - 'test', 'create_sqlite_file', 'create_db', 'drop_db', - 'create_chroot', 'alter_chroot', 'display_chroots', 'drop_chroot', - 'branch_fedora', 'comment_chroot', 'rawhide_to_release', - 'alter_user', 'add_user', 'dump_user', 'update_indexes', - 'update_indexes_quick', 'update_indexes_required']: - app.cli() - else: - manager.run() + app.cli()