#1087 frontend: python3-flask-script-removal - other commands
Merged 4 years ago by thrnciar. Opened 4 years ago by thrnciar.

@@ -1,31 +0,0 @@ 

- from coprs import db

- from coprs.logic import coprs_logic, packages_logic, actions_logic

- from commands.rawhide_to_release import RawhideToReleaseCommand

- 

- 

- class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):

- 

-     "Copy backend data of the latest successful rawhide builds into a new chroot"

- 

-     def run(self, rawhide_chroot, dest_chroot):

-         for copr in coprs_logic.CoprsLogic.get_all():

-             if not self.has_rawhide(copr):

-                 continue

- 

-             data = {"copr": copr.name,

-                     "user": copr.owner_name,

-                     "rawhide_chroot": rawhide_chroot,

-                     "dest_chroot": dest_chroot,

-                     "builds": []}

- 

-             for package in packages_logic.PackagesLogic.get_all(copr.id):

-                 last_build = package.last_build(successful=True)

-                 if last_build:

-                     data["builds"].append(last_build.result_dir)

- 

-             if len(data["builds"]):

-                 actions_logic.ActionsLogic.send_rawhide_to_release(data)

-                 print("Created copy action from {}/{} to {}/{}"

-                       .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot))

- 

-         db.session.commit()

@@ -1,15 +1,14 @@ 

- from flask_script import Command

+ import click

  from coprs import db_session_scope

  from coprs.logic.complex_logic import ComplexLogic

  

  

- class CleanExpiredProjectsCommand(Command):

+ @click.command()

+ def clean_expired_projects():

      """

      Clean all the expired temporary projects.  This command is meant to be

      executed by cron.

      """

  

-     # pylint: disable=method-hidden

-     def run(self):

-         with db_session_scope():

-             ComplexLogic.delete_expired_projects()

+     with db_session_scope():

+         ComplexLogic.delete_expired_projects()

@@ -1,15 +1,13 @@ 

- from flask_script import Command

+ import click

  from coprs import db_session_scope

  from coprs.logic.builds_logic import BuildsLogic

  

  

- class DeleteOldBuilds(Command):

+ @click.command()

+ def clean_old_builds():

      """

      This garbage collects all builds which are "obsoleted" per user

      configuration, per models.Package.max_builds configuration.

      """

- 

-     # pylint: disable=method-hidden

-     def run(self):

-         with db_session_scope():

-             BuildsLogic.clean_old_builds()

+     with db_session_scope():

+         BuildsLogic.clean_old_builds()

@@ -1,13 +1,12 @@ 

- from flask_script import Command

+ import click

  from coprs.logic.builds_logic import BuildsLogic

  from coprs.logic.packages_logic import PackagesLogic

  

  

- class DeleteOrphansCommand(Command):

+ @click.command()

+ def delete_orphans():

      """

      Deletes builds and packages associated to deleted coprs.

      """

- 

-     def run(self):

-         BuildsLogic.delete_orphaned_builds()

-         PackagesLogic.delete_orphaned_packages()

+     BuildsLogic.delete_orphaned_builds()

+     PackagesLogic.delete_orphaned_packages()

@@ -1,30 +1,31 @@ 

- from flask_script import Command, Option

+ import click

  from coprs import db

  from coprs.logic import coprs_logic, actions_logic

  

  

- class DeleteOutdatedChrootsCommand(Command):

+ @click.command()

+ @click.option(

+     "--dry-run/--no-dry-run",

+     default=False,

+     help="Do not actually remove any data, but rather print information on stdout"

+ )

+ def delete_outdated_chroots(dry_run):

      """

      Delete data in all chroots that are considered as outdated. That means, the chroot is EOL

      and the preservation period is over because admin of the project didn't extend its duration.

      """

-     option_list = [

-         Option("--dry-run", action="store_true",

-                help="Do not actually remove any data, but rather print information on stdout"),

-     ]

- 

-     def run(self, dry_run):

-         deleter = DryRunDeleter() if dry_run else Deleter()

- 

-         chroots = coprs_logic.CoprChrootsLogic \

-             .filter_outdated_to_be_deleted(coprs_logic.CoprChrootsLogic.get_multiple())

-         for i, chroot in enumerate(chroots, start=1):

-             # This command will possibly delete a lot of chroots and can be a performance issue when committing

-             # all at once. We are going to commit every x actions to avoid that.

-             if i % 1000 == 0:

-                 deleter.commit()

-             deleter.delete(chroot)

-         deleter.commit()

+ 

+     deleter = DryRunDeleter() if dry_run else Deleter()

+ 

+     chroots = coprs_logic.CoprChrootsLogic \

+         .filter_outdated_to_be_deleted(coprs_logic.CoprChrootsLogic.get_multiple())

+     for i, chroot in enumerate(chroots, start=1):

+         # This command will possibly delete a lot of chroots and can be a performance issue when committing

+         # all at once. We are going to commit every x actions to avoid that.

+         if i % 1000 == 0:

+             deleter.commit()

+         deleter.delete(chroot)

+     deleter.commit()

  

  

  class Deleter(object):

@@ -1,23 +1,21 @@ 

  import sqlalchemy

- from flask_script import Command, Option

+ import click

  from coprs import db

  from coprs.logic import builds_logic

  

  

- class FailBuildCommand(Command):

- 

+ @click.command()

+ @click.argument("build_id", type=int)

+ def fail_build(build_id):

      """

      Marks build as failed on all its non-finished chroots

      """

  

-     option_list = [Option("build_id")]

- 

-     def run(self, build_id, **kwargs):

-         try:

-             builds_logic.BuildsLogic.mark_as_failed(build_id)

-             print("Marking non-finished chroots of build {} as failed".format(build_id))

-             db.session.commit()

+     try:

+         builds_logic.BuildsLogic.mark_as_failed(build_id)

+         print("Marking non-finished chroots of build {} as failed".format(build_id))

+         db.session.commit()

  

-         except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e:

-             print("Error: No such build {}".format(build_id))

-             return 1

+     except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e:

+         print("Error: No such build {}".format(build_id))

+         return 1

@@ -1,9 +1,7 @@ 

- from flask_script import Command

+ import click

  from coprs import models

  

- 

- class GetAdminsCommand(Command):

- 

-     def run(self, **kwargs):

-         for u in models.User.query.filter(models.User.admin == True).all():

-             print(u.username)

+ @click.command()

+ def get_admins():

+     for u in models.User.query.filter(models.User.admin == True).all():

+         print(u.username)

@@ -1,74 +1,77 @@ 

  import sys

  import datetime

- from flask_script import Command, Option

+ import click

  from coprs import db, app

  from coprs.logic import coprs_logic

  from coprs.mail import send_mail, OutdatedChrootMessage

  

  

- class NotifyOutdatedChrootsCommand(Command):

+ @click.command()

+ @click.option(

+     "--dry-run/--no-dry-run",

+     default=False,

+     help="Do not actually notify the people, but rather print information on stdout"

+ )

+ @click.option(

+     "--email", "-e", "email_filter",

+     help="Notify only "

+ )

+ @click.option(

+     "--all/--not-all",

+     default=False,

+     help="Notify all (even the recently notified) relevant people"

+ )

+ def notify_outdated_chroots(dry_run, email_filter, all):

      """

      Notify all admins of projects with builds in outdated chroots about upcoming deletion.

      """

-     option_list = [

-         Option("--dry-run", action="store_true",

-                help="Do not actually notify the people, but rather print information on stdout"),

-         Option("-e", "--email", action="append", dest="email_filter",

-                help="Notify only "),

-         Option("-a", "--all", action="store_true",

-                help="Notify all (even the recently notified) relevant people"),

-     ]

- 

-     def run(self, dry_run, email_filter, all):

-         self.email_filter = email_filter

-         self.all = all

- 

-         if not dry_run:

-             self.dev_instance_warning()

- 

-         notifier = DryRunNotifier() if dry_run else Notifier()

-         outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple())

-         for user, chroots in self.get_user_chroots_map(outdated).items():

-             chroots = self.filter_chroots([chroot for chroot in chroots])

-             if not chroots:

-                 continue

-             chroots.sort(key=lambda x: x.copr.full_name)

-             notifier.notify(user, chroots)

-             notifier.store_timestamp(chroots)

  

-     def get_user_chroots_map(self, chroots):

-         user_chroot_map = {}

-         for chroot in chroots:

-             for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr):

-                 if self.email_filter and admin.mail not in self.email_filter:

-                     continue

-                 if admin not in user_chroot_map:

-                     user_chroot_map[admin] = []

-                 user_chroot_map[admin].append(chroot)

-         return user_chroot_map

- 

-     def filter_chroots(self, chroots):

-         if self.all:

-             return chroots

- 

-         filtered = []

-         for chroot in chroots:

-             if not chroot.delete_notify:

-                 filtered.append(chroot)

+     if not dry_run:

+         dev_instance_warning(email_filter)

+ 

+     notifier = DryRunNotifier() if dry_run else Notifier()

+     outdated = coprs_logic.CoprChrootsLogic.filter_outdated(coprs_logic.CoprChrootsLogic.get_multiple())

+     for user, chroots in get_user_chroots_map(outdated, email_filter).items():

+         chroots = filter_chroots([chroot for chroot in chroots], all)

+         if not chroots:

+             continue

+         chroots.sort(key=lambda x: x.copr.full_name)

+         notifier.notify(user, chroots)

+         notifier.store_timestamp(chroots)

+ 

+ def get_user_chroots_map(chroots, email_filter):

+     user_chroot_map = {}

+     for chroot in chroots:

+         for admin in coprs_logic.CoprPermissionsLogic.get_admins_for_copr(chroot.copr):

+             if email_filter and admin.mail not in email_filter:

                  continue

- 

-             # Skip the chroot if was notified in less than `n` days

-             now = datetime.datetime.now()

-             if (now - chroot.delete_notify).days >= 80:

-                 filtered.append(chroot)

- 

-         return filtered

- 

-     def dev_instance_warning(self):

-         if app.config["ENV"] != "production" and not self.email_filter:

-             sys.stderr.write("I will not let you send emails to all Copr users from the dev instance!\n")

-             sys.stderr.write("Please use this command with -e myself@foo.bar\n")

-             sys.exit(1)

+             if admin not in user_chroot_map:

+                 user_chroot_map[admin] = []

+             user_chroot_map[admin].append(chroot)

+     return user_chroot_map

+ 

+ def filter_chroots(chroots, all):

+     if all:

+         return chroots

+ 

+     filtered = []

+     for chroot in chroots:

+         if not chroot.delete_notify:

+             filtered.append(chroot)

+             continue

+ 

+         # Skip the chroot if was notified in less than `n` days

+         now = datetime.datetime.now()

+         if (now - chroot.delete_notify).days >= 80:

+             filtered.append(chroot)

+ 

+     return filtered

+ 

+ def dev_instance_warning(email_filter):

+     if app.config["ENV"] != "production" and not email_filter:

+         sys.stderr.write("I will not let you send emails to all Copr users from the dev instance!\n")

+         sys.stderr.write("Please use this command with -e myself@foo.bar\n")

+         sys.exit(1)

  

  

  class Notifier(object):

@@ -1,13 +1,12 @@ 

- from flask_script import Command

+ import click

  from coprs.logic import builds_logic

  

  

- class UpdateGraphsDataCommand(Command):

+ @click.command()

+ def update_graphs():

      """

      Generates newest graph data.

      """

- 

-     def run(self):

-         builds_logic.BuildsLogic.get_task_graph_data('10min')

-         builds_logic.BuildsLogic.get_small_graph_data('30min')

-         builds_logic.BuildsLogic.get_task_graph_data('24h')

+     builds_logic.BuildsLogic.get_task_graph_data('10min')

+     builds_logic.BuildsLogic.get_small_graph_data('30min')

+     builds_logic.BuildsLogic.get_task_graph_data('24h')

@@ -1,23 +1,21 @@ 

+ import click

  import time

  from sqlalchemy import and_, or_

- from flask_script import Command

  from coprs import db

  from coprs import models

  

- 

- class RemoveGraphsDataCommand(Command):

-     """

-     Removes old cached graph data that is no longer used.

-     """

- 

-     def run(self):

-         curr_time = int(time.time())

-         models.BuildsStatistics.query.filter(or_(

-             and_(models.BuildsStatistics.time < curr_time - 91 * 86400,

-                  models.BuildsStatistics.stat_type == '24h'),

-             and_(models.BuildsStatistics.time < curr_time - 87000,

-                  models.BuildsStatistics.stat_type == '30min'),

-             and_(models.BuildsStatistics.time < curr_time - 87000,

-                  models.BuildsStatistics.stat_type == '10min')

-         )).delete()

-         db.session.commit()

+ @click.command()

+ def vacuum_graphs():

+      """

+      Removes old cached graph data that is no longer used.

+      """

+      curr_time = int(time.time())

+      models.BuildsStatistics.query.filter(or_(

+           and_(models.BuildsStatistics.time < curr_time - 91 * 86400,

+                models.BuildsStatistics.stat_type == '24h'),

+           and_(models.BuildsStatistics.time < curr_time - 87000,

+                models.BuildsStatistics.stat_type == '30min'),

+           and_(models.BuildsStatistics.time < curr_time - 87000,

+                models.BuildsStatistics.stat_type == '10min')

+      )).delete()

+      db.session.commit()

@@ -22,79 +22,63 @@ 

  import commands.update_indexes

  import commands.update_indexes_quick

  import commands.update_indexes_required

- 

+ import commands.get_admins

+ import commands.fail_build

  import commands.rawhide_to_release

+ import commands.update_graphs

+ import commands.vacuum_graphs

+ import commands.notify_outdated_chroots

+ import commands.delete_outdated_chroots

+ import commands.clean_expired_projects

+ import commands.clean_old_builds

+ import commands.delete_orphans

  

  from flask_script import Manager

  from coprs import app

  

- 

- commands_old = {

-     # Other

-     "get_admins": "GetAdminsCommand",

-     "fail_build": "FailBuildCommand",

-     "update_graphs": "UpdateGraphsDataCommand",

-     "vacuum_graphs": "RemoveGraphsDataCommand",

-     "notify_outdated_chroots": "NotifyOutdatedChrootsCommand",

-     "delete_outdated_chroots": "DeleteOutdatedChrootsCommand",

-     "clean_expired_projects": "CleanExpiredProjectsCommand",

-     "clean_old_builds": "DeleteOldBuilds",

-     "delete_orphans": "DeleteOrphansCommand",

- }

- 

  if os.getuid() == 0:

      sys.stderr.write("Please don't run this script as a 'root' user, use:\n")

      sys.stderr.write("$ sudo -u copr-fe {}\n".format(

              ' '.join([pipes.quote(arg) for arg in sys.argv])))

      sys.exit(1)

  

- manager = Manager(app)

- for cmdname, clsname in commands_old.items():

-     module = importlib.import_module("commands.{0}".format(cmdname))

-     cls = getattr(module, clsname)

-     manager.add_command(cmdname, cls())

- 

-     # General commands

-     app.cli.add_command(commands.test.test, "test")

- 

-     # Database commands

-     app.cli.add_command(commands.create_sqlite_file.create_sqlite_file_command, "create_sqlite_file")

-     app.cli.add_command(commands.create_db.create_db, "create_db")

-     app.cli.add_command(commands.drop_db.drop_db, "drop_db")

- 

-     # Chroot commands

-     app.cli.add_command(commands.create_chroot.create_chroot_command, "create_chroot")

-     app.cli.add_command(commands.alter_chroot.alter_chroot, "alter_chroot")

-     app.cli.add_command(commands.display_chroots.display_chroots, "display_chroots")

-     app.cli.add_command(commands.drop_chroot.drop_chroot, "drop_chroot")

-     app.cli.add_command(commands.branch_fedora.branch_fedora, "branch_fedora")

-     app.cli.add_command(commands.comment_chroot.comment_chroot, "comment_chroot")

- 

-     # User commands

-     app.cli.add_command(commands.alter_user.alter_user, "alter_user")

-     app.cli.add_command(commands.add_user.add_user, "add_user")

-     app.cli.add_command(commands.dump_user.dump_user, "dump_user")

- 

-     # Whooshee indexes

-     app.cli.add_command(commands.update_indexes.update_indexes, "update_indexes")

-     app.cli.add_command(commands.update_indexes_quick.update_indexes_quick, "update_indexes_quick")

-     app.cli.add_command(commands.update_indexes_required.update_indexes_required, "update_indexes_required")

- 

- 

-     # Other

-     #TODO

-     app.cli.add_command(commands.rawhide_to_release.rawhide_to_release, "rawhide_to_release")

+ # General commands

+ app.cli.add_command(commands.test.test, "test")

+ 

+ # Database commands

+ app.cli.add_command(commands.create_sqlite_file.create_sqlite_file_command, "create_sqlite_file")

+ app.cli.add_command(commands.create_db.create_db, "create_db")

+ app.cli.add_command(commands.drop_db.drop_db, "drop_db")

+ 

+ # Chroot commands

+ app.cli.add_command(commands.create_chroot.create_chroot_command, "create_chroot")

+ app.cli.add_command(commands.alter_chroot.alter_chroot, "alter_chroot")

+ app.cli.add_command(commands.display_chroots.display_chroots, "display_chroots")

+ app.cli.add_command(commands.drop_chroot.drop_chroot, "drop_chroot")

+ app.cli.add_command(commands.branch_fedora.branch_fedora, "branch_fedora")

+ app.cli.add_command(commands.comment_chroot.comment_chroot, "comment_chroot")

+ 

+ # User commands

+ app.cli.add_command(commands.alter_user.alter_user, "alter_user")

+ app.cli.add_command(commands.add_user.add_user, "add_user")

+ app.cli.add_command(commands.dump_user.dump_user, "dump_user")

+ 

+ # Whooshee indexes

+ app.cli.add_command(commands.update_indexes.update_indexes, "update_indexes")

+ app.cli.add_command(commands.update_indexes_quick.update_indexes_quick, "update_indexes_quick")

+ app.cli.add_command(commands.update_indexes_required.update_indexes_required, "update_indexes_required")

+ 

+ # Other

+ app.cli.add_command(commands.get_admins.get_admins, "get_admins")

+ app.cli.add_command(commands.fail_build.fail_build, "fail_build")

+ app.cli.add_command(commands.rawhide_to_release.rawhide_to_release, "rawhide_to_release")

+ app.cli.add_command(commands.update_graphs.update_graphs, "update_graphs")

+ app.cli.add_command(commands.vacuum_graphs.vacuum_graphs, "vacuum_graphs")

+ app.cli.add_command(commands.notify_outdated_chroots.notify_outdated_chroots, "notify_outdated_chroots")

+ app.cli.add_command(commands.delete_outdated_chroots.delete_outdated_chroots, "delete_outdated_chroots")

+ app.cli.add_command(commands.clean_expired_projects.clean_expired_projects, "clean_expired_projects")

+ app.cli.add_command(commands.clean_old_builds.clean_old_builds, "clean_old_builds")

+ app.cli.add_command(commands.delete_orphans.delete_orphans, "delete_orphans")

  

  if __name__ == "__main__":

-     # This is just temporary while migrating to flask script,

-     # values in arrays are already migrated parameters.

-     # Else part will be removed once migration is complete.

-     if sys.argv[1] in [

-         'test', 'create_sqlite_file', 'create_db', 'drop_db',

-         'create_chroot', 'alter_chroot', 'display_chroots', 'drop_chroot',

-         'branch_fedora', 'comment_chroot', 'rawhide_to_release',

- 	    'alter_user', 'add_user', 'dump_user', 'update_indexes',

- 	    'update_indexes_quick', 'update_indexes_required']:

-         app.cli()

-     else:

-         manager.run()

+     app.cli()

get_admins.py - done
fail_build.py - done
rawhide_to_release.py - done
backend_rawhide_to_release.py - removed
update_graphs.py - done
vacuum_graphs.py - done
notify_outdated_chroots.py - done
delete_outdated_chroots.py - done
clean_expired_projects.py - done
clean_old_builds.py - done
delete_orphans.py - done

Looks good to me. I am curious about those pylints though.

rebased onto ea54cbda36f87253ef921e14cb457e86481ba415

4 years ago

Does this achieve anything?

@frostyx, seems like it were leftovers from old code. I have removed it.

rebased onto 6095c51

4 years ago

Pull-Request has been merged by thrnciar

4 years ago