#582 Improvements on the "fedora-hubs" command-line entry point
Merged 6 years ago by abompard. Opened 6 years ago by abompard.
abompard/fedora-hubs feature/command-line  into  develop

@@ -1,6 +1,6 @@ 

  web: /usr/bin/flask-3 run --host 0.0.0.0 --port 5000

- triage: fedora-hubs triage

- worker: fedora-hubs worker

+ triage: fedora-hubs run triage

+ worker: fedora-hubs run worker

  sse: /usr/bin/twistd-3 -l - --pidfile= -n hubs-sse

  fedmsg_hub: /usr/bin/fedmsg-hub-3

  fedmsg_relay: /usr/bin/fedmsg-relay-3

file removed
-46
@@ -1,46 +0,0 @@ 

- #!/usr/bin/env python

- """ cache coverage checker.

- 

- Runs through all possible cache keys and checks if there is a corresponding

- value already cached for it.

- 

- Useful as a sanity check during development.

- 

- Could make a good nagios check one day.  If the cache coverage is below 50%,

- then the site will be very slow.  We should run some sort of script that

- forcibly re-warms the cache.

- 

- Authors:    Ralph Bean <rbean@redhat.com>

- """

- 

- from __future__ import unicode_literals, print_function

- 

- import sys

- 

- import hubs.app

- import hubs.widgets

- import hubs.models

- from hubs.database import Session

- 

- db = Session()

- empty, full = 0, 0

- 

- # Register widgets

- hubs.widgets.registry.register_list(hubs.app.app.config["WIDGETS"])

- 

- for w_instance in db.query(hubs.models.Widget).all():

-     if not w_instance.enabled:

-         continue

-     for fn_name, fn_class in w_instance.module.get_cached_functions().items():

-         if fn_class(w_instance).is_cached():

-             full += 1

-         else:

-             empty += 1

-         sys.stdout.write('.')

-         sys.stdout.flush()

- sys.stdout.write('\n')

- sys.stdout.flush()

- 

- total = empty + full

- print(full, "cache values found. ", empty, "are missing.")

- print(full / float(total) * 100, "percent cache coverage.")

@@ -4,7 +4,7 @@ 

  Documentation=https://pagure.io/fedora-hubs/

  

  [Service]

- ExecStart=/usr/bin/fedora-hubs triage

+ ExecStart=/usr/bin/fedora-hubs run triage

  EnvironmentFile=/etc/sysconfig/fedora-hubs

  Type=simple

  User=hubs

@@ -4,7 +4,7 @@ 

  Documentation=https://pagure.io/fedora-hubs/

  

  [Service]

- ExecStart=/usr/bin/fedora-hubs worker

+ ExecStart=/usr/bin/fedora-hubs run worker

  EnvironmentFile=/etc/sysconfig/fedora-hubs

  Type=simple

  User=hubs

file modified
+1 -1
@@ -1,2 +1,2 @@ 

- HUBS_CONFIG=/etc/fedora-hubs/config.py

+ # HUBS_CONFIG=/etc/fedora-hubs/hubs.py

  LOGGING_CONFIG=/etc/fedora-hubs/logging.ini

file modified
+1 -3
@@ -155,8 +155,6 @@ 

  # Environment file

  mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig

  cat > $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/%{name} << EOF

- USER=%{username}

- GROUP=%{username}

  HUBS_CONFIG=%{_sysconfdir}/%{name}/hubs.py

  LOGGING_CONFIG=%{_sysconfdir}/%{name}/logging.ini

  EOF
@@ -210,7 +208,7 @@ 

  

  %files

  %license LICENSE

- %doc README.rst

+ %doc README.rst deploy/nginx.conf

  %{python3_sitelib}/*

  %{_unitdir}/*.service

  %{_bindir}/fedora-hubs

file modified
+2
@@ -31,6 +31,8 @@ 

  app.config.from_object('hubs.default_config')

  if 'HUBS_CONFIG' in os.environ:

      app.config.from_envvar('HUBS_CONFIG')

+ elif os.path.exists("/etc/fedora-hubs/hubs.py"):

+     app.config.from_pyfile("/etc/fedora-hubs/hubs.py")

  

  fedmsg_config = get_fedmsg_config()

  

file modified
+31 -5
@@ -2,7 +2,8 @@ 

  

  from hubs.backend.triage import main as triage

  from hubs.backend.worker import main as worker

- from hubs.fas.scripts import create_team_from_fas, sync_teams_from_fas

+ from hubs.fas.scripts import create_team, sync_teams

+ from hubs.utils.cache import cache_list, cache_clean, coverage

  

  

  @click.group()
@@ -10,7 +11,32 @@ 

      pass

  

  

- cli.add_command(triage)

- cli.add_command(worker)

- cli.add_command(create_team_from_fas)

- cli.add_command(sync_teams_from_fas)

+ @cli.group()

+ def run():

+     """Run daemon processes."""

+     pass

+ 

+ 

+ run.add_command(triage)

+ run.add_command(worker)

+ 

+ 

+ @cli.group()

+ def cache():

+     """Cache-related operations."""

+     pass

+ 

+ 

+ cache.add_command(cache_list)

+ cache.add_command(cache_clean)

+ cache.add_command(coverage)

+ 

+ 

+ @cli.group()

+ def fas():

+     """FAS-related operations."""

+     pass

+ 

+ 

+ fas.add_command(create_team)

+ fas.add_command(sync_teams)

file modified
+10 -10
@@ -13,19 +13,19 @@ 

  from .api import sync_team_hub, sync_team_hub_roles

  

  

- @click.command("create-team-from-fas")

+ @click.command("create-team")

  @click.argument("name")

- def create_team_from_fas(hub_name):

+ def create_team(name):

      """Create the team hub NAME from FAS."""

      fedmsg_config = get_fedmsg_config()

      hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri'])

-     hub = Hub.by_name(hub_name, "team")

+     hub = Hub.by_name(name, "team")

      if hub is not None:

          print("This hub already exists.")

          return

      fas_client = FASClient()

      try:

-         fas_client.group_by_name(hub_name)

+         fas_client.group_by_name(name)

      except AppError:

          print("Could not find this group in FAS.")

          return
@@ -33,24 +33,24 @@ 

      with hubs.app.app.app_context():

          db = hubs.database.Session()

          hubs.app.create_task_queue()

-         Hub.create_group_hub(hub_name, "")

+         Hub.create_group_hub(name, "")

          db.commit()

-     print("Hub {} created!".format(hub_name))

+     print("Hub {} created!".format(name))

      print("It will be synced from FAS in the background.")

  

  

- @click.command("sync-teams-from-fas")

+ @click.command("sync-teams")

  @click.option("--roles/--no-roles", default=False, help="Sync the roles.")

  @click.argument("name", nargs=-1)

- def sync_teams_from_fas(roles, names):

+ def sync_teams(roles, name):

      """Sync all the team hubs NAMEs from FAS."""

      fedmsg_config = get_fedmsg_config()

      hubs.database.init(fedmsg_config['hubs.sqlalchemy.uri'])

-     if not names:

+     if not name:

          hubs_to_sync = Hub.query.filter_by(hub_type="team").all()

      else:

          hubs_to_sync = []

-         for hub_name in names:

+         for hub_name in name:

              hub = hubs.models.Hub.by_name(hub_name, "team")

              if hub is None:

                  print("The team hub {} does not exist.".format(hub_name))

file modified
+80
@@ -7,6 +7,7 @@ 

  

  from __future__ import unicode_literals

  

+ import click

  import dogpile

  import dogpile.cache

  
@@ -28,3 +29,82 @@ 

  

  

  cache = _get_cache()

+ 

+ 

+ def _setup_widgets_registry():

+     import hubs.app

+     from hubs.widgets import registry

+     registry.register_list(hubs.app.app.config["WIDGETS"])

+ 

+ 

+ @click.command("list")

+ def cache_list():

+     """List widgets for which there is cached data."""

+     from hubs.models import Widget

+ 

+     _setup_widgets_registry()

+ 

+     for w_instance in Widget.query.all():

+         if not w_instance.enabled:

+             continue

+         widget = w_instance.module

+         for fn_name, fn_class in widget.get_cached_functions().items():

+             if fn_class(w_instance).is_cached():

+                 print(('- Widget cached {inst.hub_id} (#{inst.idx}) '

+                        'in {inst.plugin}').format(inst=w_instance))

+ 

+ 

+ @click.command("clean")

+ @click.argument("widget", nargs=-1)

+ def cache_clean(widget):

+     """Clean the specified WIDGETs (id or name)."""

+     from hubs.models import Widget

+ 

+     _setup_widgets_registry()

+ 

+     count = 0

+     for w in widget:

+         widget_instance = Widget.query.get(w)

+ 

+         if widget_instance is None:

+             widget_instances = Widget.query.filter_by(

+                 plugin=w).all()

+         else:

+             widget_instances = [widget_instance]

+ 

+         if not widget_instances:

+             print('No widget found for {0}'.format(w))

+ 

+         for widget_instance in widget_instances:

+             print('- Removing cached {0} (#{1}) in {2}'.format(

+                   widget_instance.hub_id,

+                   widget_instance.idx,

+                   widget_instance.plugin))

+             functions = widget_instance.module.get_cached_functions().values()

+             for fn_class in functions:

+                 fn_class(widget_instance).invalidate()

+             count += 1

+     print("Cleaned {} widget caches.".format(count))

+ 

+ 

+ @click.command()

+ def coverage():

+     """Check the cache coverage."""

+     from hubs.models import Widget

+ 

+     _setup_widgets_registry()

+     empty, full = 0, 0

+ 

+     for w_instance in Widget.query.all():

+         if not w_instance.enabled:

+             continue

+         functions = w_instance.module.get_cached_functions()

+         for fn_name, fn_class in functions.items():

+             if fn_class(w_instance).is_cached():

+                 full += 1

+             else:

+                 empty += 1

+     total = empty + full

+     print("{full} cached values found, {empty} are missing.".format(

+           full=full, empty=empty))

+     print("{:.2f} percent cache coverage.".format(full / float(total) * 100))

@@ -1,113 +0,0 @@ 

- #!/usr/bin/env python

- 

- """

- Small script to remove the cache of one or more widgets while keeping the

- other ones un-touched.

- 

- This is really useful for development purposes as it allow seeing changes

- made to a widget without dropping the entire cache database.

- """

- 

- from __future__ import print_function, unicode_literals

- 

- import argparse

- 

- import hubs.app

- import hubs.models

- import hubs.widgets.base

- from hubs.database import Session

- 

- 

- db = Session()

- # Register widgets

- hubs.widgets.registry.register_list(hubs.app.app.config["WIDGETS"])

- 

- 

- def do_list(args):

-     ''' List the different widget for which there is data cached. '''

-     for w_instance in db.query(hubs.models.Widget).all():

-         if not w_instance.enabled:

-             continue

-         widget = w_instance.module

-         for fn_name, fn_class in widget.get_cached_functions().items():

-             if fn_class(w_instance).is_cached():

-                 print(('- Widget cached {inst.hub_id} (#{inst.idx}) '

-                        'in {inst.plugin}').format(inst=w_instance))

- 

- 

- def do_clean(args):

-     ''' Clean the widget for which there is data cached. '''

-     for widget in args.widgets:

-         widget_instance = hubs.models.Widget.get(widget)

- 

-         if widget_instance is None:

-             widget_instances = db.query(hubs.models.Widget).filter_by(

-                 plugin=widget).all()

-         else:

-             widget_instances = [widget_instance]

- 

-         if not widget_instances:

-             print('No widget found for {0}'.format(widget))

- 

-         for widget_instance in widget_instances:

-             print('- Removing cached {0} (#{1}) in {2}'.format(

-                   widget_instance.hub_id,

-                   widget_instance.idx,

-                   widget_instance.plugin))

-             functions = widget_instance.module.get_cached_functions().values()

-             for fn_class in functions:

-                 fn_class(widget_instance).invalidate()

- 

- 

- def setup_parser():

-     '''

-     Set the main arguments.

-     '''

-     parser = argparse.ArgumentParser(prog="smart_cache_invalidator")

-     subparsers = parser.add_subparsers(title='actions')

- 

-     # List

-     parser_list = subparsers.add_parser(

-         'list',

-         help='List the different widgets for which there is cached data')

-     parser_list.set_defaults(func=do_list)

- 

-     # Clean

-     parser_clean = subparsers.add_parser(

-         'clean', help='Clean the specified widget')

-     parser_clean.add_argument(

-         'widgets', nargs="+",

-         help="Identifier or name of the one or moe widgets to clean")

-     parser_clean.set_defaults(func=do_clean)

- 

-     return parser

- 

- 

- def main():

-     ''' Main function '''

-     return_code = 0

- 

-     # Set up parser for global args

-     parser = setup_parser()

- 

-     # Parse the commandline

-     try:

-         arg = parser.parse_args()

-     except argparse.ArgumentTypeError as err:

-         print("\nError: {0}".format(err))

-         return 1

- 

-     try:

-         arg.func(arg)

-     except KeyboardInterrupt:

-         print("\nInterrupted by user.")

-         return_code = 2

-     except Exception as err:

-         print('Error: {0}'.format(err))

-         return_code = 3

- 

-     return return_code

- 

- 

- if __name__ == '__main__':

-     main()

file modified
+1 -1
@@ -4,7 +4,7 @@ 

  Documentation=https://pagure.io/fedora-hubs/

  

  [Service]

- ExecStart=/usr/bin/fedora-hubs triage

+ ExecStart=/usr/bin/fedora-hubs run triage

  WorkingDirectory=/srv/hubs/fedora-hubs/

  Environment=HUBS_CONFIG=/srv/hubs/fedora-hubs/config

  Type=simple

file modified
+1 -1
@@ -4,7 +4,7 @@ 

  Documentation=https://pagure.io/fedora-hubs/

  

  [Service]

- ExecStart=/usr/bin/fedora-hubs worker

+ ExecStart=/usr/bin/fedora-hubs run worker

  Environment=HUBS_CONFIG=/srv/git/fedora-hubs/config

  Type=simple

  User=root