#3387 maintenance tasks
Closed 6 months ago by tkopecek. Opened 2 years ago by tkopecek.
tkopecek/koji maint-builder  into  master

file modified
+10 -3
@@ -81,6 +81,8 @@ 

      ServerRestart

  )

  from koji.util import dslice, dslice_ex, isSuccess, parseStatus, to_list

+ import tasks  # additional tasks not contained in this file

+ 

  

  try:

      import requests_gssapi as reqgssapi
@@ -124,9 +126,13 @@ 

      logger = logging.getLogger("koji.build")

      logger.info('Starting up')

      koji.util.setup_rlimits(options.__dict__, logger)

+     allowed_handler_types = []

+     if options.allow_maintenance_tasks:

+         allowed_handler_types.append('maintenance')

      tm = TaskManager(options, session)

-     tm.findHandlers(globals())

-     tm.findHandlers(vars(koji.tasks))

+     tm.findHandlers(globals(), types=allowed_handler_types)

+     tm.findHandlers(vars(koji.tasks), types=allowed_handler_types)

+     tm.findHandlers(vars(tasks), types=allowed_handler_types)

      if options.plugin:

          # load plugins

          pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
@@ -6496,7 +6502,8 @@ 

                  'cert': None,

                  'serverca': None,

                  'allow_noverifyssl': False,

-                 'allow_password_in_scm_url': False}

+                 'allow_password_in_scm_url': False,

+                 'allow_maintenance_tasks': False}

      if config.has_section('kojid'):

          for name, value in config.items('kojid'):

              if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',

@@ -0,0 +1,18 @@ 

+ import pkgutil

+ import importlib

+ import inspect

+ import koji

+ 

+ __all__ = []

+ 

+ # py3 only

+ for loader, name, is_pkg in pkgutil.walk_packages(__path__):

+     spec = loader.find_spec(name)

+     module = importlib.util.module_from_spec(spec)

+     spec.loader.exec_module(module)

+ 

+     for name, entry in inspect.getmembers(module):

+         if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and \

+                 issubclass(entry, koji.tasks.BaseTaskHandler):

+             globals()[name] = entry

+             __all__.append(name)

@@ -0,0 +1,17 @@ 

+ import pkgutil

+ import importlib

+ import inspect

+ import koji

+ 

+ __all__ = []

+ 

+ for loader, name, is_pkg in pkgutil.walk_packages(__path__):

+     spec = loader.find_spec(name)

+     module = importlib.util.module_from_spec(spec)

+     spec.loader.exec_module(module)

+ 

+     for name, entry in inspect.getmembers(module):

+         if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and \

+                 issubclass(entry, koji.tasks.BaseTaskHandler):

+             globals()[name] = entry

+             __all__.append(name)

@@ -0,0 +1,74 @@ 

+ import errno

+ import koji

+ import time

+ import os

+ from koji.tasks import BaseTaskHandler

+ from koji.util import multi_fnmatch, rmtree

+ 

+ """

+ # we completely remove those that are old enough

+ # scratch directories are /mnt/brew/scratch/$username/task_$taskid/

+ # note that $username might contain a slash (e.g. host principals)

+ find /mnt/brew/scratch/ -mindepth 2 -type d -name 'task_*' -prune -mtime +21 -exec rm -rf {} \;

+ 

+ # For content besides srpms/logs/poms we prune much more aggressively

+ # note that this step normally alters the mtime of the task dir, effectively

+ # adding to the above retention window.

+ for taskdir in $(find /mnt/brew/scratch/ -mindepth 2 -type d -name 'task_*' -prune -mtime +14)

+ do

+     find "$taskdir" -type f \! -name '*.src.rpm' \! -name '*.log' \! -name '*.pom' -delete

+ done

+ 

+ find /mnt/brew/scratch/ -maxdepth 1 -type d -mtime +1 -empty -delete

+ """

+ 

+ 

+ class CleanScratchTask(BaseTaskHandler):

+     HandlerType = 'maintenance'

+     Methods = ['maintCleanScratch']

+     _taskWeight = 0.2

+ 

+     def handler(self):

+         scratch_dir = koji.pathinfo.scratch()

+         if not os.access(scratch_dir, os.R_OK | os.W_OK | os.X_OK):

+             raise koji.ConfigurationError(

+                 f"This builder doesn't have RW access to scratch dir {scratch_dir}")

+ 

+         # we completely remove those that are old enough

+         # scratch directories are /mnt/brew/scratch/$username/task_$taskid/

+         # note that $username might contain a slash (e.g. host principals)

+         now = time.time()

+         prune_limit = now - 21 * 24 * 60 * 60

+         partial_prune_limit = now - 14 * 24 * 60 * 60

+         partial_prune_list = ['.src.rpm', '.log', '.pom']

+         empty_userdir_limit = 1 * 24 * 60 * 60

+ 

+         for userdir in os.listdir(scratch_dir):

+             fuserdir = os.path.join(scratch_dir, userdir)

+             empty_userdir = True

+             for taskdir in os.listdir(fuserdir):

+                 empty_userdir = False

+                 if not taskdir.startswith('task_'):

+                     # skip anything not produced by kojid

+                     pass

+                 ftaskdir = os.path.join(fuserdir, taskdir)

+                 mtime = os.path.getmtime(ftaskdir)

+                 if mtime < prune_limit:

+                     # delete old task directories

+                     rmtree(ftaskdir)

+                 elif mtime < partial_prune_limit:

+                     # delete most of the content except srpms, logs, ...

+                     for root, _, files in os.walk(ftaskdir):

+                         files = [f for f in files if multi_fnmatch(f, partial_prune_list)]

+                         for f in files:

+                             fpath = os.path.join(root, f)

+                             if os.path.getmtime(fpath) < partial_prune_limit:

+                                 os.unlink(fpath)

+             # remove userdir if it is empty for some time

+             if empty_userdir and os.path.getmtime(fuserdir) < empty_userdir_limit:

+                 try:

+                     os.rmdir(fuserdir)

+                 except OSError as ex:

+                     # there could be a race condition that some scratch build is being created

+                     if ex.errno != errno.ENOTEMPTY:

+                         raise

file modified
+8 -4
@@ -712,17 +712,21 @@ 

          self.start_ts = self.session.getSessionInfo()['start_ts']

          self.logger = logging.getLogger("koji.TaskManager")

  

-     def findHandlers(self, vars):

+     def findHandlers(self, vars, types=None):

          """Find and index task handlers"""

          for v in vars.values():

-             self.registerHandler(v)

+             self.registerHandler(v, types=types)

  

-     def registerHandler(self, entry):

+     def registerHandler(self, entry, types=None):

          """register and index task handler"""

          if isinstance(entry, type(koji.tasks.BaseTaskHandler)) and \

                  issubclass(entry, koji.tasks.BaseTaskHandler):

              for method in entry.Methods:

-                 self.handlers[method] = entry

+                 # add allowed types and all non-restricted tasks

+                 # currently there is only 'maintenance' handler type

+                 # 'image' or 'vm' could be other filters

+                 if not hasattr(entry, 'HandlerType') or entry.HandlerType in types:

+                     self.handlers[method] = entry

  

      def registerCallback(self, entry):

          """register and index callback plugins"""

PoC for maintenance tasks. Privileged builder with e.g. RW access for
/mnt/koji could do GC, run simplified kojira tasks, etc. Cron scripts
just trigger new task and it will be processed by standard koji
mechanisms than running separate tool on some other machine. In future
delayed tasks can be used, so cron/external machinery could be
completely dropped out.

We need special builder setting to be able to take up such jobs -
allow_maintenance_tasks here. Maybe we will need additional
permissions for maintenance builders. These could be set up with regular
permissions mechanism for builder user.

Sample maintenance tasks included - rewrite of koji-prune-scratch from
koji-tools.

rebased onto 08a059b3afec9e00afc73ff1e0419e6131cbf314

2 years ago

rebased onto c994c46dda8324e654ae20fee66efc3fbac9d669

2 years ago

rebased onto 67a9ca3

2 years ago

pretty please pagure-ci rebuild

2 years ago

I'm really hesitant to move towards setting up a privileged builder this way.

I see the point in recycling the task framework and the kojid code, but I'd like to have much stronger separation between these sorts of tasks and the normal ones.

It might be best to see how the task scheduler shakes out before we go down this sort of road.

Pull-Request has been closed by tkopecek

6 months ago