From aa7eb74fc718f6921713c9c5778ef7e86e1e6871 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: May 21 2019 16:14:38 +0000 Subject: PR#1327: volume option for dist-repo Merges #1327 https://pagure.io/koji/pull-request/1327 Fixes: #1366 https://pagure.io/koji/issue/1366 volume option for dist-repo --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 02c3f6b..b8a77de 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7123,6 +7123,7 @@ def handle_dist_repo(options, session, args): 'multiple times.')) parser.add_option('--event', type='int', help=_('Use tag content at event')) + parser.add_option("--volume", help=_("Generate repo on given volume")) parser.add_option('--non-latest', dest='latest', default=True, action='store_false', help='Include older builds, not just the latest') parser.add_option('--multilib', default=None, metavar="CONFIG", @@ -7207,6 +7208,7 @@ def handle_dist_repo(options, session, args): 'comps': task_opts.comps, 'delta': old_repos, 'event': task_opts.event, + 'volume': task_opts.volume, 'inherit': not task_opts.noinherit, 'latest': task_opts.latest, 'multilib': task_opts.multilib, diff --git a/hub/kojihub.py b/hub/kojihub.py index f7c8fe2..466bdcc 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2406,6 +2406,18 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): with open("%s/comps.xml" % groupsdir, 'w') as fo: fo.write(comps) + # write repo info to disk + repo_info = { + 'id': repo_id, + 'tag': tinfo['name'], + 'tag_id': tinfo['id'], + 'event_id': event_id, + 'with_src': with_src, + 'with_debuginfo': with_debuginfo, + } + with open('%s/repo.json' % repodir, 'w') as fp: + json.dump(repo_info, fp, indent=2) + #get build dirs relpathinfo = koji.PathInfo(topdir='toplink') builddirs = {} @@ -2527,6 +2539,9 @@ def dist_repo_init(tag, keys, task_opts): tinfo = get_tag(tag, strict=True) tag_id = tinfo['id'] event = task_opts.get('event') + volume = task_opts.get('volume') + if volume is not None: + volume = lookup_name('volume', volume, strict=True)['name'] arches = list(set([koji.canonArch(a) for a in task_opts['arch']])) # note: we need to match args from the other preRepoInit callback koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False, @@ -2539,15 +2554,32 @@ def dist_repo_init(tag, keys, task_opts): insert.set(id=repo_id, create_event=event, tag_id=tag_id, state=state, dist=True) insert.execute() - repodir = koji.pathinfo.distrepo(repo_id, tinfo['name']) + repodir = koji.pathinfo.distrepo(repo_id, tinfo['name'], volume=volume) for arch in arches: koji.ensuredir(os.path.join(repodir, arch)) + if volume and volume != 'DEFAULT': + # symlink from main volume to this one + basedir = koji.pathinfo.distrepo(repo_id, tinfo['name']) + relpath = os.path.relpath(repodir, os.path.dirname(basedir)) + koji.ensuredir(os.path.dirname(basedir)) + os.symlink(relpath, basedir) # handle comps if task_opts.get('comps'): groupsdir = os.path.join(repodir, 'groups') koji.ensuredir(groupsdir) shutil.copyfile(os.path.join(koji.pathinfo.work(), task_opts['comps']), groupsdir + '/comps.xml') + # write repo info to disk + repo_info = { + 'id': repo_id, + 'tag': tinfo['name'], + 'tag_id': tinfo['id'], + 'keys': keys, + 'volume': volume, + 'task_opts': task_opts, + } + with open('%s/repo.json' % repodir, 'w') as fp: + json.dump(repo_info, fp, indent=2) # note: we need to match args from the other postRepoInit callback koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False, with_debuginfo=False, event=event, repo_id=repo_id, @@ -12925,6 +12957,8 @@ class HostExports(object): workdir = koji.pathinfo.work() rinfo = repo_info(repo_id, strict=True) repodir = koji.pathinfo.distrepo(repo_id, rinfo['tag_name']) + # Note: if repo is on a different volume then repodir should be a + # valid symlink and this function should still do the right thing archdir = "%s/%s" % (repodir, koji.canonArch(arch)) if not os.path.isdir(archdir): raise koji.GenericError("Repo arch directory missing: %s" % archdir) diff --git a/koji/__init__.py b/koji/__init__.py index 0b9b5bc..09e9812 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1919,9 +1919,9 @@ class PathInfo(object): """Return the directory where a repo belongs""" return self.topdir + ("/repos/%(tag_str)s/%(repo_id)s" % locals()) - def distrepo(self, repo_id, tag): + def distrepo(self, repo_id, tag, volume=None): """Return the directory with a dist repo lives""" - return os.path.join(self.topdir, 'repos-dist', tag, str(repo_id)) + return self.volumedir(volume) + '/repos-dist/%s/%s' % (tag, repo_id) def repocache(self, tag_str): """Return the directory where a repo belongs""" diff --git a/tests/test_cli/test_dist_repo.py b/tests/test_cli/test_dist_repo.py index 0ef6d55..6d38fbd 100644 --- a/tests/test_cli/test_dist_repo.py +++ b/tests/test_cli/test_dist_repo.py @@ -266,6 +266,7 @@ Options: repo or the name of a tag that has a dist repo. May be specified multiple times. --event=EVENT Use tag content at event + --volume=VOLUME Generate repo on given volume --non-latest Include older builds, not just the latest --multilib=CONFIG Include multilib packages in the repository using the given config file diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py index e0637b1..0d25bc0 100644 --- a/tests/test_hub/test_dist_repo.py +++ b/tests/test_hub/test_dist_repo.py @@ -27,6 +27,10 @@ class TestDistRepoInit(unittest.TestCase): def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.pathinfo = koji.PathInfo(self.tempdir) + mock.patch('koji.pathinfo', new=self.pathinfo).start() + self.InsertProcessor = mock.patch('kojihub.InsertProcessor', side_effect=self.getInsert).start() self.inserts = [] @@ -34,7 +38,6 @@ class TestDistRepoInit(unittest.TestCase): self.get_tag = mock.patch('kojihub.get_tag').start() self.get_event = mock.patch('kojihub.get_event').start() self.nextval = mock.patch('kojihub.nextval').start() - self.ensuredir = mock.patch('koji.ensuredir').start() self.copyfile = mock.patch('shutil.copyfile').start() self.get_tag.return_value = {'id': 42, 'name': 'tag'} @@ -77,7 +80,6 @@ class TestDistRepoInit(unittest.TestCase): self.assertEquals(ip.data, data) self.assertEquals(ip.rawdata, {}) - # no comps option self.copyfile.assert_called_once() diff --git a/util/kojira b/util/kojira index f218655..83c31ce 100755 --- a/util/kojira +++ b/util/kojira @@ -29,10 +29,12 @@ from koji.util import rmtree, parseStatus, to_list from optparse import OptionParser from six.moves.configparser import ConfigParser import errno +import json import logging import logging.handlers import pprint import signal +import stat import time import threading import traceback @@ -97,6 +99,36 @@ class ManagedRepo(object): rinfo = self.session.repoInfo(self.repo_id) self._dist = rinfo['dist'] + def get_info(self): + "Fetch data from repo.json" + path = self.get_path() + if not path: + # can this be an error yet? + return None + fn = '%s/repo.json' % path + if not os.path.exists(fn): + self.logger.warn('Repo info file missing: %s', fn) + return None + with open(fn, 'r') as fp: + return json.load(fp) + + def get_path(self, volume=None): + """Return the path to the repo directory""" + tag_info = getTag(self.session, self.tag_id) + if not tag_info: + tag_info = getTag(self.session, self.tag_id, self.event_id) + if not tag_info: + self.logger.warn('Could not get info for tag %i, referenced by repo %i' % + (self.tag_id, self.repo_id)) + return None + tag_name = tag_info['name'] + if self.dist: + path = pathinfo.distrepo(self.repo_id, tag_name, volume=volume) + else: + # currently only dist repos can be on another volume + path = pathinfo.repo(self.repo_id, tag_name) + return path + def expire(self): """Mark the repo expired""" if self.state == koji.REPO_EXPIRED: @@ -136,19 +168,13 @@ class ManagedRepo(object): def tryDelete(self): """Remove the repo from disk, if possible""" - tag_info = getTag(self.session, self.tag_id) - if not tag_info: - tag_info = getTag(self.session, self.tag_id, self.event_id) - if not tag_info: - self.logger.warn('Could not get info for tag %i, skipping delete of repo %i' % - (self.tag_id, self.repo_id)) + path = self.get_path() + if not path: + # get_path already warned return False - tag_name = tag_info['name'] if self.dist: - path = pathinfo.distrepo(self.repo_id, tag_name) lifetime = self.options.dist_repo_lifetime else: - path = pathinfo.repo(self.repo_id, tag_name) lifetime = self.options.deleted_repo_lifetime # (should really be called expired_repo_lifetime) try: @@ -180,7 +206,31 @@ class ManagedRepo(object): return False self.logger.info("Deleted repo %s" % self.repo_id) self.state = koji.REPO_DELETED - self.manager.rmtree(path) + if os.path.islink(path): + # expected for repos on other volumes + info = self.get_info() + if not os.path.exists(path): + self.logger.error('Repo volume link broken: %s', path) + return False + if not info or 'volume' not in info: + self.logger.error('Missing repo.json in %s', path) + return False + realpath = self.get_path(volume=info['volume']) + if not os.path.exists(realpath): + self.logger.error('Repo real path missing: %s', realpath) + return False + if not os.path.samefile(path, realpath): + self.logger.error('Incorrect volume link: %s', path) + return False + # ok, try to remove the symlink + try: + os.unlink(path) + except OSError: + self.logger.error('Unable to remove volume link: %s', path) + # and remove the real path + self.manager.rmtree(realpath) + else: + self.manager.rmtree(path) return True def ready(self): @@ -374,7 +424,15 @@ class RepoManager(object): finally: session.logout() - def pruneLocalRepos(self, topdir, timername): + def pruneLocalRepos(self): + for volinfo in self.session.listVolumes(): + volumedir = pathinfo.volumedir(volinfo['name']) + repodir = "%s/repos" % volumedir + self._pruneLocalRepos(repodir, self.options.deleted_repo_lifetime) + distrepodir = "%s/repos-dist" % volumedir + self._pruneLocalRepos(distrepodir, self.options.dist_repo_lifetime) + + def _pruneLocalRepos(self, topdir, max_age): """Scan filesystem for repos and remove any deleted ones Also, warn about any oddities""" @@ -382,14 +440,13 @@ class RepoManager(object): #skip return if not os.path.exists(topdir): - self.logger.warn("%s doesn't exist, skipping", topdir) + self.logger.debug("%s doesn't exist, skipping", topdir) return - if os.path.isfile(topdir): + if not os.path.isdir(topdir): self.logger.warn("%s is not directory, skipping", topdir) return self.logger.debug("Scanning %s for repos", topdir) - self.logger.debug('max age allowed: %s seconds (from %s)', - getattr(self.options, timername), timername) + self.logger.debug('max age allowed: %s seconds', max_age) for tag in os.listdir(topdir): tagdir = "%s/%s" % (topdir, tag) if not os.path.isdir(tagdir): @@ -404,28 +461,36 @@ class RepoManager(object): except ValueError: self.logger.debug("%s/%s not an int, skipping", tagdir, repo_id) continue - repodir = "%s/%s" % (tagdir, repo_id) - if not os.path.isdir(repodir): - self.logger.debug("%s not a directory, skipping", repodir) - continue if repo_id in self.repos: #we're already managing it, no need to deal with it here continue + repodir = "%s/%s" % (tagdir, repo_id) try: - dir_ts = os.stat(repodir).st_mtime + # lstat because it could be link to another volume + dirstat = os.lstat(repodir) except OSError: #just in case something deletes the repo out from under us self.logger.debug("%s deleted already?!", repodir) continue + symlink = False + if stat.S_ISLNK(dirstat.st_mode): + symlink = True + elif stat.S_ISDIR(dirstat.st_mode): + self.logger.debug("%s not a directory, skipping", repodir) + continue + dir_ts = dirstat.st_mtime rinfo = self.session.repoInfo(repo_id) if rinfo is None: if not self.options.ignore_stray_repos: age = time.time() - dir_ts self.logger.debug("did not expect %s; age: %s", repodir, age) - if age > getattr(self.options, timername): + if age > max_age: self.logger.info("Removing unexpected directory (no such repo): %s", repodir) - self.rmtree(repodir) + if symlink: + os.unlink(repodir) + else: + self.rmtree(repodir) continue if rinfo['tag_name'] != tag: self.logger.warn("Tag name mismatch (rename?): %s vs %s", tag, rinfo['tag_name']) @@ -433,9 +498,12 @@ class RepoManager(object): if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): age = time.time() - max(rinfo['create_ts'], dir_ts) self.logger.debug("potential removal candidate: %s; age: %s" % (repodir, age)) - if age > getattr(self.options, timername): + if age > max_age: logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) - self.rmtree(repodir) + if symlink: + os.unlink(repodir) + else: + self.rmtree(repodir) def tagUseStats(self, tag_id): stats = self.tag_use_stats.get(tag_id) @@ -753,15 +821,12 @@ def main(options, session): regen_thread = start_regen_loop(session, repomgr) # TODO also move rmtree jobs to threads logger.info("Entering main loop") - repodir = "%s/repos" % pathinfo.topdir - distrepodir = "%s/repos-dist" % pathinfo.topdir while True: try: repomgr.updateRepos() repomgr.checkQueue() repomgr.printState() - repomgr.pruneLocalRepos(repodir, 'deleted_repo_lifetime') - repomgr.pruneLocalRepos(distrepodir, 'dist_repo_lifetime') + repomgr.pruneLocalRepos() if not curr_chk_thread.isAlive(): logger.error("Currency checker thread died. Restarting it.") curr_chk_thread = start_currency_checker(session, repomgr)