From 6107198f665227cf73b8f22a9693e57cf50d2b83 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 15 2023 13:58:24 +0000 Subject: [PATCH 1/3] distrepo will not skip rpm stat by default There is also CLI option which can override this behaviour if needed. Related: https://pagure.io/koji/issue/3829 --- diff --git a/builder/kojid b/builder/kojid index 4ce767e..0cf3111 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5886,6 +5886,9 @@ class createDistRepoTask(BaseTaskHandler): def handler(self, tag, repo_id, arch, keys, opts): # arch is the arch of the repo, not the task + createrepo_skip_stat = opts.get('createrepo_skip_stat') + if createrepo_skip_stat: + raise koji.ParameterError("createrepo_skip_stat could be only None/False") self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) @@ -5936,7 +5939,8 @@ class createDistRepoTask(BaseTaskHandler): oldrepodata = os.path.join(oldrepodir, arch, 'repodata') self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir, groupdata, oldpkgs=oldpkgs, oldrepodata=oldrepodata, - zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir')) + zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir'), + createrepo_skip_stat=createrepo_skip_stat) for subrepo in self.subrepos: if oldrepo: oldrepodata = os.path.join(oldrepodir, arch, subrepo, 'repodata') @@ -5998,11 +6002,18 @@ class createDistRepoTask(BaseTaskHandler): self.session.uploadWrapper(fn, self.uploadpath) def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, - logname=None, oldrepodata=None, zck=False, zck_dict_dir=None): + logname=None, oldrepodata=None, zck=False, zck_dict_dir=None, + createrepo_skip_stat=None): """Run createrepo This is derived from CreaterepoTask.create_local_repo, but adapted to our requirements here + + :param bool|None createrepo_skip_stat: Override default set in kojid.conf. Note, that + in True variant could resulting repo contain + unexpected rpms. + + """ koji.ensuredir(repodir) if self.options.use_createrepo_c: @@ -6028,7 +6039,11 @@ class createDistRepoTask(BaseTaskHandler): # to rewrite it (if we have external repos to merge) os.unlink(oldorigins) cmd.append('--update') - if self.options.createrepo_skip_stat: + if createrepo_skip_stat is not None: + skip_stat = createrepo_skip_stat + else: + skip_stat = self.options.distrepo_skip_stat + if skip_stat: cmd.append('--skip-stat') if oldpkgs: # generate delta-rpms @@ -6457,6 +6472,7 @@ def get_options(): 'use_createrepo_c': True, 'createrepo_skip_stat': True, 'createrepo_update': True, + 'distrepo_skip_stat': False, 'mock_bootstrap_image': False, 'pkgurl': None, 'allowed_scms': '', @@ -6492,7 +6508,8 @@ def get_options(): 'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout', 'build_arch_can_fail', 'no_ssl_verify', 'log_timestamps', 'allow_noverifyssl', 'allowed_scms_use_config', - 'allowed_scms_use_policy', 'allow_password_in_scm_url']: + 'allowed_scms_use_policy', 'allow_password_in_scm_url', + 'distrepo_skip_stat']: defaults[name] = config.getboolean('kojid', name) elif name in ['plugin', 'plugins']: defaults['plugin'] = value.split() diff --git a/builder/kojid.conf b/builder/kojid.conf index 0cb4595..391a65d 100644 --- a/builder/kojid.conf +++ b/builder/kojid.conf @@ -60,6 +60,19 @@ topurl=http://hub.example.com/kojifiles ; use createrepo_c rather than createrepo ; use_createrepo_c=True +; for faster repo regeneration reuse last repodata for given repo +; createrepo_update=True + +; for createrepo tasks believe that rpms were not changed and +; we don't even need to stat them again. Turn off if some rewrites are expected. +; createrepo_skip_stat=True + +; same as createrepo_skip_stat but for distrepo tasks. Here is the expectation +; different as distrepo can be run with different signing keys. So, tasks can refer +; to different binary versions of rpms. Turn on if you're sure that the task will +; be always run in same way. Not recommended +; distrepo_skip_stat=False + ; A space-separated list of tuples from which kojid is allowed to checkout. ; The format of those tuples is: ; diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index fc78c6d..48e7af1 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7297,6 +7297,11 @@ def handle_dist_repo(options, session, args): '(on builder)') parser.add_option("--write-signed-rpms", action='store_true', default=False, help='Write a signed rpms for given tag') + parser.add_option("--skip-stat", action='store_true', default=None, + help="Skip rpm stat during createrepo (override default builder setting)") + parser.add_option("--no-skip-stat", action='store_false', default=None, + help="Don't skip rpm stat during createrepo " + "(override default builder setting)") task_opts, args = parser.parse_args(args) if len(args) < 1: parser.error('You must provide a tag to generate the repo from') @@ -7387,6 +7392,8 @@ def handle_dist_repo(options, session, args): 'zck_dict_dir': task_opts.zck_dict_dir, 'write_signed_rpms': task_opts.write_signed_rpms, } + if task_opts.skip_stat is not None: + opts['createrepo_skip_stat'] = task_opts.skip_stat task_id = session.distRepo(tag, keys, **opts) print("Creating dist repo for tag " + tag) if task_opts.wait or (task_opts.wait is None and not _running_in_bg()): diff --git a/tests/test_cli/test_dist_repo.py b/tests/test_cli/test_dist_repo.py index 48f3ca2..591b9b7 100644 --- a/tests/test_cli/test_dist_repo.py +++ b/tests/test_cli/test_dist_repo.py @@ -353,6 +353,10 @@ Options: Directory containing compression dictionaries for use by zchunk (on builder) --write-signed-rpms Write a signed rpms for given tag + --skip-stat Skip rpm stat during createrepo (override default + builder setting) + --no-skip-stat Don't skip rpm stat during createrepo (override + default builder setting) """ % self.progname) From 7c079bc011a00d8311b036b62c1dddbfa0e1bf59 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jun 16 2023 10:38:31 +0000 Subject: [PATCH 2/3] remove option test --- diff --git a/builder/kojid b/builder/kojid index 0cf3111..5f25e48 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5886,9 +5886,6 @@ class createDistRepoTask(BaseTaskHandler): def handler(self, tag, repo_id, arch, keys, opts): # arch is the arch of the repo, not the task - createrepo_skip_stat = opts.get('createrepo_skip_stat') - if createrepo_skip_stat: - raise koji.ParameterError("createrepo_skip_stat could be only None/False") self.rinfo = self.session.repoInfo(repo_id, strict=True) if self.rinfo['state'] != koji.REPO_INIT: raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo) @@ -5940,7 +5937,7 @@ class createDistRepoTask(BaseTaskHandler): self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir, groupdata, oldpkgs=oldpkgs, oldrepodata=oldrepodata, zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir'), - createrepo_skip_stat=createrepo_skip_stat) + createrepo_skip_stat=opts.get('createrepo_skip_stat')) for subrepo in self.subrepos: if oldrepo: oldrepodata = os.path.join(oldrepodir, arch, subrepo, 'repodata') From 6fedb7aa8286b80ffa94e2d3918b6c846b04e656 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 17 2023 13:05:53 +0000 Subject: [PATCH 3/3] fix CLI option target --- diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 48e7af1..376dd6b 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -7297,9 +7297,9 @@ def handle_dist_repo(options, session, args): '(on builder)') parser.add_option("--write-signed-rpms", action='store_true', default=False, help='Write a signed rpms for given tag') - parser.add_option("--skip-stat", action='store_true', default=None, + parser.add_option("--skip-stat", action='store_true', default=None, dest='skip_stat', help="Skip rpm stat during createrepo (override default builder setting)") - parser.add_option("--no-skip-stat", action='store_false', default=None, + parser.add_option("--no-skip-stat", action='store_false', default=None, dest='skip_stat', help="Don't skip rpm stat during createrepo " "(override default builder setting)") task_opts, args = parser.parse_args(args)