#3838 distrepo will not skip rpm stat by default
Merged 9 months ago by tkopecek. Opened 10 months ago by tkopecek.
tkopecek/koji issue3829  into  master

file modified
+18 -4
@@ -5936,7 +5936,8 @@ 

              oldrepodata = os.path.join(oldrepodir, arch, 'repodata')

          self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,

                             groupdata, oldpkgs=oldpkgs, oldrepodata=oldrepodata,

-                            zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir'))

+                            zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir'),

+                            createrepo_skip_stat=opts.get('createrepo_skip_stat'))

          for subrepo in self.subrepos:

              if oldrepo:

                  oldrepodata = os.path.join(oldrepodir, arch, subrepo, 'repodata')
@@ -5998,11 +5999,18 @@ 

          self.session.uploadWrapper(fn, self.uploadpath)

  

      def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None,

-                       logname=None, oldrepodata=None, zck=False, zck_dict_dir=None):

+                       logname=None, oldrepodata=None, zck=False, zck_dict_dir=None,

+                       createrepo_skip_stat=None):

          """Run createrepo

  

          This is derived from CreaterepoTask.create_local_repo, but adapted to

          our requirements here

+ 

+         :param bool|None createrepo_skip_stat: Override default set in kojid.conf. Note, that

+                                                in True variant could resulting repo contain

+                                                unexpected rpms.

+ 

+ 

          """

          koji.ensuredir(repodir)

          if self.options.use_createrepo_c:
@@ -6028,7 +6036,11 @@ 

                      # to rewrite it (if we have external repos to merge)

                      os.unlink(oldorigins)

                  cmd.append('--update')

-                 if self.options.createrepo_skip_stat:

+                 if createrepo_skip_stat is not None:

+                     skip_stat = createrepo_skip_stat

+                 else:

+                     skip_stat = self.options.distrepo_skip_stat

+                 if skip_stat:

                      cmd.append('--skip-stat')

          if oldpkgs:

              # generate delta-rpms
@@ -6457,6 +6469,7 @@ 

                  'use_createrepo_c': True,

                  'createrepo_skip_stat': True,

                  'createrepo_update': True,

+                 'distrepo_skip_stat': False,

                  'mock_bootstrap_image': False,

                  'pkgurl': None,

                  'allowed_scms': '',
@@ -6492,7 +6505,8 @@ 

                            'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout',

                            'build_arch_can_fail', 'no_ssl_verify', 'log_timestamps',

                            'allow_noverifyssl', 'allowed_scms_use_config',

-                           'allowed_scms_use_policy', 'allow_password_in_scm_url']:

+                           'allowed_scms_use_policy', 'allow_password_in_scm_url',

+                           'distrepo_skip_stat']:

                  defaults[name] = config.getboolean('kojid', name)

              elif name in ['plugin', 'plugins']:

                  defaults['plugin'] = value.split()

file modified
+13
@@ -60,6 +60,19 @@ 

  ; use createrepo_c rather than createrepo

  ; use_createrepo_c=True

  

+ ; for faster repo regeneration reuse last repodata for given repo

+ ; createrepo_update=True

+ 

+ ; for createrepo tasks believe that rpms were not changed and

+ ; we don't even need to stat them again. Turn off if some rewrites are expected.

+ ; createrepo_skip_stat=True

+ 

+ ; same as createrepo_skip_stat but for distrepo tasks. Here is the expectation

+ ; different as distrepo can be run with different signing keys. So, tasks can refer

+ ; to different binary versions of rpms. Turn on if you're sure that the task will

+ ; be always run in same way. Not recommended

+ ; distrepo_skip_stat=False

+ 

  ; A space-separated list of tuples from which kojid is allowed to checkout.

  ; The format of those tuples is:

  ;

@@ -7297,6 +7297,11 @@ 

                             '(on builder)')

      parser.add_option("--write-signed-rpms", action='store_true', default=False,

                        help='Write a signed rpms for given tag')

+     parser.add_option("--skip-stat", action='store_true', default=None, dest='skip_stat',

+                       help="Skip rpm stat during createrepo (override default builder setting)")

+     parser.add_option("--no-skip-stat", action='store_false', default=None, dest='skip_stat',

+                       help="Don't skip rpm stat during createrepo "

+                            "(override default builder setting)")

      task_opts, args = parser.parse_args(args)

      if len(args) < 1:

          parser.error('You must provide a tag to generate the repo from')
@@ -7387,6 +7392,8 @@ 

          'zck_dict_dir': task_opts.zck_dict_dir,

          'write_signed_rpms': task_opts.write_signed_rpms,

      }

+     if task_opts.skip_stat is not None:

+         opts['createrepo_skip_stat'] = task_opts.skip_stat

      task_id = session.distRepo(tag, keys, **opts)

      print("Creating dist repo for tag " + tag)

      if task_opts.wait or (task_opts.wait is None and not _running_in_bg()):

@@ -353,6 +353,10 @@ 

                          Directory containing compression dictionaries for use

                          by zchunk (on builder)

    --write-signed-rpms   Write a signed rpms for given tag

+   --skip-stat           Skip rpm stat during createrepo (override default

+                         builder setting)

+   --no-skip-stat        Don't skip rpm stat during createrepo (override

+                         default builder setting)

  """ % self.progname)

  

  

There is also CLI option which can override this behaviour if needed.

Related: https://pagure.io/koji/issue/3829

rebased onto c34f7219fba4555b21bb8b68d3a94d6b01e57731

10 months ago

The logic in the cli is inverted. If you pass --skip-stat to the cli, this sets createrepo_skip_stat to False, which actually blocks passing the option of the same name in the task handler. If we're only going to allow the option to block passing --skip-stat when the builder is otherwise configured to pass it, then we probably need a different cli option name, if not a different option name through the chain.

That said, it's not clear to me what we want to allow and when here. In a comment you have

True variant is not possible to reduce security risk

A True variant (modulo the logic issue above) would allow the user to force the use of --skip-stat even when the kojid config has changed createrepo_skip_stat to False. I get that this is not ideal, but is it a security risk? If --skip-stat is never a good idea for dist repos, then should we ignore the kojid option here?

Also, were we going to change the default for createrepo_skip_stat?

A few options to think about.

  1. a new kojid option distrepo_skip_stat which is used instead of createrepo_skip_stat for dist repos
  2. distrepo_skip_stat could default to False, or it could default to None (meaning use the value of createrepo_skip_stat)
  3. --skip-stat/--no-skip-stat options in the cli that override the setting
  4. perhaps some kojid config that indicates the degree of override allowed. This could be yet another option, or perhaps an explicit (non-None)distrepo_skip_stat can't be overridden
  5. probably only error on the cli override if it is actually overriding in a way we want to block. I.e. if we're configured to skip-stat anyway, why complain?

1 new commit added

  • add kojid option
10 months ago

I've added kojid option. None could be a bit confusing - maybe better to have it explicit? And allow both overrides from CLI.

Main handler is passing createrepo_skip_stat=createrepo_skip_stat to do_createrepo, but the method arg is now expecting distrepo_skip_stat.

The introduction of distrepo_skip_stat makes for a bit of naming confusion. It might be best to take the approach that we only use the name "distrepo_skip_stat" for the kojid option and use "createrepo_skip_stat" for the other parameter/var names involved here. In other words, maybe best to address the above by switching the do_createrepo arg back.

The handler still asserts that the createrepo_skip_stat opt is None/False, but we seem to want to allow True now.

   if self.options.distrepo_skip_stat or distrepo_skip_stat:
       cmd.append('--skip-stat')

This checks the kojid option first, so if it is True we will pass --skip-stat regardless of task parameters. This doesn't match the "override" verbage. Granted, we might want to limit the override options somehow?

I wonder if something like the following would make sense?

if self.options.distrepo_skip_stat is not None:
    skip_stat = self.options.distrepo_skip_stat
elif distrepo_skip_stat is not None:   # also maybe rename this param as mentioned above
    skip_stat = self.options.distrepo_skip_stat
else:
    skip_stat = self.options.createrepo_skip_stat
if skip_stat:
    cmd.append('--skip-stat')

i.e. the cli option allows you to override createrepo_skip_stat, but if kojid has distrepo_skip_stat that will always win. We could add an additional assertion to error on an invalid override.

Alternately, we could swap the first to stanzas and allow the cli to override.

What would be the default value in kojid.conf then? If we set None in defaults (kojid.get_options), it is a bit inconsistent with other booleans (none of them can be unset). Further, it takes ugly default from createrepo_skip_stat=True. Does it make sense like this?

if createrepo_skip_stat is not None: # renamed distrepo_skip_stat
    skip_stat = createrepo_skip_stat
else:
    skip_stat = self.options.distrepo_skip_stat:
if skip_stat:
    cmd.append('--skip-stat')

in such case we still can have distrepo_skip_stat=False as default

So:

  • cli option wins, if given
  • otherwise we use the new distrepo_skip_stat setting
  • createrepo_skip_stat is not considered

Seems fine to me.

It doesn't address your initial concerns about limiting the override to one-way, but I'm not sure that's needed and we can always add another option for that later if it is an issue.

rebased onto 6107198

10 months ago

We still have this

if createrepo_skip_stat:
    raise koji.ParameterError("createrepo_skip_stat could be only None/False")

Even though we have an option in the cli that explicitly passes it as True

1 new commit added

  • remove option test
10 months ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

10 months ago

1 new commit added

  • fix CLI option target
9 months ago

Metadata Update from @mfilip:
- Pull-request tagged with: testing-done

9 months ago

Commit 26287a8 fixes this pull-request

Pull-Request has been merged by tkopecek

9 months ago