#1273 Allow generating separate src repo for build repos
Merged a year ago by mikem. Opened a year ago by mizdebsk.
mizdebsk/koji separate-src-repo  into  master

file modified
+3 -1

@@ -4990,13 +4990,15 @@ 

      Methods = ['newRepo']

      _taskWeight = 0.1

  

-     def handler(self, tag, event=None, src=False, debuginfo=False):

+     def handler(self, tag, event=None, src=False, debuginfo=False, separate_src=False):

          tinfo = self.session.getTag(tag, strict=True, event=event)

          kwargs = {}

          if event is not None:

              kwargs['event'] = event

          if src:

              kwargs['with_src'] = True

+         if separate_src:

+             kwargs['with_separate_src'] = True

          if debuginfo:

              kwargs['with_debuginfo'] = True

          repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)

file modified
+4 -1

@@ -7011,7 +7011,8 @@ 

      parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name"))

      parser.add_option("--nowait", action="store_true", help=_("Don't wait on for regen to finish"))

      parser.add_option("--debuginfo", action="store_true", help=_("Include debuginfo rpms in repo"))

-     parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in the repo"))

+     parser.add_option("--source", "--src", action="store_true", help=_("Include source rpms in each of repos"))

+     parser.add_option("--separate-source", "--separate-src", action="store_true", help=_("Include source rpms in separate src repo"))

      (suboptions, args) = parser.parse_args(args)

      if len(args) == 0:

          parser.error(_("A tag name must be specified"))

@@ -7047,6 +7048,8 @@ 

          repo_opts['debuginfo'] = True

      if suboptions.source:

          repo_opts['src'] = True

+     if suboptions.separate_source:

+         repo_opts['separate_src'] = True

      task_id = session.newRepo(tag, **repo_opts)

      print("Regenerating repo for tag: %s" % tag)

      print("Created task: %d" % task_id)

file modified
+15 -8

@@ -2345,7 +2345,7 @@ 

                      yield archive

      return _iter_archives()

  

- def repo_init(tag, with_src=False, with_debuginfo=False, event=None):

+ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False):

      """Create a new repo entry in the INIT state, return full repo data

  

      Returns a dictionary containing

@@ -2355,9 +2355,11 @@ 

      state = koji.REPO_INIT

      tinfo = get_tag(tag, strict=True, event=event)

      koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo,

-                               event=event, repo_id=None)

+                               event=event, repo_id=None, with_separate_src=with_separate_src)

      tag_id = tinfo['id']

      repo_arches = {}

+     if with_separate_src:

+         repo_arches['src'] = 1

      if tinfo['arches']:

          for arch in tinfo['arches'].split():

              arch = koji.canonArch(arch)

@@ -2420,6 +2422,8 @@ 

              if with_src:

                  for repoarch in repo_arches:

                      pkglist[repoarch].write(relpath)

+             if with_separate_src:

+                 pkglist[arch].write(relpath)

          elif arch == 'noarch':

              for repoarch in repo_arches:

                  pkglist[repoarch].write(relpath)

@@ -2474,7 +2478,7 @@ 

              _write_maven_repo_metadata(artifact_dir, artifacts)

  

      koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=with_src, with_debuginfo=with_debuginfo,

-                               event=event, repo_id=repo_id)

+                               event=event, repo_id=repo_id, with_separate_src=with_separate_src)

      return [repo_id, event_id]

  

  def _write_maven_repo_metadata(destdir, artifacts):

@@ -2519,7 +2523,8 @@ 

      # note: we need to match args from the other preRepoInit callback

      koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,

              with_debuginfo=False, event=event, repo_id=None,

-             dist=True, keys=keys, arches=arches, task_opts=task_opts)

+             dist=True, keys=keys, arches=arches, task_opts=task_opts,

+             with_separate_src=False)

      if not event:

          event = get_event()

      repo_id = nextval('repo_id_seq')

@@ -2540,7 +2545,7 @@ 

      koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,

              with_debuginfo=False, event=event, repo_id=repo_id,

              dist=True, keys=keys, arches=arches, task_opts=task_opts,

-             repodir=repodir)

+             repodir=repodir, with_reparate_src=False)

      return repo_id, event

  

  

@@ -10521,7 +10526,7 @@ 

          task_opts['event'] = event_id

          return make_task('distRepo', [tag, repo_id, keys, task_opts], priority=15, channel='createrepo')

  

-     def newRepo(self, tag, event=None, src=False, debuginfo=False):

+     def newRepo(self, tag, event=None, src=False, debuginfo=False, separate_src=False):

          """Create a newRepo task. returns task id"""

          if context.session.hasPerm('regen-repo'):

              pass

@@ -10532,6 +10537,8 @@ 

              opts['event'] = event

          if src:

              opts['src'] = True

+         if separate_src:

+             opts['separate_src'] = True

          if debuginfo:

              opts['debuginfo'] = True

          args = koji.encode_args(tag, **opts)

@@ -12699,11 +12706,11 @@ 

  

          return br.updateArchiveList(archives, project)

  

-     def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None):

+     def repoInit(self, tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False):

          """Initialize a new repo for tag"""

          host = Host()

          host.verify()

-         return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event)

+         return repo_init(tag, with_src=with_src, with_debuginfo=with_debuginfo, event=event, with_separate_src=with_separate_src)

  

      def repoAddRPM(self, repo_id, path):

          """Add an uploaded rpm to a repo"""

file modified
+1 -1

@@ -200,7 +200,7 @@ 

          [['builds', 'target', 'opts'], None, None, (None,)],

      ],

      'newRepo' : [

-         [['tag', 'event', 'src', 'debuginfo'], None, None, (None, False, False)],

+         [['tag', 'event', 'src', 'debuginfo', 'separate_src'], None, None, (None, False, False, False)],

      ],

      'createImage' : [

          [['name', 'version', 'release', 'arch', 'target_info', 'build_tag', 'repo_info', 'inst_tree', 'opts'], None, None, (None,)],

@@ -168,11 +168,13 @@ 

  (Specify the --help global option for a list of other help options)

  

  Options:

-   -h, --help       show this help message and exit

-   --target         Interpret the argument as a build target name

-   --nowait         Don't wait on for regen to finish

-   --debuginfo      Include debuginfo rpms in repo

-   --source, --src  Include source rpms in the repo

+   -h, --help            show this help message and exit

+   --target              Interpret the argument as a build target name

+   --nowait              Don't wait on for regen to finish

+   --debuginfo           Include debuginfo rpms in repo

+   --source, --src       Include source rpms in each of repos

+   --separate-source, --separate-src

+                         Include source rpms in separate src repo

  """ % self.progname)

  

  

file modified
+5 -1

@@ -671,6 +671,7 @@ 

  

          debuginfo_pat = self.options.debuginfo_tags.split()

          src_pat = self.options.source_tags.split()

+         separate_src_pat = self.options.separate_source_tags.split()

          order = sorted(self.needed_tags.values(), key=lambda t: t['score'])

          for tag in order:

              if running_tasks >= self.options.max_repo_tasks:

@@ -699,6 +700,8 @@ 

                  taskopts['debuginfo'] = True

              if koji.util.multi_fnmatch(tagname, src_pat):

                  taskopts['src'] = True

+             if koji.util.multi_fnmatch(tagname, separate_src_pat):

+                 taskopts['separate_src'] = True

              maven = tag['taginfo']['maven_support']

              if maven:

                  if running_tasks_maven >= self.options.max_repo_tasks_maven:

@@ -836,6 +839,7 @@ 

      defaults = {'with_src': False,

                  'debuginfo_tags': '',

                  'source_tags': '',

+                 'separate_source_tags': '',

                  'ignore_tags': '',

                  'verbose': False,

                  'debug': False,

@@ -876,7 +880,7 @@ 

                      'recent_tasks_lifetime')

          str_opts = ('topdir', 'server', 'user', 'password', 'logfile', 'principal', 'keytab', 'krbservice',

                      'cert', 'ca', 'serverca', 'debuginfo_tags',

-                     'source_tags', 'ignore_tags')  # FIXME: remove ca here

+                     'source_tags', 'separate_source_tags', 'ignore_tags')  # FIXME: remove ca here

          bool_opts = ('with_src','verbose','debug','ignore_stray_repos', 'offline_retry',

                       'krb_rdns', 'krb_canon_host', 'no_ssl_verify')

          for name in config.options(section):

Fixes #1266

Currently Koji has an option that can be used to include source rpms in each of generated arch repos. Howewer it makes repositories significantly bigger and slower to generate. Source metadata is duplicated across all arch repos. Increased metadata size makes builds slower.

This pull request takes a slightly different approach - it makes it possible to generate build repos with source rpms in separate repos. Such repos don't change size of each arch repo and can be generated faster - generation is done in a separate createrepo task that can run on separate host. For example:

newRepo
  ├  createrepo (src)
  ├  createrepo (aarch64)
  ├  createrepo (ppc64le)
  └  createrepo (x86_64)

CC @ignatenkobrain

======================================================================
FAIL: test_legacy_data (tests.test_builder.test_taskparams.TestParseTaskParams)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/srv/jenkins/workspace/koji/label/EL7/tests/test_builder/test_taskparams.py", line 61, in test_legacy_data
    self.assertIn(list(spec), koji.tasks.LEGACY_SIGNATURES[method])
AssertionError: [['tag', 'event', 'src', 'debuginfo', 'separate_src'], None, None, (None, False, False, False)] not found in [[['tag', 'event', 'src', 'debuginfo'], None, None, (None, False, False)]]

======================================================================
FAIL: Test handle_regen_repo help message
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/srv/jenkins/workspace/koji/label/EL7/tests/test_cli/test_regen_repo.py", line 176, in test_handle_regen_repo_help
    """ % self.progname)
  File "/srv/jenkins/workspace/koji/label/EL7/kojienv/lib/python2.7/site-packages/mock/mock.py", line 1305, in patched
    return func(*args, **keywargs)
  File "/srv/jenkins/workspace/koji/label/EL7/tests/test_cli/utils.py", line 209, in assert_help
    exit_code=0)
  File "/srv/jenkins/workspace/koji/label/EL7/tests/test_cli/utils.py", line 190, in assert_system_exit
    self.assert_console_message(stdout, **message['stdout'])
  File "/srv/jenkins/workspace/koji/label/EL7/tests/test_cli/utils.py", line 86, in assert_console_message
    self.assertMultiLineEqual(output, message)
AssertionError: "Usag[136 chars]          show this help message and exit\n  -[344 chars]po\n" != "Usag[136 chars]     show this help message and exit\n  --targ[211 chars]po\n"
  Usage: nosetests regen-repo [options] <tag>
  (Specify the --help global option for a list of other help options)

  Options:
-   -h, --help            show this help message and exit
?                    -----
+   -h, --help       show this help message and exit
-   --target              Interpret the argument as a build target name
?           -----
+   --target         Interpret the argument as a build target name
-   --nowait              Don't wait on for regen to finish
?                    -----
+   --nowait         Don't wait on for regen to finish
-   --debuginfo           Include debuginfo rpms in repo
?                    -----
+   --debuginfo      Include debuginfo rpms in repo
-   --source, --src       Include source rpms in each of repos
?                  -----                          ------     -
+   --source, --src  Include source rpms in the repo
?                                           ++
-   --separate-source, --separate-src
-                         Include source rpms in separate src repo

You need to adjust tests :)

You need to adjust tests :)

Thanks. I forgot that Koji now has test suite - it didn't have them for long time.

rebased onto 3590ab0

a year ago

Test failures are fixed now.

@mizdebsk Is there a reason we wouldn't just make this the only behavior? I feel like that would make more sense than allowing the current behavior to persist...

I kept current behaviour for backwards-compatibility. I don't know of any Koji instance using it and personally I would be fine with dropping it. But it's not my call to make.

@mikem @tkopecek I think we should drop the old behavior as this new behavior is more efficient.

I have put a few general comments on the issue (#1266).

My concern with the current PR is how we expect to manage this. Is a set of glob patterns in kojira.conf the right way to control which tags get this behavior?

The existing source_tags option in kojira is pretty old (2010), and I think we have better options now. Consider how the repo_include_all option is handled (PR#590). Would it be better to have this live in tag config? I probably depends on how folks intend to use this. So, it would really be nice if we could get more detail on use cases in issue #1266.

For my use cases source_tags option in kojira config should be enough. When implementing this PR I tried to be consistent with current approach (src repos combined with arch repos), but I can move configuration to tag_config in database if @mikem thinks it would be better.

@mikem @tkopecek what is blocking this? I really need this functionality and I would like to not ask @nirik to patch this in Fedora infra.

Sorry for the delay. I'm not super happy with managing this in the kojira config, but I guess that's good enough for now.

Rebased with a couple fixes here:

  • don't add noarch rpms to src repos
  • include separate_src opt in repo info file

https://github.com/mikem23/koji-playground/commits/pagure/pr/1273

If that looks good I can merge it

Commit b122dcd fixes this pull-request

Pull-Request has been merged by mikem

a year ago