#1066 Simple mode for mergerepos
Merged 3 years ago by mikem. Opened 3 years ago by tkopecek.
tkopecek/koji ext-repo-mode  into  master

file modified
+26 -4
@@ -5098,30 +5098,52 @@ 

                      % parseStatus(status, ' '.join(cmd)))

  

      def merge_repos(self, external_repos, arch, groupdata):

-         repos = []

+         # group repos by merge type

+         repos_by_mode = {}

+         for repo in external_repos:

+             repos_by_mode.setdefault(

+                     repo.get('merge_mode', 'koji'), []).append(repo)

+ 

+         # figure out merge mode

+         if len(repos_by_mode) > 1:

+             # TODO: eventually support mixing merge modes

+             raise koji.GenericError('Found multiple merge modes for external '

+                     'repos: %s\n' % repos_by_mode.keys())

+         merge_mode = repos_by_mode.keys()[0]

+ 

+         # move current repo to the premerge location

          localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)

          os.rename(self.outdir, localdir)

          koji.ensuredir(self.outdir)

-         repos.append('file://' + localdir + '/')

  

+         # generate repo url list, starting with our local premerge repo

+         repos = ['file://' + localdir + '/']

          for repo in external_repos:

              ext_url = repo['url']

              # substitute $arch in the url with the arch of the repo we're generating

              ext_url = ext_url.replace('$arch', arch)

              repos.append(ext_url)

  

-         blocklist = self.repodir + '/blocklist'

-         if self.options.use_createrepo_c:

+         # construct command

+         if merge_mode == 'simple':

+             # currently only supported by our own mergerepos script

+             # (we need it to write pkgorigins)

+             cmd = ['/usr/libexec/kojid/mergerepos',

+                    '--mode', 'simple',

+                    '--tempdir', self.workdir]

+         elif self.options.use_createrepo_c:

              cmd = ['/usr/bin/mergerepo_c', '--koji']

          else:

              cmd = ['/usr/libexec/kojid/mergerepos']

              cmd.extend(['--tempdir', self.workdir])

+         blocklist = self.repodir + '/blocklist'

          cmd.extend(['-a', arch, '-b', blocklist, '-o', self.outdir])

          if os.path.isfile(groupdata):

              cmd.extend(['-g', groupdata])

          for repo in repos:

              cmd.extend(['-r', repo])

  

+         # run command

          logfile = '%s/mergerepos.log' % self.workdir

          status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)

          if not isSuccess(status):

file modified
+58 -2
@@ -72,6 +72,7 @@ 

                        help="List of arches to include in the repo")

      parser.add_option("-b", "--blocked", default=None,

                        help="A file containing a list of srpm names to exclude from the merged repo")

+     parser.add_option("--mode", default='koji', help="Select the merge mode")

      parser.add_option("-o", "--outputdir", default=None,

                        help="Location to create the repository")

      parser.add_option("--tempdir", default=None,
@@ -112,7 +113,8 @@ 

  

  

  class RepoMerge(object):

-     def __init__(self, repolist, arches, groupfile, blocked, outputdir, tempdir=None):

+     def __init__(self, repolist, arches, groupfile, blocked, outputdir,

+                  tempdir=None, mode='koji'):

          self.repolist = repolist

          self.outputdir = outputdir

          self.tempdir = tempdir
@@ -134,6 +136,7 @@ 

          self.archlist = arches

          self.mdconf.groupfile = groupfile

          self.blocked = blocked

+         self.mode = mode

  

      def close(self):

          if self.yumbase is not None:
@@ -182,6 +185,10 @@ 

          allows a package to be tracked back to its origin, even if the location field in the repodata does

          not match the original repo location.

          """

+ 

+         if self.mode == 'simple':

+             return self.do_simple_sort()

+ 

          # sort the repos by _merge_rank

          # lowest number is the highest rank (1st place, 2nd place, etc.)

          repos = self.yumbase.repos.listEnabled()
@@ -264,6 +271,55 @@ 

          origins.close()

          self.mdconf.additional_metadata['origin'] = pkgorigins

  

+     def do_simple_sort(self):

+         """

+         Handle the 'sort_and_filter' case when mode=simple

+ 

+         As the name implies, this is a much simpler approach. Mainly, we need

+         to generate the pkgorigins file.

+         """

+ 

+         # sort the repos by _merge_rank

+         # lowest number is the highest rank (1st place, 2nd place, etc.)

+         repos = self.yumbase.repos.listEnabled()

+         repos.sort(key=lambda o: o._merge_rank)

+ 

+         # TODO: reduce duplication between this function and sort_and_filter()

+ 

+         # We lack the complex filtration of mode=koji, but we still need to:

+         # - fix urls for primary repo

+         # - enforce blocked list

+         for reponum, repo in enumerate(repos):

+             for pkg in repo.sack:

+                 if reponum == 0 and not pkg.basepath:

+                     # this is the first repo (i.e. the koji repo) and appears

+                     # to be using relative urls

+                     #XXX - kind of a hack, but yum leaves us little choice

+                     #force the pkg object to report a relative location

+                     loc = """<location href="%s"/>\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True)

+                     pkg._return_remote_location = make_const_func(loc)

+ 

+         pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')

+         origins = open(pkgorigins, 'w')

+ 

+         seen_rpms = {}

+         for repo in repos:

+             for pkg in repo.sack:

+                 srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)

+                 pkg_nvra = str(pkg)

+                 if pkg_nvra in seen_rpms:

+                     sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra)

+                     # note: we warn, but do not omit it

+                 if srpm_name in self.blocked:

+                     sys.stderr.write('Ignoring blocked package: %s\n\n' %

+                                      pkg.sourcerpm)

+                     repo.sack.delPackage(pkg)

+                 if pkg_nvra not in seen_rpms:

+                     origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0]))

+                 seen_rpms[pkg_nvra] = 1

+         origins.close()

+         self.mdconf.additional_metadata['origin'] = pkgorigins

+ 

      def write_metadata(self):

          self.mdconf.pkglist = self.yumbase.pkgSack

          self.mdconf.directory = self.outputdir
@@ -292,7 +348,7 @@ 

          blocked = {}

  

      merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked,

-                       opts.outputdir, opts.tempdir)

+                       opts.outputdir, opts.tempdir, opts.mode)

  

      try:

          merge.merge_repos()

file modified
+23 -7
@@ -5177,17 +5177,19 @@ 

          header1 = "%-25s %s" % ("External repo name", "URL")

          header2 = "%s %s" % ("-"*25, "-"*40)

      elif format == "tag":

-         format = "%(priority)-3i %(external_repo_name)-25s %(url)s"

-         header1 = "%-3s %-25s %s" % ("Pri", "External repo name", "URL")

-         header2 = "%s %s %s" % ("-"*3, "-"*25, "-"*40)

+         format = "%(priority)-3i %(external_repo_name)-25s %(merge_mode)-10s %(url)s"

+         header1 = "%-3s %-25s %-10s URL" % ("Pri", "External repo name", "Mode")

+         header2 = "%s %s %s %s" % ("-"*3, "-"*25, "-"*10, "-"*40)

      elif format == "multitag":

-         format = "%(tag_name)-20s %(priority)-3i %(external_repo_name)s"

-         header1 = "%-20s %-3s %s" % ("Tag", "Pri", "External repo name")

-         header2 = "%s %s %s" % ("-"*20, "-"*3, "-"*25)

+         format = "%(tag_name)-20s %(priority)-3i %(merge_mode)-10s %(external_repo_name)s"

+         header1 = "%-20s %-3s %-10s %s" % ("Tag", "Pri", "Mode", "External repo name")

+         header2 = "%s %s %s %s" % ("-"*20, "-"*3, "-"*10, "-"*25)

      if not options.quiet:

          print(header1)

          print(header2)

      for rinfo in data:

+         # older hubs do not support merge_mode

+         rinfo.setdefault('merge_mode', None)

          print(format % rinfo)

  

  
@@ -5227,8 +5229,14 @@ 

                        help=_("Also add repo to tag. Use tag::N to set priority"))

      parser.add_option("-p", "--priority", type='int',

                        help=_("Set priority (when adding to tag)"))

+     parser.add_option("-m", "--mode", help=_("Set merge mode"))

      (options, args) = parser.parse_args(args)

      activate_session(session, goptions)

+     if options.mode:

+         if options.mode not in koji.REPO_MERGE_MODES:

+             parser.error('Invalid mode: %s' % options.mode)

+         if not options.tag:

+             parser.error('The --mode option can only be used with --tag')

      if len(args) == 1:

          name = args[0]

          rinfo = session.getExternalRepo(name, strict=True)
@@ -5249,7 +5257,10 @@ 

                      priority = options.priority

                  else:

                      priority = _pick_external_repo_priority(session, tag)

-             session.addExternalRepoToTag(tag, rinfo['name'], priority)

+             callopts = {}

+             if options.mode:

+                 callopts['merge_mode'] = options.mode

+             session.addExternalRepoToTag(tag, rinfo['name'], priority, **callopts)

              print("Added external repo %s to tag %s (priority %i)" \

                      % (rinfo['name'], tag, priority))

  
@@ -5261,6 +5272,7 @@ 

      parser = OptionParser(usage=usage)

      parser.add_option("--url",  help=_("Change the url"))

      parser.add_option("--name",  help=_("Change the name"))

+     parser.add_option("-m", "--mode", help=_("Set merge mode"))

      (options, args) = parser.parse_args(args)

      if len(args) != 1:

          parser.error(_("Incorrect number of arguments"))
@@ -5271,6 +5283,10 @@ 

          opts['url'] = options.url

      if options.name:

          opts['name'] = options.name

+     if options.mode:

+         if options.mode not in koji.REPO_MERGE_MODES:

+             parser.error('Invalid mode: %s' % options.mode)

+         opts['merge_mode'] = options.mode

      if not opts:

          parser.error(_("No changes specified"))

      activate_session(session, goptions)

@@ -7,4 +7,7 @@ 

  -- Change VARCHAR field for build_target names to TEXT to allow longer names

  ALTER TABLE build_target ALTER COLUMN name TYPE TEXT;

  

+ -- Allow different merge modes for mergerepo

+ ALTER TABLE tag_external_repos ADD COLUMN merge_mode TEXT DEFAULT 'koji';

+ 

  COMMIT;

file modified
+1 -1
@@ -450,7 +450,6 @@ 

  create table external_repo_config (

  	external_repo_id INTEGER NOT NULL REFERENCES external_repo(id),

  	url TEXT NOT NULL,

-         merge_mode TEXT DEFAULT 'koji',

  -- versioned - see earlier description of versioning

  	create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(),

  	revoke_event INTEGER REFERENCES events(id),
@@ -468,6 +467,7 @@ 

  	tag_id INTEGER NOT NULL REFERENCES tag(id),

  	external_repo_id INTEGER NOT NULL REFERENCES external_repo(id),

  	priority INTEGER NOT NULL,

+ 	merge_mode TEXT DEFAULT 'koji',

  -- versioned - see earlier description of versioning

  	create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(),

  	revoke_event INTEGER REFERENCES events(id),

file modified
+22 -7
@@ -3362,11 +3362,14 @@ 

      update.make_revoke()

      update.execute()

  

- def add_external_repo_to_tag(tag_info, repo_info, priority):

+ def add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode='koji'):

      """Add an external repo to a tag"""

  

      context.session.assertPerm('admin')

  

+     if merge_mode not in koji.REPO_MERGE_MODES:

+         raise koji.GenericError('Invalid merge mode: %s' % merge_mode)

+ 

      tag = get_tag(tag_info, strict=True)

      tag_id = tag['id']

      repo = get_external_repo(repo_info, strict=True)
@@ -3381,7 +3384,8 @@ 

              (tag['name'], priority))

  

      insert = InsertProcessor('tag_external_repos')

-     insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority)

+     insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority,

+                merge_mode=merge_mode)

      insert.make_create()

      insert.execute()

  
@@ -3435,14 +3439,23 @@ 

      external_repo_id

      external_repo_name

      url

+     merge_mode

      priority

      """

      tables = ['tag_external_repos']

      joins = ['tag ON tag_external_repos.tag_id = tag.id',

               'external_repo ON tag_external_repos.external_repo_id = external_repo.id',

               'external_repo_config ON external_repo.id = external_repo_config.external_repo_id']

-     columns = ['tag.id', 'tag.name', 'external_repo.id', 'external_repo.name', 'url', 'priority']

-     aliases = ['tag_id', 'tag_name', 'external_repo_id', 'external_repo_name', 'url', 'priority']

+     fields = {

+             'external_repo.id': 'external_repo_id',

+             'external_repo.name': 'external_repo_name',

+             'priority': 'priority',

+             'tag.id': 'tag_id',

+             'tag.name': 'tag_name',

+             'url': 'url',

+             'merge_mode': 'merge_mode',

+             }

+     columns, aliases = zip(*fields.items())

  

      clauses = [eventCondition(event, table='tag_external_repos'), eventCondition(event, table='external_repo_config')]

      if tag_info:
@@ -3474,6 +3487,7 @@ 

      external_repo_id

      external_repo_name

      url

+     merge_mode

      priority

      """

      tag = get_tag(tag_info, strict=True, event=event)
@@ -6725,7 +6739,7 @@ 

          'external_repo_config': ['external_repo_id', 'url'],

          'host_config': ['host_id', 'arches', 'capacity', 'description', 'comment', 'enabled'],

          'host_channels': ['host_id', 'channel_id'],

-         'tag_external_repos': ['tag_id', 'external_repo_id', 'priority'],

+         'tag_external_repos': ['tag_id', 'external_repo_id', 'priority', 'merge_mode'],

          'tag_listing': ['build_id', 'tag_id'],

          'tag_packages': ['package_id', 'tag_id', 'owner', 'blocked', 'extra_arches'],

          'group_config': ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', 'uservisible',
@@ -9175,10 +9189,11 @@ 

      editExternalRepo = staticmethod(edit_external_repo)

      deleteExternalRepo = staticmethod(delete_external_repo)

  

-     def addExternalRepoToTag(self, tag_info, repo_info, priority):

+     def addExternalRepoToTag(self, tag_info, repo_info, priority,

+                 merge_mode='koji'):

          """Add an external repo to a tag"""

          # wrap the local method so we don't expose the event parameter

-         add_external_repo_to_tag(tag_info, repo_info, priority)

+         add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode)

  

      def removeExternalRepoFromTag(self, tag_info, repo_info):

          """Remove an external repo from a tag"""

file modified
+2
@@ -236,6 +236,8 @@ 

  REPO_DELETED = REPO_STATES['DELETED']

  REPO_PROBLEM = REPO_STATES['PROBLEM']

  

+ REPO_MERGE_MODES = set(['koji', 'simple'])

+ 

  # buildroot states

  BR_STATES = Enum((

      'INIT',

no initial comment

rebased onto 48eff69

3 years ago

:thumbsup: (but it's mostly my code)

I take it you're happy with it, @tkopecek ?

Can we please just drop the bundled mergerepos code and switch to mergerepos_c?

@ngompa - we can't do it for now, as mergerepos_c doesn't provide needed behaviour. But I'm planning to write a patch for it making same things., so we can use it also for this mode.

It doesn't? mergerepos_c has a --koji mode that is intended to behave the same way as our embedded one...

@ngompa - yes, mergrepo_c with --koji option is used when use_createrepo_c is set to true. In this PR we're adding different mode ("simple") for merging external repos, which allows us to use "malformed" repodata. In such case --koji behaviour should be limited to create pkgorigins file, but not to enforce other checks of that mode.

So, for normal usage (before this patch) is mergerepo_c used already. See switch-part in https://pagure.io/koji/pull-request/1066#_1,36

Commit e08e279 fixes this pull-request

Pull-Request has been merged by mikem

3 years ago