From e08e279f6b56a67eb0f9fa4dcc321a7b4733c910 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Sep 20 2018 16:37:32 +0000 Subject: PR#1066: Simple mode for mergerepos Merges #1066 https://pagure.io/koji/pull-request/1066 --- diff --git a/builder/kojid b/builder/kojid index 06c14e7..bc0b5b3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5098,30 +5098,52 @@ class CreaterepoTask(BaseTaskHandler): % parseStatus(status, ' '.join(cmd))) def merge_repos(self, external_repos, arch, groupdata): - repos = [] + # group repos by merge type + repos_by_mode = {} + for repo in external_repos: + repos_by_mode.setdefault( + repo.get('merge_mode', 'koji'), []).append(repo) + + # figure out merge mode + if len(repos_by_mode) > 1: + # TODO: eventually support mixing merge modes + raise koji.GenericError('Found multiple merge modes for external ' + 'repos: %s\n' % repos_by_mode.keys()) + merge_mode = repos_by_mode.keys()[0] + + # move current repo to the premerge location localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id) os.rename(self.outdir, localdir) koji.ensuredir(self.outdir) - repos.append('file://' + localdir + '/') + # generate repo url list, starting with our local premerge repo + repos = ['file://' + localdir + '/'] for repo in external_repos: ext_url = repo['url'] # substitute $arch in the url with the arch of the repo we're generating ext_url = ext_url.replace('$arch', arch) repos.append(ext_url) - blocklist = self.repodir + '/blocklist' - if self.options.use_createrepo_c: + # construct command + if merge_mode == 'simple': + # currently only supported by our own mergerepos script + # (we need it to write pkgorigins) + cmd = ['/usr/libexec/kojid/mergerepos', + '--mode', 'simple', + '--tempdir', self.workdir] + elif self.options.use_createrepo_c: cmd = ['/usr/bin/mergerepo_c', '--koji'] else: cmd = ['/usr/libexec/kojid/mergerepos'] cmd.extend(['--tempdir', self.workdir]) + blocklist = self.repodir + '/blocklist' cmd.extend(['-a', arch, '-b', blocklist, '-o', self.outdir]) if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) for repo in repos: cmd.extend(['-r', repo]) + # run command logfile = '%s/mergerepos.log' % self.workdir status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True) if not isSuccess(status): diff --git a/builder/mergerepos b/builder/mergerepos index 5e4b357..dcd7647 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -72,6 +72,7 @@ def parse_args(args): help="List of arches to include in the repo") parser.add_option("-b", "--blocked", default=None, help="A file containing a list of srpm names to exclude from the merged repo") + parser.add_option("--mode", default='koji', help="Select the merge mode") parser.add_option("-o", "--outputdir", default=None, help="Location to create the repository") parser.add_option("--tempdir", default=None, @@ -112,7 +113,8 @@ def make_const_func(value): class RepoMerge(object): - def __init__(self, repolist, arches, groupfile, blocked, outputdir, tempdir=None): + def __init__(self, repolist, arches, groupfile, blocked, outputdir, + tempdir=None, mode='koji'): self.repolist = repolist self.outputdir = outputdir self.tempdir = tempdir @@ -134,6 +136,7 @@ class RepoMerge(object): self.archlist = arches self.mdconf.groupfile = groupfile self.blocked = blocked + self.mode = mode def close(self): if self.yumbase is not None: @@ -182,6 +185,10 @@ class RepoMerge(object): allows a package to be tracked back to its origin, even if the location field in the repodata does not match the original repo location. """ + + if self.mode == 'simple': + return self.do_simple_sort() + # sort the repos by _merge_rank # lowest number is the highest rank (1st place, 2nd place, etc.) repos = self.yumbase.repos.listEnabled() @@ -264,6 +271,55 @@ class RepoMerge(object): origins.close() self.mdconf.additional_metadata['origin'] = pkgorigins + def do_simple_sort(self): + """ + Handle the 'sort_and_filter' case when mode=simple + + As the name implies, this is a much simpler approach. Mainly, we need + to generate the pkgorigins file. + """ + + # sort the repos by _merge_rank + # lowest number is the highest rank (1st place, 2nd place, etc.) + repos = self.yumbase.repos.listEnabled() + repos.sort(key=lambda o: o._merge_rank) + + # TODO: reduce duplication between this function and sort_and_filter() + + # We lack the complex filtration of mode=koji, but we still need to: + # - fix urls for primary repo + # - enforce blocked list + for reponum, repo in enumerate(repos): + for pkg in repo.sack: + if reponum == 0 and not pkg.basepath: + # this is the first repo (i.e. the koji repo) and appears + # to be using relative urls + #XXX - kind of a hack, but yum leaves us little choice + #force the pkg object to report a relative location + loc = """\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True) + pkg._return_remote_location = make_const_func(loc) + + pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') + origins = open(pkgorigins, 'w') + + seen_rpms = {} + for repo in repos: + for pkg in repo.sack: + srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm) + pkg_nvra = str(pkg) + if pkg_nvra in seen_rpms: + sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra) + # note: we warn, but do not omit it + if srpm_name in self.blocked: + sys.stderr.write('Ignoring blocked package: %s\n\n' % + pkg.sourcerpm) + repo.sack.delPackage(pkg) + if pkg_nvra not in seen_rpms: + origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0])) + seen_rpms[pkg_nvra] = 1 + origins.close() + self.mdconf.additional_metadata['origin'] = pkgorigins + def write_metadata(self): self.mdconf.pkglist = self.yumbase.pkgSack self.mdconf.directory = self.outputdir @@ -292,7 +348,7 @@ def main(args): blocked = {} merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked, - opts.outputdir, opts.tempdir) + opts.outputdir, opts.tempdir, opts.mode) try: merge.merge_repos() diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 7341490..a5c87fa 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -5177,17 +5177,19 @@ def anon_handle_list_external_repos(goptions, session, args): header1 = "%-25s %s" % ("External repo name", "URL") header2 = "%s %s" % ("-"*25, "-"*40) elif format == "tag": - format = "%(priority)-3i %(external_repo_name)-25s %(url)s" - header1 = "%-3s %-25s %s" % ("Pri", "External repo name", "URL") - header2 = "%s %s %s" % ("-"*3, "-"*25, "-"*40) + format = "%(priority)-3i %(external_repo_name)-25s %(merge_mode)-10s %(url)s" + header1 = "%-3s %-25s %-10s URL" % ("Pri", "External repo name", "Mode") + header2 = "%s %s %s %s" % ("-"*3, "-"*25, "-"*10, "-"*40) elif format == "multitag": - format = "%(tag_name)-20s %(priority)-3i %(external_repo_name)s" - header1 = "%-20s %-3s %s" % ("Tag", "Pri", "External repo name") - header2 = "%s %s %s" % ("-"*20, "-"*3, "-"*25) + format = "%(tag_name)-20s %(priority)-3i %(merge_mode)-10s %(external_repo_name)s" + header1 = "%-20s %-3s %-10s %s" % ("Tag", "Pri", "Mode", "External repo name") + header2 = "%s %s %s %s" % ("-"*20, "-"*3, "-"*10, "-"*25) if not options.quiet: print(header1) print(header2) for rinfo in data: + # older hubs do not support merge_mode + rinfo.setdefault('merge_mode', None) print(format % rinfo) @@ -5227,8 +5229,14 @@ def handle_add_external_repo(goptions, session, args): help=_("Also add repo to tag. Use tag::N to set priority")) parser.add_option("-p", "--priority", type='int', help=_("Set priority (when adding to tag)")) + parser.add_option("-m", "--mode", help=_("Set merge mode")) (options, args) = parser.parse_args(args) activate_session(session, goptions) + if options.mode: + if options.mode not in koji.REPO_MERGE_MODES: + parser.error('Invalid mode: %s' % options.mode) + if not options.tag: + parser.error('The --mode option can only be used with --tag') if len(args) == 1: name = args[0] rinfo = session.getExternalRepo(name, strict=True) @@ -5249,7 +5257,10 @@ def handle_add_external_repo(goptions, session, args): priority = options.priority else: priority = _pick_external_repo_priority(session, tag) - session.addExternalRepoToTag(tag, rinfo['name'], priority) + callopts = {} + if options.mode: + callopts['merge_mode'] = options.mode + session.addExternalRepoToTag(tag, rinfo['name'], priority, **callopts) print("Added external repo %s to tag %s (priority %i)" \ % (rinfo['name'], tag, priority)) @@ -5261,6 +5272,7 @@ def handle_edit_external_repo(goptions, session, args): parser = OptionParser(usage=usage) parser.add_option("--url", help=_("Change the url")) parser.add_option("--name", help=_("Change the name")) + parser.add_option("-m", "--mode", help=_("Set merge mode")) (options, args) = parser.parse_args(args) if len(args) != 1: parser.error(_("Incorrect number of arguments")) @@ -5271,6 +5283,10 @@ def handle_edit_external_repo(goptions, session, args): opts['url'] = options.url if options.name: opts['name'] = options.name + if options.mode: + if options.mode not in koji.REPO_MERGE_MODES: + parser.error('Invalid mode: %s' % options.mode) + opts['merge_mode'] = options.mode if not opts: parser.error(_("No changes specified")) activate_session(session, goptions) diff --git a/docs/schema-upgrade-1.16-1.17.sql b/docs/schema-upgrade-1.16-1.17.sql index 9ee2254..6cb9ad6 100644 --- a/docs/schema-upgrade-1.16-1.17.sql +++ b/docs/schema-upgrade-1.16-1.17.sql @@ -7,4 +7,7 @@ BEGIN; -- Change VARCHAR field for build_target names to TEXT to allow longer names ALTER TABLE build_target ALTER COLUMN name TYPE TEXT; +-- Allow different merge modes for mergerepo +ALTER TABLE tag_external_repos ADD COLUMN merge_mode TEXT DEFAULT 'koji'; + COMMIT; diff --git a/docs/schema.sql b/docs/schema.sql index e525e86..86f799a 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -467,6 +467,7 @@ create table tag_external_repos ( tag_id INTEGER NOT NULL REFERENCES tag(id), external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), priority INTEGER NOT NULL, + merge_mode TEXT DEFAULT 'koji', -- versioned - see earlier description of versioning create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(), revoke_event INTEGER REFERENCES events(id), diff --git a/hub/kojihub.py b/hub/kojihub.py index 2364d0a..c6d13b8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3362,11 +3362,14 @@ def delete_external_repo(info): update.make_revoke() update.execute() -def add_external_repo_to_tag(tag_info, repo_info, priority): +def add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode='koji'): """Add an external repo to a tag""" context.session.assertPerm('admin') + if merge_mode not in koji.REPO_MERGE_MODES: + raise koji.GenericError('Invalid merge mode: %s' % merge_mode) + tag = get_tag(tag_info, strict=True) tag_id = tag['id'] repo = get_external_repo(repo_info, strict=True) @@ -3381,7 +3384,8 @@ def add_external_repo_to_tag(tag_info, repo_info, priority): (tag['name'], priority)) insert = InsertProcessor('tag_external_repos') - insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority) + insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority, + merge_mode=merge_mode) insert.make_create() insert.execute() @@ -3435,14 +3439,23 @@ def get_tag_external_repos(tag_info=None, repo_info=None, event=None): external_repo_id external_repo_name url + merge_mode priority """ tables = ['tag_external_repos'] joins = ['tag ON tag_external_repos.tag_id = tag.id', 'external_repo ON tag_external_repos.external_repo_id = external_repo.id', 'external_repo_config ON external_repo.id = external_repo_config.external_repo_id'] - columns = ['tag.id', 'tag.name', 'external_repo.id', 'external_repo.name', 'url', 'priority'] - aliases = ['tag_id', 'tag_name', 'external_repo_id', 'external_repo_name', 'url', 'priority'] + fields = { + 'external_repo.id': 'external_repo_id', + 'external_repo.name': 'external_repo_name', + 'priority': 'priority', + 'tag.id': 'tag_id', + 'tag.name': 'tag_name', + 'url': 'url', + 'merge_mode': 'merge_mode', + } + columns, aliases = zip(*fields.items()) clauses = [eventCondition(event, table='tag_external_repos'), eventCondition(event, table='external_repo_config')] if tag_info: @@ -3474,6 +3487,7 @@ def get_external_repo_list(tag_info, event=None): external_repo_id external_repo_name url + merge_mode priority """ tag = get_tag(tag_info, strict=True, event=event) @@ -6725,7 +6739,7 @@ def query_history(tables=None, **kwargs): 'external_repo_config': ['external_repo_id', 'url'], 'host_config': ['host_id', 'arches', 'capacity', 'description', 'comment', 'enabled'], 'host_channels': ['host_id', 'channel_id'], - 'tag_external_repos': ['tag_id', 'external_repo_id', 'priority'], + 'tag_external_repos': ['tag_id', 'external_repo_id', 'priority', 'merge_mode'], 'tag_listing': ['build_id', 'tag_id'], 'tag_packages': ['package_id', 'tag_id', 'owner', 'blocked', 'extra_arches'], 'group_config': ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', 'uservisible', @@ -9175,10 +9189,11 @@ class RootExports(object): editExternalRepo = staticmethod(edit_external_repo) deleteExternalRepo = staticmethod(delete_external_repo) - def addExternalRepoToTag(self, tag_info, repo_info, priority): + def addExternalRepoToTag(self, tag_info, repo_info, priority, + merge_mode='koji'): """Add an external repo to a tag""" # wrap the local method so we don't expose the event parameter - add_external_repo_to_tag(tag_info, repo_info, priority) + add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode) def removeExternalRepoFromTag(self, tag_info, repo_info): """Remove an external repo from a tag""" diff --git a/koji/__init__.py b/koji/__init__.py index 8786063..823ebcf 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -236,6 +236,8 @@ REPO_EXPIRED = REPO_STATES['EXPIRED'] REPO_DELETED = REPO_STATES['DELETED'] REPO_PROBLEM = REPO_STATES['PROBLEM'] +REPO_MERGE_MODES = set(['koji', 'simple']) + # buildroot states BR_STATES = Enum(( 'INIT',