PR#914: dist repo updates - koji

		`@@ -5055,7 +5055,7 @@`
		`self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)`
		`return [uploadpath, files]`

		`- def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None):`
		`+ def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):`
		`koji.ensuredir(self.outdir)`
		`if self.options.use_createrepo_c:`
		`cmd = ['/usr/bin/createrepo_c']`
		`@@ -5082,11 +5082,6 @@`
		`cmd.append('--update')`
		`if self.options.createrepo_skip_stat:`
		`cmd.append('--skip-stat')`
		`- if oldpkgs:`
		`- # generate delta-rpms`
		`- cmd.append('--deltas')`
		`- for op_dir in oldpkgs:`
		`- cmd.extend(['--oldpackagedirs', op_dir])`
		`# note: we can't easily use a cachedir because we do not have write`
		`# permission. The good news is that with --update we won't need to`
		`# be scanning many rpms.`
		`@@ -5139,10 +5134,9 @@`

		`def handler(self, tag, repo_id, keys, task_opts):`
		`tinfo = self.session.getTag(tag, strict=True, event=task_opts['event'])`
		`- path = koji.pathinfo.distrepo(repo_id, tinfo['name'])`
		`if len(task_opts['arch']) == 0:`
		`- arches = tinfo['arches'] or ''`
		`- task_opts['arch'] = arches.split()`
		`+ arches = tinfo['arches'] or ''`
		`+ task_opts['arch'] = arches.split()`
		`if len(task_opts['arch']) == 0:`
		`raise koji.GenericError('No arches specified nor for the tag!')`
		`subtasks = {}`
		`@@ -5162,13 +5156,12 @@`
		`method='createdistrepo', arglist=arglist, label=arch,`
		`parent=self.id, arch='noarch')`
		`if len(subtasks) > 0 and task_opts['multilib']:`
		`- results = self.wait(subtasks.values(), all=True, failany=True)`
		`+ self.wait(subtasks.values(), all=True, failany=True)`
		`for arch in arch32s:`
		`# move the 32-bit task output to the final resting place`
		`# so the 64-bit arches can use it for multilib`
		`- upload, files, sigmap = results[subtasks[arch]]`
		`- self.session.host.distRepoMove(`
		`- repo_id, upload, files, arch, sigmap)`
		`+ upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])`
		`+ self.session.host.distRepoMove(repo_id, upload_dir, arch)`
		`for arch in canonArches:`
		`# do the other arches`
		`if arch not in arch32s:`
		`@@ -5177,23 +5170,18 @@`
		`method='createdistrepo', arglist=arglist, label=arch,`
		`parent=self.id, arch='noarch')`
		`# wait for 64-bit subtasks to finish`
		`- data = {}`
		`- results = self.wait(subtasks.values(), all=True, failany=True)`
		`+ self.wait(subtasks.values(), all=True, failany=True)`
		`for (arch, task_id) in subtasks.iteritems():`
		`- data[arch] = results[task_id]`
		`- self.logger.debug("DEBUG: %r : %r " % (arch, data[arch]))`
		`if task_opts['multilib'] and arch in arch32s:`
		`# already moved above`
		`continue`
		`- #else`
		`- upload, files, sigmap = results[subtasks[arch]]`
		`- self.session.host.distRepoMove(`
		`- repo_id, upload, files, arch, sigmap)`
		`- self.session.host.repoDone(repo_id, data, expire=False)`
		`+ upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])`
		`+ self.session.host.distRepoMove(repo_id, upload_dir, arch)`
		`+ self.session.host.repoDone(repo_id, {}, expire=False)`
		`return 'Dist repository #%s successfully generated' % repo_id`


		`- class createDistRepoTask(CreaterepoTask):`
		`+ class createDistRepoTask(BaseTaskHandler):`
		`Methods = ['createdistrepo']`
		`_taskWeight = 1.5`

		`@@ -5223,17 +5211,17 @@`
		`self.rinfo = self.session.repoInfo(repo_id, strict=True)`
		`if self.rinfo['state'] != koji.REPO_INIT:`
		`raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo)`
		`- self.repo_id = self.rinfo['id']`
		`- self.pathinfo = koji.PathInfo(self.options.topdir)`
		`groupdata = os.path.join(`
		`- self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),`
		`+ koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),`
		`'groups', 'comps.xml')`
		`- #set up our output dir`
		`+`
		`+ # set up our output dir`
		`self.repodir = '%s/repo' % self.workdir`
		`+ self.repo_files = []`
		`koji.ensuredir(self.repodir)`
		`- self.outdir = self.repodir # workaround create_local_repo use`
		`- self.datadir = '%s/repodata' % self.repodir`
		`- self.sigmap = {}`
		`+ self.subrepos = set()`
		`+`
		`+ # gather oldpkgs data if delta option in use`
		`oldpkgs = []`
		`if opts.get('delta'):`
		`# should be a list of repo ids to delta against`
		`@@ -5246,63 +5234,136 @@`
		`path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name'])`
		`if not os.path.exists(path):`
		`raise koji.GenericError('Base drpm repo missing: %s' % path)`
		`+ # note: since we're using the top level dir, this will handle`
		`+ # split repos as well`
		`oldpkgs.append(path)`
		`+`
		`+ # sort out our package list(s)`
		`self.uploadpath = self.getUploadDir()`
		`- self.pkglist = self.make_pkglist(tag, arch, keys, opts)`
		`+ self.get_rpms(tag, arch, keys, opts)`
		`if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):`
		`self.do_multilib(arch, self.archmap[arch], opts['multilib'])`
		`+ self.split_pkgs(opts)`
		`self.write_kojipkgs()`
		`- self.logger.debug('package list is %s' % self.pkglist)`
		`- self.session.uploadWrapper(self.pkglist, self.uploadpath,`
		`- os.path.basename(self.pkglist))`
		`- if os.path.getsize(self.pkglist) == 0:`
		`- self.pkglist = None`
		`- self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs)`
		`- if self.pkglist is None:`
		`- fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')`
		`- fo.write("This repo is empty because its tag has no content for this arch\n")`
		`- fo.close()`
		`- files = ['pkglist', 'kojipkgs']`
		`- for f in os.listdir(self.datadir):`
		`- files.append(f)`
		`- self.session.uploadWrapper('%s/%s' % (self.datadir, f),`
		`- self.uploadpath, f)`
		`- if opts['delta']:`
		`- ddir = os.path.join(self.repodir, 'drpms')`
		`- for f in os.listdir(ddir):`
		`- files.append(f)`
		`- self.session.uploadWrapper('%s/%s' % (ddir, f),`
		`- self.uploadpath, f)`
		`- return [self.uploadpath, files, self.sigmap.items()]`
		`+ self.write_pkglist()`
		`+ self.link_pkgs()`
		`+`
		`+ # generate the repodata`
		`+ self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,`
		`+ groupdata, oldpkgs=oldpkgs)`
		`+ for subrepo in self.subrepos:`
		`+ self.do_createrepo(`
		`+ '%s/%s' % (self.repodir, subrepo),`
		`+ '%s/%s/pkglist' % (self.repodir, subrepo),`
		`+ groupdata, oldpkgs=oldpkgs,`
		`+ logname='createrepo_%s' % subrepo)`
		`+ if len(self.kojipkgs) == 0:`
		`+ fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")`
		`+ with open(fn, 'w') as fp:`
		`+ fp.write("This repo is empty because its tag has no content "`
		`+ "for this arch\n")`
		`+`
		`+ # upload repo files`
		`+ self.upload_repo()`
		`+ self.upload_repo_manifest()`
		`+`
		`+ def upload_repo_file(self, relpath):`
		`+ """Upload a file from the repo`
		`+`
		`+ relpath should be relative to self.repodir`
		`+ """`
		`+`
		`+ localpath = '%s/%s' % (self.repodir, relpath)`
		`+ reldir = os.path.dirname(relpath)`
		`+ if reldir:`
		`+ uploadpath = "%s/%s" % (self.uploadpath, reldir)`
		`+ fn = os.path.basename(relpath)`
		`+ else:`
		`+ uploadpath = self.uploadpath`
		`+ fn = relpath`
		`+ self.session.uploadWrapper(localpath, uploadpath, fn)`
		`+ self.repo_files.append(relpath)`
		`+`
		`+ def upload_repo(self):`
		`+ """Traverse the repo and upload needed files`
		`+`
		`+ We omit the symlinks we made for the rpms`
		`+ """`
		`+ for dirpath, dirs, files in os.walk(self.repodir):`
		`+ reldir = os.path.relpath(dirpath, self.repodir)`
		`+ for filename in files:`
		`+ path = "%s/%s" % (dirpath, filename)`
		`+ if os.path.islink(path):`
		`+ continue`
		`+ relpath = "%s/%s" % (reldir, filename)`
		`+ self.upload_repo_file(relpath)`
		`+`
		`+ def upload_repo_manifest(self):`
		`+ """Upload a list of the repo files we've uploaded"""`
		`+ fn = '%s/repo_manifest' % self.workdir`
		`+ with open(fn, 'w') as fp:`
		`+ json.dump(self.repo_files, fp, indent=4)`
		`+ self.session.uploadWrapper(fn, self.uploadpath)`
		`+`
		`+ def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):`
		`+ """Run createrepo`
		`+`
		`+ This is derived from CreaterepoTask.create_local_repo, but adapted to`
		`+ our requirements here`
		`+ """`
		`+ koji.ensuredir(repodir)`
		`+ if self.options.use_createrepo_c:`
		`+ cmd = ['/usr/bin/createrepo_c']`
		`+ else:`
		`+ cmd = ['/usr/bin/createrepo']`
		`+ cmd.extend(['-vd', '-i', pkglist])`
		`+ if groupdata and os.path.isfile(groupdata):`
		`+ cmd.extend(['-g', groupdata])`
		`+ # TODO: can we recycle data (with --update) as in create_local_repo?`
		`+ if oldpkgs:`
		`+ # generate delta-rpms`
		`+ cmd.append('--deltas')`
		`+ for op_dir in oldpkgs:`
		`+ cmd.extend(['--oldpackagedirs', op_dir])`
		`+ cmd.append(repodir)`
		`+`
		`+ if logname is None:`
		`+ logname = 'createrepo'`
		`+ logfile = '%s/%s.log' % (self.workdir, logname)`
		`+ status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)`
		`+ if not isSuccess(status):`
		`+ raise koji.GenericError('failed to create repo: %s' \`
		`+ % parseStatus(status, ' '.join(cmd)))`

		`def do_multilib(self, arch, ml_arch, conf):`
		`- self.repo_id = self.rinfo['id']`
		`- pathinfo = koji.PathInfo(self.options.topdir)`
		`- repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])`
		`+ repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])`
		`mldir = os.path.join(repodir, koji.canonArch(ml_arch))`
		`- ml_true = set() # multilib packages we need to include before depsolve`
		`- ml_conf = os.path.join(self.pathinfo.work(), conf)`
		`+ ml_true = set() # multilib packages we need to include before depsolve`
		`+ ml_conf = os.path.join(koji.pathinfo.work(), conf)`
		`+`
		`+ # read pkgs data from multilib repo`
		`+ ml_pkgfile = os.path.join(mldir, 'kojipkgs')`
		`+ ml_pkgs = json.load(open(ml_pkgfile, 'r'))`

		`# step 1: figure out which packages are multilib (should already exist)`
		`mlm = multilib.DevelMultilibMethod(ml_conf)`
		`fs_missing = set()`
		`- with open(self.pkglist) as pkglist:`
		`- for pkg in pkglist:`
		`- ppath = os.path.join(self.repodir, pkg.strip())`
		`+ for bnp in self.kojipkgs:`
		`+ rpminfo = self.kojipkgs[bnp]`
		`+ ppath = rpminfo['_pkgpath']`
		`po = yum.packages.YumLocalPackage(filename=ppath)`
		`- if mlm.select(po) and arch in self.archmap:`
		`+ if mlm.select(po):`
		`# we need a multilib package to be included`
		`- # we assume the same signature level is available`
		`- # XXX: what is a subarchitecture is the right answer?`
		`- pl_path = pkg.replace(arch, self.archmap[arch]).strip()`
		`- # assume this exists in the task results for the ml arch`
		`- real_path = os.path.join(mldir, pl_path)`
		`- if not os.path.exists(real_path):`
		`- self.logger.error('%s (multilib) is not on the filesystem' % real_path)`
		`- fs_missing.add(real_path)`
		`+ ml_bnp = bnp.replace(arch, self.archmap[arch])`
		`+ ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)`
		`+ # ^ XXX - should actually generate this`
		`+ if ml_bnp not in ml_pkgs:`
		`+ # not in our multilib repo`
		`+ self.logger.error('%s (multilib) is not on the filesystem' % ml_path)`
		`+ fs_missing.add(ml_path)`
		`# we defer failure so can report all the missing deps`
		`continue`
		`- ml_true.add(real_path)`
		`+ ml_true.add(ml_path)`

		`# step 2: set up architectures for yum configuration`
		`self.logger.info("Resolving multilib for %s using method devel" % arch)`
		`@@ -5392,29 +5453,22 @@`
		`raise koji.GenericError('multilib packages missing. '`
		`'See missing_multilib.log')`

		`- # get rpm ids for ml pkgs`
		`- kpkgfile = os.path.join(mldir, 'kojipkgs')`
		`- kojipkgs = json.load(open(kpkgfile, 'r'))`
		`-`
		`- # step 5: add dependencies to our package list`
		`- pkgwriter = open(self.pkglist, 'a')`
		`+ # step 5: update kojipkgs`
		`for dep_path in ml_needed:`
		`tspkg = ml_needed[dep_path]`
		`bnp = os.path.basename(dep_path)`
		`- bnplet = bnp[0].lower()`
		`- koji.ensuredir(os.path.join(self.repodir, bnplet))`
		`- dst = os.path.join(self.repodir, bnplet, bnp)`
		`- if os.path.exists(dst):`
		`+ if bnp in self.kojipkgs:`
		`# we expect duplication with noarch, but not other arches`
		`if tspkg.arch != 'noarch':`
		`- self.logger.warning("Path exists: %r", dst)`
		`+ self.logger.warning("Multilib duplicate: %s", bnp)`
		`continue`
		`- pkgwriter.write(bnplet + '/' + bnp + '\n')`
		`- self.logger.debug("os.symlink(%r, %r)", dep_path, dst)`
		`- os.symlink(dep_path, dst)`
		`- rpminfo = kojipkgs[bnp]`
		`- self.sigmap[rpminfo['id']] = rpminfo['sigkey']`
		`-`
		`+ rpminfo = ml_pkgs[bnp].copy()`
		`+ # fix _pkgpath, which comes from another task and could be wrong`
		`+ # for us`
		`+ # TODO: would be better if we could use the proper path here`
		`+ rpminfo['_pkgpath'] = dep_path`
		`+ rpminfo['_multilib'] = True`
		`+ self.kojipkgs[bnp] = rpminfo`

		`def pick_key(self, keys, avail_keys):`
		`best = None`
		`@@ -5430,21 +5484,20 @@`
		`best_idx = idx`
		`return best`

		`-`
		`- def make_pkglist(self, tag_id, arch, keys, opts):`
		`+ def get_rpms(self, tag_id, arch, keys, opts):`
		`# get the rpm data`
		`rpms = []`
		`builddirs = {}`
		`- for a in self.compat[arch] + ('noarch',):`
		`+ for a in self.compat[arch]:`
		`+ # note: self.compat includes noarch for non-src already`
		`rpm_iter, builds = self.session.listTaggedRPMS(tag_id,`
		`event=opts['event'], arch=a, latest=opts['latest'],`
		`inherit=opts['inherit'], rpmsigs=True)`
		`for build in builds:`
		`- builddirs[build['id']] = self.pathinfo.build(build)`
		`+ builddirs[build['id']] = koji.pathinfo.build(build)`
		`rpms += list(rpm_iter)`

		`# index by id and key`
		`- preferred = {}`
		`rpm_idx = {}`
		`for rpminfo in rpms:`
		`sigidx = rpm_idx.setdefault(rpminfo['id'], {})`
		`@@ -5464,9 +5517,7 @@`
		`else:`
		`selected[rpm_id] = rpm_idx[rpm_id][best_key]`

		`- #generate pkglist files`
		`- pkgfile = os.path.join(self.repodir, 'pkglist')`
		`- pkglist = file(pkgfile, 'w')`
		`+ # generate kojipkgs data and note missing files`
		`fs_missing = []`
		`sig_missing = []`
		`kojipkgs = {}`
		`@@ -5478,25 +5529,18 @@`
		`continue`
		`# use the primary copy, if allowed (checked below)`
		`pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],`
		`- self.pathinfo.rpm(rpminfo))`
		`+ koji.pathinfo.rpm(rpminfo))`
		`else:`
		`# use the signed copy`
		`pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],`
		`- self.pathinfo.signed(rpminfo, rpminfo['sigkey']))`
		`+ koji.pathinfo.signed(rpminfo, rpminfo['sigkey']))`
		`if not os.path.exists(pkgpath):`
		`fs_missing.append(pkgpath)`
		`# we'll raise an error below`
		`else:`
		`bnp = os.path.basename(pkgpath)`
		`- bnplet = bnp[0].lower()`
		`- pkglist.write(bnplet + '/' + bnp + '\n')`
		`- koji.ensuredir(os.path.join(self.repodir, bnplet))`
		`- self.sigmap[rpminfo['id']] = rpminfo['sigkey']`
		`- dst = os.path.join(self.repodir, bnplet, bnp)`
		`- self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)`
		`- os.symlink(pkgpath, dst)`
		`+ rpminfo['_pkgpath'] = pkgpath`
		`kojipkgs[bnp] = rpminfo`
		`- pkglist.close()`
		`self.kojipkgs = kojipkgs`

		`# report problems`
		`@@ -5534,19 +5578,54 @@`
		`and not opts['allow_missing_signatures']):`
		`raise koji.GenericError('Unsigned packages found. See '`
		`'missing_signatures.log')`
		`- return pkgfile`

		`+ def link_pkgs(self):`
		`+ for bnp in self.kojipkgs:`
		`+ bnplet = bnp[0].lower()`
		`+ ddir = os.path.join(self.repodir, 'Packages', bnplet)`
		`+ koji.ensuredir(ddir)`
		`+ dst = os.path.join(ddir, bnp)`
		`+ pkgpath = self.kojipkgs[bnp]['_pkgpath']`
		`+ self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)`
		`+ os.symlink(pkgpath, dst)`
		`+`
		`+ def split_pkgs(self, opts):`
		`+ '''Direct rpms to subrepos if needed'''`
		`+ for rpminfo in self.kojipkgs.values():`
		`+ if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']):`
		`+ rpminfo['_subrepo'] = 'debug'`
		`+ self.subrepos.add('debug')`
		`+`
		`+ def write_pkglist(self):`
		`+ pkgs = []`
		`+ subrepo_pkgs = {}`
		`+ for bnp in self.kojipkgs:`
		`+ rpminfo = self.kojipkgs[bnp]`
		`+ bnplet = bnp[0].lower()`
		`+ subrepo = rpminfo.get('_subrepo')`
		`+ if subrepo:`
		`+ # note the ../`
		`+ subrepo_pkgs.setdefault(subrepo, []).append(`
		`+ '../Packages/%s/%s\n' % (bnplet, bnp))`
		`+ else:`
		`+ pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))`
		`+`
		`+ with open('%s/pkglist' % self.repodir, 'w') as fo:`
		`+ for line in pkgs:`
		`+ fo.write(line)`
		`+ for subrepo in subrepo_pkgs:`
		`+ koji.ensuredir('%s/%s' % (self.repodir, subrepo))`
		`+ with open('%s/%s/pkglist' % (self.repodir, subrepo), 'w') as fo:`
		`+ for line in subrepo_pkgs[subrepo]:`
		`+ fo.write(line)`

		`def write_kojipkgs(self):`
		`filename = os.path.join(self.repodir, 'kojipkgs')`
		`datafile = file(filename, 'w')`
		`try:`
		`- json.dump(self.kojipkgs, datafile, indent=4)`
		`+ json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True)`
		`finally:`
		`datafile.close()`
		`- # and upload too`
		`- self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')`
		`-`


		`class WaitrepoTask(BaseTaskHandler):`

cli/koji_cli/commands.py

file modified

+10 -5

		`@@ -6911,10 +6911,13 @@`
		`default=False,`
		`help=_('For RPMs not signed with a desired key, fall back to the '`
		`'primary copy'))`
		`- parser.add_option("--arch", action='append', default=[],`
		`+ parser.add_option("-a", "--arch", action='append', default=[],`
		`help=_("Indicate an architecture to consider. The default is all " +`
		`"architectures associated with the given tag. This option may " +`
		`"be specified multiple times."))`
		`+ parser.add_option("--with-src", action='store_true', help='Also generate a src repo')`
		`+ parser.add_option("--split-debuginfo", action='store_true', default=False,`
		`+ help='Split debuginfo info a separate repo for each arch')`
		`parser.add_option('--comps', help='Include a comps file in the repodata')`
		`parser.add_option('--delta-rpms', metavar='REPO',default=[],`
		`action='append',`
		`@@ -6922,7 +6925,7 @@`
		`'or the name of a tag that has a dist repo. May be specified '`
		`'multiple times.'))`
		`parser.add_option('--event', type='int',`
		`- help=_('create a dist repository based on a Brew event'))`
		`+ help=_('Use tag content at event'))`
		`parser.add_option('--non-latest', dest='latest', default=True,`
		`action='store_false', help='Include older builds, not just the latest')`
		`parser.add_option('--multilib', default=None, metavar="CONFIG",`
		`@@ -6996,9 +6999,10 @@`
		`task_opts.multilib = os.path.join(stuffdir,`
		`os.path.basename(task_opts.multilib))`
		`print('')`
		`- for f in ('noarch', 'src'):`
		`- if f in task_opts.arch:`
		`- task_opts.arch.remove(f)`
		`+ if 'noarch' in task_opts.arch:`
		`+ task_opts.arch.remove('noarch')`
		`+ if task_opts.with_src and 'src' not in task_opts.arch:`
		`+ task_opts.arch.append('src')`
		`opts = {`
		`'arch': task_opts.arch,`
		`'comps': task_opts.comps,`
		`@@ -7007,6 +7011,7 @@`
		`'inherit': not task_opts.noinherit,`
		`'latest': task_opts.latest,`
		`'multilib': task_opts.multilib,`
		`+ 'split_debuginfo': task_opts.split_debuginfo,`
		`'skip_missing_signatures': task_opts.skip_missing_signatures,`
		`'allow_missing_signatures': task_opts.allow_missing_signatures`
		`}`

hub/kojihub.py

file modified

+89 -62

		`@@ -2470,7 +2470,7 @@`
		`tinfo = get_tag(tag, strict=True)`
		`tag_id = tinfo['id']`
		`event = task_opts.get('event')`
		`- arches = set([koji.canonArch(a) for a in task_opts['arch']])`
		`+ arches = list(set([koji.canonArch(a) for a in task_opts['arch']]))`
		`# note: we need to match args from the other preRepoInit callback`
		`koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,`
		`with_debuginfo=False, event=event, repo_id=None,`
		`@@ -2493,7 +2493,9 @@`
		`task_opts['comps']), groupsdir + '/comps.xml')`
		`# note: we need to match args from the other postRepoInit callback`
		`koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,`
		`- with_debuginfo=False, event=event, repo_id=repo_id)`
		`+ with_debuginfo=False, event=event, repo_id=repo_id,`
		`+ dist=True, keys=keys, arches=arches, task_opts=task_opts,`
		`+ repodir=repodir)`
		`return repo_id, event`


		`@@ -2550,15 +2552,23 @@`
		`repo_set_state(repo_id, koji.REPO_DELETED)`
		`return len(references)`

		`- def repo_expire_older(tag_id, event_id):`
		`- """Expire repos for tag older than event"""`
		`+`
		`+ def repo_expire_older(tag_id, event_id, dist=None):`
		`+ """Expire repos for tag older than event`
		`+`
		`+ If dist is not None, then only expire repos with the given dist value`
		`+ """`
		`st_ready = koji.REPO_READY`
		`- st_expired = koji.REPO_EXPIRED`
		`- q = """UPDATE repo SET state=%(st_expired)i`
		`- WHERE tag_id = %(tag_id)i`
		`- AND create_event < %(event_id)i`
		`- AND state = %(st_ready)i"""`
		`- _dml(q, locals())`
		`+ clauses=['tag_id = %(tag_id)s',`
		`+ 'create_event < %(event_id)s',`
		`+ 'state = %(st_ready)s']`
		`+ if dist is not None:`
		`+ dist = bool(dist)`
		`+ clauses.append('dist = %(dist)s')`
		`+ update = UpdateProcessor('repo', values=locals(), clauses=clauses)`
		`+ update.set(state=koji.REPO_EXPIRED)`
		`+ update.execute()`
		`+`

		`def repo_references(repo_id):`
		`"""Return a list of buildroots that reference the repo"""`
		`@@ -12564,14 +12574,21 @@`
		`safer_move(filepath, dst)`

		`def repoDone(self, repo_id, data, expire=False):`
		`- """Move repo data into place, mark as ready, and expire earlier repos`
		`+ """Finalize a repo`

		`repo_id: the id of the repo`
		`- data: a dictionary of the form { arch: (uploadpath, files), ...}`
		`- expire(optional): if set to true, mark the repo expired immediately*`
		`+ data: a dictionary of repo files in the form:`
		`+ { arch: [uploadpath, [file1, file2, ...]], ...}`
		`+ expire: if set to true, mark the repo expired immediately [*]`
		`+`
		`+ Actions:`
		`+ * Move uploaded repo files into place`
		`+ * Mark repo ready`
		`+ * Expire earlier repos`
		`+ * Move/create 'latest' symlink`

		`- If this is a dist repo, also hardlink the rpms in the final`
		`- directory.`
		`+ For dist repos, the move step is skipped (that is handled in`
		`+ distRepoMove).`

		`* This is used when a repo from an older event is generated`
		`"""`
		`@@ -12605,7 +12622,7 @@`
		`return`
		`#else:`
		`repo_ready(repo_id)`
		`- repo_expire_older(rinfo['tag_id'], rinfo['create_event'])`
		`+ repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist'])`

		`#make a latest link`
		`if rinfo['dist']:`
		`@@ -12623,25 +12640,22 @@`
		`koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)`


		`- def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):`
		`+ def distRepoMove(self, repo_id, uploadpath, arch):`
		`"""`
		`- Move a dist repo into its final location`
		`+ Move one arch of a dist repo into its final location`

		`-`
		`- Unlike normal repos (which are moved into place by repoDone), dist`
		`- repos have all their content linked (or copied) into place.`
		`+ Unlike normal repos, dist repos have all their content linked (or`
		`+ copied) into place.`

		`repo_id - the repo to move`
		`uploadpath - where the uploaded files are`
		`- files - a list of the uploaded file names`
		`arch - the arch of the repo`
		`- sigmap - a list of [rpm_id, sig] pairs`

		`- The rpms from sigmap should match the contents of the uploaded pkglist`
		`- file.`
		`+ uploadpath should contain a repo_manifest file`

		`- In sigmap, use sig=None to use the primary copy of the rpm instead of a`
		`- signed copy.`
		`+ The uploaded files should include:`
		`+ - kojipkgs: json file with information about the component rpms`
		`+ - repo metadata files`
		`"""`
		`host = Host()`
		`host.verify()`
		`@@ -12651,33 +12665,48 @@`
		`archdir = "%s/%s" % (repodir, koji.canonArch(arch))`
		`if not os.path.isdir(archdir):`
		`raise koji.GenericError("Repo arch directory missing: %s" % archdir)`
		`- datadir = "%s/repodata" % archdir`
		`- koji.ensuredir(datadir)`
		`+ repo_state = koji.REPO_STATES[rinfo['state']]`
		`+ if repo_state != 'INIT':`
		`+ raise koji.GenericError('Repo is in state: %s' % repo_state)`

		`- pkglist = set()`
		`- for fn in files:`
		`- src = "%s/%s/%s" % (workdir, uploadpath, fn)`
		`- if fn.endswith('.drpm'):`
		`- koji.ensuredir(os.path.join(archdir, 'drpms'))`
		`- dst = "%s/drpms/%s" % (archdir, fn)`
		`- elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):`
		`- dst = '%s/%s' % (archdir, fn)`
		`- else:`
		`- dst = "%s/%s" % (datadir, fn)`
		`+ # read manifest`
		`+ fn = '%s/%s/repo_manifest' % (workdir, uploadpath)`
		`+ if not os.path.isfile(fn):`
		`+ raise koji.GenericError('Missing repo manifest')`
		`+ with open(fn) as fp:`
		`+ files = json.load(fp)`
		`+`
		`+ # Read package data`
		`+ fn = '%s/%s/kojipkgs' % (workdir, uploadpath)`
		`+ if not os.path.isfile(fn):`
		`+ raise koji.GenericError('Missing kojipkgs file')`
		`+ with open(fn) as fp:`
		`+ kojipkgs = json.load(fp)`
		`+`
		`+ # Figure out where to send the uploaded files`
		`+ file_moves = []`
		`+ for relpath in files:`
		`+ src = "%s/%s/%s" % (workdir, uploadpath, relpath)`
		`+ dst = "%s/%s" % (archdir, relpath)`
		`if not os.path.exists(src):`
		`raise koji.GenericError("uploaded file missing: %s" % src)`
		`- if fn.endswith('pkglist'):`
		`- with open(src) as pkgfile:`
		`- for pkg in pkgfile:`
		`- pkg = os.path.basename(pkg.strip())`
		`- pkglist.add(pkg)`
		`- safer_move(src, dst)`
		`+ file_moves.append([src, dst])`

		`# get rpms`
		`build_dirs = {}`
		`rpmdata = {}`
		`- for rpm_id, sigkey in sigmap:`
		`- rpminfo = get_rpm(rpm_id, strict=True)`
		`+ rpm_check_keys = ['name', 'version', 'release', 'arch', 'epoch',`
		`+ 'size', 'payloadhash', 'build_id']`
		`+ for bnp in kojipkgs:`
		`+ rpminfo = kojipkgs[bnp]`
		`+ rpm_id = rpminfo['id']`
		`+ sigkey = rpminfo['sigkey']`
		`+ _rpminfo = get_rpm(rpm_id, strict=True)`
		`+ for key in rpm_check_keys:`
		`+ if key not in rpminfo or rpminfo[key] != _rpminfo[key]:`
		`+ raise koji.GenericError(`
		`+ 'kojipkgs entry does not match db: file %s, key %s'`
		`+ % (bnp, key))`
		`if sigkey is None or sigkey == '':`
		`relpath = koji.pathinfo.rpm(rpminfo)`
		`else:`
		`@@ -12690,26 +12719,25 @@`
		`builddir = koji.pathinfo.build(binfo)`
		`build_dirs[rpminfo['build_id']] = builddir`
		`rpminfo['_fullpath'] = os.path.join(builddir, relpath)`
		`- basename = os.path.basename(relpath)`
		`- rpmdata[basename] = rpminfo`
		`+ rpmdata[bnp] = rpminfo`

		`- # sanity check`
		`- for fn in rpmdata:`
		`- if fn not in pkglist:`
		`- raise koji.GenericError("No signature data for: %s" % fn)`
		`- for fn in pkglist:`
		`- if fn not in rpmdata:`
		`- raise koji.GenericError("RPM missing from pkglist: %s" % fn)`
		`+ # move the uploaded files`
		`+ dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])`
		`+ for dirname in dirnames:`
		`+ koji.ensuredir(dirname)`
		`+ for src, dst in file_moves:`
		`+ safer_move(src, dst)`

		`+ # hardlink or copy the rpms into the final repodir`
		`+ # TODO: properly consider split-volume functionality`
		`for fn in rpmdata:`
		`- # hardlink or copy the rpms into the final repodir`
		`- # TODO: properly consider split-volume functionality`
		`rpminfo = rpmdata[fn]`
		`rpmpath = rpminfo['_fullpath']`
		`bnp = fn`
		`bnplet = bnp[0].lower()`
		`- koji.ensuredir(os.path.join(archdir, bnplet))`
		`- l_dst = os.path.join(archdir, bnplet, bnp)`
		`+ ddir = os.path.join(archdir, 'Packages', bnplet)`
		`+ koji.ensuredir(ddir)`
		`+ l_dst = os.path.join(ddir, bnp)`
		`if os.path.exists(l_dst):`
		`raise koji.GenericError("File already in repo: %s", l_dst)`
		`logger.debug("os.link(%r, %r)", rpmpath, l_dst)`
		`@@ -12717,8 +12745,7 @@`
		`os.link(rpmpath, l_dst)`
		`except OSError as ose:`
		`if ose.errno == 18:`
		`- shutil.copy2(`
		`- rpmpath, os.path.join(archdir, bnplet, bnp))`
		`+ shutil.copy2(rpmpath, l_dst)`
		`else:`
		`raise`

tests/test_cli/test_dist_repo.py

file modified

+4 -2

		`@@ -252,14 +252,16 @@`
		`--allow-missing-signatures`
		`For RPMs not signed with a desired key, fall back to`
		`the primary copy`
		`- --arch=ARCH Indicate an architecture to consider. The default is`
		`+ -a ARCH, --arch=ARCH Indicate an architecture to consider. The default is`
		`all architectures associated with the given tag. This`
		`option may be specified multiple times.`
		`+ --with-src Also generate a src repo`
		`+ --split-debuginfo Split debuginfo info a separate repo for each arch`
		`--comps=COMPS Include a comps file in the repodata`
		`--delta-rpms=REPO Create delta rpms. REPO can be the id of another dist`
		`repo or the name of a tag that has a dist repo. May be`
		`specified multiple times.`
		`- --event=EVENT create a dist repository based on a Brew event`
		`+ --event=EVENT Use tag content at event`
		`--non-latest Include older builds, not just the latest`
		`--multilib=CONFIG Include multilib packages in the repository using the`
		`given config file`

tests/test_hub/test_dist_repo.py

file modified

+23 -9

		`@@ -1,5 +1,6 @@`

		`import unittest`
		`+ import json`
		`import mock`
		`import os`
		`import shutil`
		`@@ -106,7 +107,7 @@`
		`'create_ts': 1487256924.72718,`
		`'creation_time': '2017-02-16 14:55:24.727181',`
		`'id': 47,`
		`- 'state': 1,`
		`+ 'state': 0, # INIT`
		`'tag_id': 2,`
		`'tag_name': 'my-tag'}`
		`self.arch = 'x86_64'`
		`@@ -123,19 +124,18 @@`
		`os.makedirs(uploaddir)`

		`# place some test files`
		`- self.files = ['foo.drpm', 'repomd.xml']`
		`+ self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']`
		`self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']`
		`for fn in self.files:`
		`path = os.path.join(uploaddir, fn)`
		`koji.ensuredir(os.path.dirname(path))`
		`with open(path, 'w') as fo:`
		`- fo.write('%s' % fn)`
		`+ fo.write('%s' % os.path.basename(fn))`

		`- # generate pkglist file and sigmap`
		`+ # generate pkglist file`
		`self.files.append('pkglist')`
		`plist = os.path.join(uploaddir, 'pkglist')`
		`nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']`
		`- self.sigmap = []`
		`self.rpms = {}`
		`self.builds ={}`
		`self.key = '4c8da725'`
		`@@ -153,15 +153,30 @@`
		`fo.write('%s' % basename)`
		`f_pkglist.write(path)`
		`f_pkglist.write('\n')`
		`- self.expected.append('x86_64/%s/%s' % (basename[0], basename))`
		`+ self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))`
		`build_id = len(self.builds) + 10000`
		`rpm_id = len(self.rpms) + 20000`
		`binfo['id'] = build_id`
		`rpminfo['build_id'] = build_id`
		`rpminfo['id'] = rpm_id`
		`+ rpminfo['sigkey'] = self.key`
		`+ rpminfo['size'] = 1024`
		`+ rpminfo['payloadhash'] = 'helloworld'`
		`self.builds[build_id] = binfo`
		`self.rpms[rpm_id] = rpminfo`
		`- self.sigmap.append([rpm_id, self.key])`
		`+`
		`+ # write kojipkgs`
		`+ kojipkgs = {}`
		`+ for rpminfo in self.rpms.values():`
		`+ bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo`
		`+ kojipkgs[bnp] = rpminfo`
		`+ with open("%s/kojipkgs" % uploaddir, "w") as fp:`
		`+ json.dump(kojipkgs, fp, indent=4)`
		`+ self.files.append('kojipkgs')`
		`+`
		`+ # write manifest`
		`+ with open("%s/repo_manifest" % uploaddir, "w") as fp:`
		`+ json.dump(self.files, fp, indent=4)`

		`# mocks`
		`self.repo_info = mock.patch('kojihub.repo_info').start()`
		`@@ -187,8 +202,7 @@`

		`def test_distRepoMove(self):`
		`exports = kojihub.HostExports()`
		`- exports.distRepoMove(self.rinfo['id'], self.uploadpath,`
		`- list(self.files), self.arch, self.sigmap)`
		`+ exports.distRepoMove(self.rinfo['id'], self.uploadpath, self.arch)`
		`# check result`
		`repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo`
		`for relpath in self.expected:`

tests/test_hub/test_repos.py

file added

+75

		`@@ -0,0 +1,75 @@`
		`+ import mock`
		`+ import unittest`
		`+`
		`+ import koji`
		`+ import kojihub`
		`+`
		`+`
		`+ QP = kojihub.QueryProcessor`
		`+ IP = kojihub.InsertProcessor`
		`+ UP = kojihub.UpdateProcessor`
		`+`
		`+`
		`+ class TestRepoFunctions(unittest.TestCase):`
		`+`
		`+ def setUp(self):`
		`+ self.QueryProcessor = mock.patch('kojihub.QueryProcessor',`
		`+ side_effect=self.getQuery).start()`
		`+ self.queries = []`
		`+ self.InsertProcessor = mock.patch('kojihub.InsertProcessor',`
		`+ side_effect=self.getInsert).start()`
		`+ self.inserts = []`
		`+ self.UpdateProcessor = mock.patch('kojihub.UpdateProcessor',`
		`+ side_effect=self.getUpdate).start()`
		`+ self.updates = []`
		`+ self._dml = mock.patch('kojihub._dml').start()`
		`+`
		`+ def tearDown(self):`
		`+ mock.patch.stopall()`
		`+`
		`+ def getQuery(self, args, *kwargs):`
		`+ query = QP(args, *kwargs)`
		`+ query.execute = mock.MagicMock()`
		`+ self.queries.append(query)`
		`+ return query`
		`+`
		`+ def getInsert(self, args, *kwargs):`
		`+ insert = IP(args, *kwargs)`
		`+ insert.execute = mock.MagicMock()`
		`+ self.inserts.append(insert)`
		`+ return insert`
		`+`
		`+ def getUpdate(self, args, *kwargs):`
		`+ update = UP(args, *kwargs)`
		`+ update.execute = mock.MagicMock()`
		`+ self.updates.append(update)`
		`+ return update`
		`+`
		`+ def test_repo_expire_older(self):`
		`+ kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id)`
		`+ self.assertEqual(len(self.updates), 1)`
		`+ update = self.updates[0]`
		`+ self.assertEqual(update.table, 'repo')`
		`+ self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})`
		`+ self.assertEqual(update.rawdata, {})`
		`+ self.assertEqual(update.values['event_id'], mock.sentinel.event_id)`
		`+ self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)`
		`+ self.assertEqual(update.values['dist'], None)`
		`+ if 'dist = %(dist)s' in update.clauses:`
		`+ raise Exception('Unexpected dist condition')`
		`+`
		`+ # and with dist specified`
		`+ for dist in True, False:`
		`+ self.updates = []`
		`+ kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id,`
		`+ dist=dist)`
		`+ self.assertEqual(len(self.updates), 1)`
		`+ update = self.updates[0]`
		`+ self.assertEqual(update.table, 'repo')`
		`+ self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})`
		`+ self.assertEqual(update.rawdata, {})`
		`+ self.assertEqual(update.values['event_id'], mock.sentinel.event_id)`
		`+ self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)`
		`+ self.assertEqual(update.values['dist'], dist)`
		`+ if 'dist = %(dist)s' not in update.clauses:`
		`+ raise Exception('Missing dist condition')`

www/kojiweb/taskinfo.chtml

file modified

+2 -2

		`@@ -225,7 +225,7 @@`
		`#end if`
		`#elif $task.method == 'distRepo'`
		`<strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>`
		`- <strong>Repo ID:</strong> $params[1]<br/>`
		`+ <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>`
		`<strong>Keys:</strong> $printValue(0, $params[2])<br/>`
		`$printOpts($params[3])`
		`#elif $task.method == 'prepRepo'`
		`@@ -243,7 +243,7 @@`
		`#end if`
		`#elif $task.method == 'createdistrepo'`
		`<strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>`
		`- <strong>Repo ID:</strong> $params[1]<br/>`
		`+ <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>`
		`<strong>Arch:</strong> $printValue(0, $params[2])<br/>`
		`<strong>Keys:</strong> $printValue(0, $params[3])<br/>`
		`<strong>Options:</strong> $printMap($params[4], '    ')`

mikem commented 4 years ago

support split debuginfo
avoid expiring not-dist repos
general cleanup

Fixes #409
Fixes #457

ngompa commented 4 years ago

:thumbsup:

tkopecek commented on line 400 of builder/kojid 4 years ago

What about os.path.join(self.repodir, 'Packages', bnplet, bnp)? It is more close to current repos (at least in Fedora). Or maybe make it configurable?

tkopecek commented on line 409 of builder/kojid 4 years ago

Hmm, also unexpected structure. What about placing 'normal' packages to 'os', so all subrepos are on same (highest) level?

mikem commented 4 years ago

What about os.path.join(self.repodir, 'Packages', bnplet, bnp)? It is more close to current repos (at least in Fedora). Or maybe make it configurable?

This is the layout dist repos had before. Changing that wasn't on the list, though certainly we could.

ngompa commented 4 years ago

Are we talking about a layout like so?

dist-arch
|
|---- Packages
|---- repodata
|---- debug
     |
     |---- Packages
     |---- repodata

Edited 4 years ago by ngompa

mikem commented 4 years ago

Hmm, also unexpected structure. What about placing 'normal' packages to 'os', so all subrepos are on same (highest) level?

Trying not to break existing paths, but maybe that doesn't matter. I'm not sure how many folks are relying on the feature as-is, or how much trouble it would be for them to change their urls.

'os' might not make sense for all tags, though.

tkopecek commented 4 years ago

Current PR creates following:

tag/repo_id
├── src
│   ├── a
│   ├── repodata
│   └── z
└── x86_64
    ├── a
    ├── debug
    │   └── repodata
    ├── repodata
    └── z

I've suggested:

tag/repo_id
├── src
│   ├── Packages
│   │   ├── a
│   │   └── z
│   └── repodata
└── x86_64
    ├── debug
    │   ├── Packages       
    │   │   ├── a
    │   │   └── z
    │   └── repodata
    └── os
        ├── Packages       
        │   ├── a
        │   └── z
        └── repodata

But yes, it is not compatible with current layout. On the other hand, I find it little messy to have all these data mixed in one level (especially letter-based dirs with its repodata and debug directory).

About 'os' - do you mean just the name doesn't make sense, or even location when everything is put together (default behaviour)?

ngompa commented 4 years ago

@tkopecek I actually prefer your suggested layout, but the name os is definitely wrong. Maybe make that name configurable, so that it can be release, updates, testing, etc.?

The reason I like your layout is that it makes it easy to merge tag exports together easily on the filesystem.

That said, the current layout proposed probably makes sense too, and is less difficult for naming. And the tag/repo-id already exports out as a top level directory...

Edited 4 years ago by ngompa

mikem commented 4 years ago

About 'os' - do you mean just the name doesn't make sense, or even location when everything is put together (default behaviour)?

I mean the name. The content of a tag might not constitute an os. It could just be a small set of packages.

mikem commented 4 years ago

So, we're inflating the feature quite a lot here....

ngompa commented 4 years ago

It'd probably be easier to go without the os directory. The current proposed layout works, though I like the idea of the [a-z] directory being under a Packages directory.

Edited 4 years ago by ngompa

tkopecek commented 4 years ago

Yep, I think we can just store rpms in Packages dirs, so there is a little cleanup and no change to current repomd path.

mikem commented 4 years ago

Ok, that much is easy enough and not a compat problem

mikem commented 4 years ago

Will update PR shortly

ngompa commented 4 years ago

@mikem: Would it also be possible to consider lib{,x32,64}* as a separate prefix to l? This would be similar to what Debian/Ubuntu does.

For example: http://archive.ubuntu.com/ubuntu/pool/main/

mikem commented 4 years ago

Oh, because of dir size? Interesting, but I don't know if we need to rush that in right now.

ngompa commented 4 years ago

@mikem Yes. This gets really painful in Mageia because we do lib packages for every library, so the enumeration of the l directory is obnoxious.

mikem commented 4 years ago

For repos, traversing the directory should rarely be done. The whole point of a repo is to use the repodata as an index.

ngompa commented 4 years ago

@mikem It's mainly for partial syncs and a few other things done by people in our community...

mikem commented 4 years ago

In Fedora, l is one of the bad ones, but not nearly as bad as p.

$ cat rpms | cut -c 1|sort|uniq -c|sort -n|tail
   2128 a
   2427 c
   3007 s
   3021 n
   3039 r
   3646 m
   6316 l
   6458 g
   7576 t
  13858 p

So, I'm open to making an adjustment here, but probably not for 1.16, which we are currently trying to freeze for.

mikem commented 4 years ago

that's f28 x86_64 data btw. Looks like p accounts for almost 20% of rpm names there. l is only 8.9%. For reference 1/26 is about 3.8%.

ngompa commented 4 years ago

For current Mageia Cauldron, this is what it looks like:

$ cat cauldron-rpms | cut -c 1|sort|uniq -c|sort -n|tail
    715 o
    833 j
    899 n
    954 a
   1008 r
   1356 s
   1452 g
   1500 m
   6859 l
   7340 p

p is a bit higher than l, but they're both up there...

mikem commented 4 years ago

This probably needs to be either 1) a configurable regex, or 2) dynamically split based on the content in the repo. Either way, I don't think we need to hold up 1.16 over it. Do you?

mikem commented 4 years ago

Maybe you could file this is a separate rfe?

ngompa commented 4 years ago

@mikem No, I don't think it needs to be held up, but I wanted to give that initial feedback for consideration.

ngompa commented 4 years ago

@mikem For what it's worth, a configurable regex makes sense, because we may want to also split on perl- and python[23]- prefixes, too...

1 new commit added