From 18600b24c19b3965b2b9ded7804483ad350cbac1 Mon Sep 17 00:00:00 2001
From: Mike McLean <mikem@redhat.com>
Date: May 03 2018 20:08:09 +0000
Subject: PR#914: dist repo updates


Merges #914
https://pagure.io/koji/pull-request/914

Fixes: #409
https://pagure.io/koji/issue/409
dist-repos are missing critical functionality

Fixes: #457
https://pagure.io/koji/issue/457
creating dist-repos can expire regular (non-dist) repos for same tag

---

diff --git a/builder/kojid b/builder/kojid
index 42d6269..eb78d94 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -5055,7 +5055,7 @@ class CreaterepoTask(BaseTaskHandler):
             self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
         return [uploadpath, files]
 
-    def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None):
+    def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
         koji.ensuredir(self.outdir)
         if self.options.use_createrepo_c:
             cmd = ['/usr/bin/createrepo_c']
@@ -5082,11 +5082,6 @@ class CreaterepoTask(BaseTaskHandler):
                 cmd.append('--update')
                 if self.options.createrepo_skip_stat:
                     cmd.append('--skip-stat')
-        if oldpkgs:
-            # generate delta-rpms
-            cmd.append('--deltas')
-            for op_dir in oldpkgs:
-                cmd.extend(['--oldpackagedirs', op_dir])
         # note: we can't easily use a cachedir because we do not have write
         # permission. The good news is that with --update we won't need to
         # be scanning many rpms.
@@ -5139,10 +5134,9 @@ class NewDistRepoTask(BaseTaskHandler):
 
     def handler(self, tag, repo_id, keys, task_opts):
         tinfo = self.session.getTag(tag, strict=True, event=task_opts['event'])
-        path = koji.pathinfo.distrepo(repo_id, tinfo['name'])
         if len(task_opts['arch']) == 0:
-             arches = tinfo['arches'] or ''
-             task_opts['arch'] = arches.split()
+            arches = tinfo['arches'] or ''
+            task_opts['arch'] = arches.split()
         if len(task_opts['arch']) == 0:
             raise koji.GenericError('No arches specified nor for the tag!')
         subtasks = {}
@@ -5162,13 +5156,12 @@ class NewDistRepoTask(BaseTaskHandler):
                 method='createdistrepo', arglist=arglist, label=arch,
                 parent=self.id, arch='noarch')
         if len(subtasks) > 0 and task_opts['multilib']:
-            results = self.wait(subtasks.values(), all=True, failany=True)
+            self.wait(subtasks.values(), all=True, failany=True)
             for arch in arch32s:
                 # move the 32-bit task output to the final resting place
                 # so the 64-bit arches can use it for multilib
-                upload, files, sigmap = results[subtasks[arch]]
-                self.session.host.distRepoMove(
-                    repo_id, upload, files, arch, sigmap)
+                upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
+                self.session.host.distRepoMove(repo_id, upload_dir, arch)
         for arch in canonArches:
             # do the other arches
             if arch not in arch32s:
@@ -5177,23 +5170,18 @@ class NewDistRepoTask(BaseTaskHandler):
                     method='createdistrepo', arglist=arglist, label=arch,
                     parent=self.id, arch='noarch')
         # wait for 64-bit subtasks to finish
-        data = {}
-        results = self.wait(subtasks.values(), all=True, failany=True)
+        self.wait(subtasks.values(), all=True, failany=True)
         for (arch, task_id) in subtasks.iteritems():
-            data[arch] = results[task_id]
-            self.logger.debug("DEBUG: %r : %r " % (arch, data[arch]))
             if task_opts['multilib'] and arch in arch32s:
                 # already moved above
                 continue
-            #else
-            upload, files, sigmap = results[subtasks[arch]]
-            self.session.host.distRepoMove(
-                repo_id, upload, files, arch, sigmap)
-        self.session.host.repoDone(repo_id, data, expire=False)
+            upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
+            self.session.host.distRepoMove(repo_id, upload_dir, arch)
+        self.session.host.repoDone(repo_id, {}, expire=False)
         return 'Dist repository #%s successfully generated' % repo_id
 
 
-class createDistRepoTask(CreaterepoTask):
+class createDistRepoTask(BaseTaskHandler):
     Methods = ['createdistrepo']
     _taskWeight = 1.5
 
@@ -5223,17 +5211,17 @@ class createDistRepoTask(CreaterepoTask):
         self.rinfo = self.session.repoInfo(repo_id, strict=True)
         if self.rinfo['state'] != koji.REPO_INIT:
             raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo)
-        self.repo_id = self.rinfo['id']
-        self.pathinfo = koji.PathInfo(self.options.topdir)
         groupdata = os.path.join(
-            self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),
+            koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),
             'groups', 'comps.xml')
-        #set up our output dir
+
+        # set up our output dir
         self.repodir = '%s/repo' % self.workdir
+        self.repo_files = []
         koji.ensuredir(self.repodir)
-        self.outdir = self.repodir # workaround create_local_repo use
-        self.datadir = '%s/repodata' % self.repodir
-        self.sigmap = {}
+        self.subrepos = set()
+
+        # gather oldpkgs data if delta option in use
         oldpkgs = []
         if opts.get('delta'):
             # should be a list of repo ids to delta against
@@ -5246,63 +5234,136 @@ class createDistRepoTask(CreaterepoTask):
                 path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name'])
                 if not os.path.exists(path):
                     raise koji.GenericError('Base drpm repo missing: %s' % path)
+                # note: since we're using the top level dir, this will handle
+                #       split repos as well
                 oldpkgs.append(path)
+
+        # sort out our package list(s)
         self.uploadpath = self.getUploadDir()
-        self.pkglist = self.make_pkglist(tag, arch, keys, opts)
+        self.get_rpms(tag, arch, keys, opts)
         if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):
             self.do_multilib(arch, self.archmap[arch], opts['multilib'])
+        self.split_pkgs(opts)
         self.write_kojipkgs()
-        self.logger.debug('package list is %s' % self.pkglist)
-        self.session.uploadWrapper(self.pkglist, self.uploadpath,
-            os.path.basename(self.pkglist))
-        if os.path.getsize(self.pkglist) == 0:
-            self.pkglist = None
-        self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs)
-        if self.pkglist is None:
-            fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')
-            fo.write("This repo is empty because its tag has no content for this arch\n")
-            fo.close()
-        files = ['pkglist', 'kojipkgs']
-        for f in os.listdir(self.datadir):
-            files.append(f)
-            self.session.uploadWrapper('%s/%s' % (self.datadir, f),
-                self.uploadpath, f)
-        if opts['delta']:
-            ddir = os.path.join(self.repodir, 'drpms')
-            for f in os.listdir(ddir):
-                files.append(f)
-                self.session.uploadWrapper('%s/%s' % (ddir, f),
-                    self.uploadpath, f)
-        return [self.uploadpath, files, self.sigmap.items()]
+        self.write_pkglist()
+        self.link_pkgs()
+
+        # generate the repodata
+        self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,
+                    groupdata, oldpkgs=oldpkgs)
+        for subrepo in self.subrepos:
+            self.do_createrepo(
+                    '%s/%s' % (self.repodir, subrepo),
+                    '%s/%s/pkglist' % (self.repodir, subrepo),
+                    groupdata, oldpkgs=oldpkgs,
+                    logname='createrepo_%s' % subrepo)
+        if len(self.kojipkgs) == 0:
+            fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")
+            with open(fn, 'w') as fp:
+                fp.write("This repo is empty because its tag has no content "
+                         "for this arch\n")
+
+        # upload repo files
+        self.upload_repo()
+        self.upload_repo_manifest()
+
+    def upload_repo_file(self, relpath):
+        """Upload a file from the repo
+
+        relpath should be relative to self.repodir
+        """
+
+        localpath = '%s/%s' % (self.repodir, relpath)
+        reldir = os.path.dirname(relpath)
+        if reldir:
+            uploadpath = "%s/%s" % (self.uploadpath, reldir)
+            fn = os.path.basename(relpath)
+        else:
+            uploadpath = self.uploadpath
+            fn = relpath
+        self.session.uploadWrapper(localpath, uploadpath, fn)
+        self.repo_files.append(relpath)
+
+    def upload_repo(self):
+        """Traverse the repo and upload needed files
+
+        We omit the symlinks we made for the rpms
+        """
+        for dirpath, dirs, files in os.walk(self.repodir):
+            reldir = os.path.relpath(dirpath, self.repodir)
+            for filename in files:
+                path = "%s/%s" % (dirpath, filename)
+                if os.path.islink(path):
+                    continue
+                relpath = "%s/%s" % (reldir, filename)
+                self.upload_repo_file(relpath)
+
+    def upload_repo_manifest(self):
+        """Upload a list of the repo files we've uploaded"""
+        fn = '%s/repo_manifest' % self.workdir
+        with open(fn, 'w') as fp:
+            json.dump(self.repo_files, fp, indent=4)
+        self.session.uploadWrapper(fn, self.uploadpath)
+
+    def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):
+        """Run createrepo
+
+        This is derived from CreaterepoTask.create_local_repo, but adapted to
+        our requirements here
+        """
+        koji.ensuredir(repodir)
+        if self.options.use_createrepo_c:
+            cmd = ['/usr/bin/createrepo_c']
+        else:
+            cmd = ['/usr/bin/createrepo']
+        cmd.extend(['-vd', '-i', pkglist])
+        if groupdata and os.path.isfile(groupdata):
+            cmd.extend(['-g', groupdata])
+        # TODO: can we recycle data (with --update) as in create_local_repo?
+        if oldpkgs:
+            # generate delta-rpms
+            cmd.append('--deltas')
+            for op_dir in oldpkgs:
+                cmd.extend(['--oldpackagedirs', op_dir])
+        cmd.append(repodir)
+
+        if logname is None:
+            logname = 'createrepo'
+        logfile = '%s/%s.log' % (self.workdir, logname)
+        status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
+        if not isSuccess(status):
+            raise koji.GenericError('failed to create repo: %s' \
+                    % parseStatus(status, ' '.join(cmd)))
 
     def do_multilib(self, arch, ml_arch, conf):
-        self.repo_id = self.rinfo['id']
-        pathinfo = koji.PathInfo(self.options.topdir)
-        repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
+        repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
         mldir = os.path.join(repodir, koji.canonArch(ml_arch))
-        ml_true = set() # multilib packages we need to include before depsolve
-        ml_conf = os.path.join(self.pathinfo.work(), conf)
+        ml_true = set()  # multilib packages we need to include before depsolve
+        ml_conf = os.path.join(koji.pathinfo.work(), conf)
+
+        # read pkgs data from multilib repo
+        ml_pkgfile = os.path.join(mldir, 'kojipkgs')
+        ml_pkgs = json.load(open(ml_pkgfile, 'r'))
 
         # step 1: figure out which packages are multilib (should already exist)
         mlm = multilib.DevelMultilibMethod(ml_conf)
         fs_missing = set()
-        with open(self.pkglist) as pkglist:
-            for pkg in pkglist:
-                ppath = os.path.join(self.repodir, pkg.strip())
+        for bnp in self.kojipkgs:
+                rpminfo = self.kojipkgs[bnp]
+                ppath = rpminfo['_pkgpath']
                 po = yum.packages.YumLocalPackage(filename=ppath)
-                if mlm.select(po) and arch in self.archmap:
+                if mlm.select(po):
                     # we need a multilib package to be included
-                    # we assume the same signature level is available
-                    # XXX: what is a subarchitecture is the right answer?
-                    pl_path = pkg.replace(arch, self.archmap[arch]).strip()
-                    # assume this exists in the task results for the ml arch
-                    real_path = os.path.join(mldir, pl_path)
-                    if not os.path.exists(real_path):
-                        self.logger.error('%s (multilib) is not on the filesystem' % real_path)
-                        fs_missing.add(real_path)
+                    ml_bnp = bnp.replace(arch, self.archmap[arch])
+                    ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)
+                    # ^ XXX - should actually generate this
+                    if ml_bnp not in ml_pkgs:
+                        # not in our multilib repo
+                        self.logger.error('%s (multilib) is not on the filesystem' % ml_path)
+                        fs_missing.add(ml_path)
                         # we defer failure so can report all the missing deps
                         continue
-                    ml_true.add(real_path)
+                    ml_true.add(ml_path)
 
         # step 2: set up architectures for yum configuration
         self.logger.info("Resolving multilib for %s using method devel" % arch)
@@ -5392,29 +5453,22 @@ enabled=1
             raise koji.GenericError('multilib packages missing. '
                     'See missing_multilib.log')
 
-        # get rpm ids for ml pkgs
-        kpkgfile = os.path.join(mldir, 'kojipkgs')
-        kojipkgs = json.load(open(kpkgfile, 'r'))
-
-        # step 5: add dependencies to our package list
-        pkgwriter = open(self.pkglist, 'a')
+        # step 5: update kojipkgs
         for dep_path in ml_needed:
             tspkg = ml_needed[dep_path]
             bnp = os.path.basename(dep_path)
-            bnplet = bnp[0].lower()
-            koji.ensuredir(os.path.join(self.repodir, bnplet))
-            dst = os.path.join(self.repodir, bnplet, bnp)
-            if os.path.exists(dst):
+            if bnp in self.kojipkgs:
                 # we expect duplication with noarch, but not other arches
                 if tspkg.arch != 'noarch':
-                    self.logger.warning("Path exists: %r", dst)
+                    self.logger.warning("Multilib duplicate: %s", bnp)
                 continue
-            pkgwriter.write(bnplet + '/' + bnp + '\n')
-            self.logger.debug("os.symlink(%r, %r)", dep_path, dst)
-            os.symlink(dep_path, dst)
-            rpminfo = kojipkgs[bnp]
-            self.sigmap[rpminfo['id']] = rpminfo['sigkey']
-
+            rpminfo = ml_pkgs[bnp].copy()
+            # fix _pkgpath, which comes from another task and could be wrong
+            # for us
+            # TODO: would be better if we could use the proper path here
+            rpminfo['_pkgpath'] = dep_path
+            rpminfo['_multilib'] = True
+            self.kojipkgs[bnp] = rpminfo
 
     def pick_key(self, keys, avail_keys):
         best = None
@@ -5430,21 +5484,20 @@ enabled=1
                 best_idx = idx
         return best
 
-
-    def make_pkglist(self, tag_id, arch, keys, opts):
+    def get_rpms(self, tag_id, arch, keys, opts):
         # get the rpm data
         rpms = []
         builddirs = {}
-        for a in self.compat[arch] + ('noarch',):
+        for a in self.compat[arch]:
+            # note: self.compat includes noarch for non-src already
             rpm_iter, builds = self.session.listTaggedRPMS(tag_id,
                 event=opts['event'], arch=a, latest=opts['latest'],
                 inherit=opts['inherit'], rpmsigs=True)
             for build in builds:
-                builddirs[build['id']] = self.pathinfo.build(build)
+                builddirs[build['id']] = koji.pathinfo.build(build)
             rpms += list(rpm_iter)
 
         # index by id and key
-        preferred = {}
         rpm_idx = {}
         for rpminfo in rpms:
             sigidx = rpm_idx.setdefault(rpminfo['id'], {})
@@ -5464,9 +5517,7 @@ enabled=1
             else:
                 selected[rpm_id] = rpm_idx[rpm_id][best_key]
 
-        #generate pkglist files
-        pkgfile = os.path.join(self.repodir, 'pkglist')
-        pkglist = file(pkgfile, 'w')
+        # generate kojipkgs data and note missing files
         fs_missing = []
         sig_missing = []
         kojipkgs = {}
@@ -5478,25 +5529,18 @@ enabled=1
                     continue
                 # use the primary copy, if allowed (checked below)
                 pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
-                    self.pathinfo.rpm(rpminfo))
+                    koji.pathinfo.rpm(rpminfo))
             else:
                 # use the signed copy
                 pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
-                    self.pathinfo.signed(rpminfo, rpminfo['sigkey']))
+                    koji.pathinfo.signed(rpminfo, rpminfo['sigkey']))
             if not os.path.exists(pkgpath):
                 fs_missing.append(pkgpath)
                 # we'll raise an error below
             else:
                 bnp = os.path.basename(pkgpath)
-                bnplet = bnp[0].lower()
-                pkglist.write(bnplet + '/' + bnp + '\n')
-                koji.ensuredir(os.path.join(self.repodir, bnplet))
-                self.sigmap[rpminfo['id']] = rpminfo['sigkey']
-                dst = os.path.join(self.repodir, bnplet, bnp)
-                self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
-                os.symlink(pkgpath, dst)
+                rpminfo['_pkgpath'] = pkgpath
                 kojipkgs[bnp] = rpminfo
-        pkglist.close()
         self.kojipkgs = kojipkgs
 
         # report problems
@@ -5534,19 +5578,54 @@ enabled=1
                         and not opts['allow_missing_signatures']):
                 raise koji.GenericError('Unsigned packages found. See '
                         'missing_signatures.log')
-        return pkgfile
 
+    def link_pkgs(self):
+        for bnp in self.kojipkgs:
+            bnplet = bnp[0].lower()
+            ddir = os.path.join(self.repodir, 'Packages', bnplet)
+            koji.ensuredir(ddir)
+            dst = os.path.join(ddir, bnp)
+            pkgpath = self.kojipkgs[bnp]['_pkgpath']
+            self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
+            os.symlink(pkgpath, dst)
+
+    def split_pkgs(self, opts):
+        '''Direct rpms to subrepos if needed'''
+        for rpminfo in self.kojipkgs.values():
+            if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']):
+                rpminfo['_subrepo'] = 'debug'
+                self.subrepos.add('debug')
+
+    def write_pkglist(self):
+        pkgs = []
+        subrepo_pkgs = {}
+        for bnp in self.kojipkgs:
+            rpminfo = self.kojipkgs[bnp]
+            bnplet = bnp[0].lower()
+            subrepo = rpminfo.get('_subrepo')
+            if subrepo:
+                # note the ../
+                subrepo_pkgs.setdefault(subrepo, []).append(
+                        '../Packages/%s/%s\n' % (bnplet, bnp))
+            else:
+                pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))
+
+        with open('%s/pkglist' % self.repodir, 'w') as fo:
+            for line in pkgs:
+                fo.write(line)
+        for subrepo in subrepo_pkgs:
+            koji.ensuredir('%s/%s' % (self.repodir, subrepo))
+            with open('%s/%s/pkglist' % (self.repodir, subrepo), 'w') as fo:
+                for line in subrepo_pkgs[subrepo]:
+                    fo.write(line)
 
     def write_kojipkgs(self):
         filename = os.path.join(self.repodir, 'kojipkgs')
         datafile = file(filename, 'w')
         try:
-            json.dump(self.kojipkgs, datafile, indent=4)
+            json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True)
         finally:
             datafile.close()
-        # and upload too
-        self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')
-
 
 
 class WaitrepoTask(BaseTaskHandler):
diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py
index ef0ceac..a9c14ac 100644
--- a/cli/koji_cli/commands.py
+++ b/cli/koji_cli/commands.py
@@ -6911,10 +6911,13 @@ def handle_dist_repo(options, session, args):
         default=False,
         help=_('For RPMs not signed with a desired key, fall back to the '
             'primary copy'))
-    parser.add_option("--arch", action='append', default=[],
+    parser.add_option("-a", "--arch", action='append', default=[],
         help=_("Indicate an architecture to consider. The default is all " +
             "architectures associated with the given tag. This option may " +
             "be specified multiple times."))
+    parser.add_option("--with-src", action='store_true', help='Also generate a src repo')
+    parser.add_option("--split-debuginfo", action='store_true', default=False,
+            help='Split debuginfo info a separate repo for each arch')
     parser.add_option('--comps', help='Include a comps file in the repodata')
     parser.add_option('--delta-rpms', metavar='REPO',default=[],
         action='append',
@@ -6922,7 +6925,7 @@ def handle_dist_repo(options, session, args):
             'or the name of a tag that has a dist repo. May be specified '
             'multiple times.'))
     parser.add_option('--event', type='int',
-        help=_('create a dist repository based on a Brew event'))
+        help=_('Use tag content at event'))
     parser.add_option('--non-latest', dest='latest', default=True,
         action='store_false', help='Include older builds, not just the latest')
     parser.add_option('--multilib', default=None, metavar="CONFIG",
@@ -6996,9 +6999,10 @@ def handle_dist_repo(options, session, args):
         task_opts.multilib = os.path.join(stuffdir,
             os.path.basename(task_opts.multilib))
         print('')
-    for f in ('noarch', 'src'):
-        if f in task_opts.arch:
-            task_opts.arch.remove(f)
+    if 'noarch' in task_opts.arch:
+        task_opts.arch.remove('noarch')
+    if task_opts.with_src and 'src' not in task_opts.arch:
+        task_opts.arch.append('src')
     opts = {
         'arch': task_opts.arch,
         'comps': task_opts.comps,
@@ -7007,6 +7011,7 @@ def handle_dist_repo(options, session, args):
         'inherit': not task_opts.noinherit,
         'latest': task_opts.latest,
         'multilib': task_opts.multilib,
+        'split_debuginfo': task_opts.split_debuginfo,
         'skip_missing_signatures': task_opts.skip_missing_signatures,
         'allow_missing_signatures': task_opts.allow_missing_signatures
     }
diff --git a/hub/kojihub.py b/hub/kojihub.py
index 3f281d1..af87e77 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -2470,7 +2470,7 @@ def dist_repo_init(tag, keys, task_opts):
     tinfo = get_tag(tag, strict=True)
     tag_id = tinfo['id']
     event = task_opts.get('event')
-    arches = set([koji.canonArch(a) for a in task_opts['arch']])
+    arches = list(set([koji.canonArch(a) for a in task_opts['arch']]))
     # note: we need to match args from the other preRepoInit callback
     koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,
             with_debuginfo=False, event=event, repo_id=None,
@@ -2493,7 +2493,9 @@ def dist_repo_init(tag, keys, task_opts):
             task_opts['comps']), groupsdir + '/comps.xml')
     # note: we need to match args from the other postRepoInit callback
     koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,
-            with_debuginfo=False, event=event, repo_id=repo_id)
+            with_debuginfo=False, event=event, repo_id=repo_id,
+            dist=True, keys=keys, arches=arches, task_opts=task_opts,
+            repodir=repodir)
     return repo_id, event
 
 
@@ -2550,15 +2552,23 @@ def repo_delete(repo_id):
         repo_set_state(repo_id, koji.REPO_DELETED)
     return len(references)
 
-def repo_expire_older(tag_id, event_id):
-    """Expire repos for tag older than event"""
+
+def repo_expire_older(tag_id, event_id, dist=None):
+    """Expire repos for tag older than event
+
+    If dist is not None, then only expire repos with the given dist value
+    """
     st_ready = koji.REPO_READY
-    st_expired = koji.REPO_EXPIRED
-    q = """UPDATE repo SET state=%(st_expired)i
-    WHERE tag_id = %(tag_id)i
-        AND create_event < %(event_id)i
-        AND state = %(st_ready)i"""
-    _dml(q, locals())
+    clauses=['tag_id = %(tag_id)s',
+             'create_event < %(event_id)s',
+             'state = %(st_ready)s']
+    if dist is not None:
+        dist = bool(dist)
+        clauses.append('dist = %(dist)s')
+    update = UpdateProcessor('repo', values=locals(), clauses=clauses)
+    update.set(state=koji.REPO_EXPIRED)
+    update.execute()
+
 
 def repo_references(repo_id):
     """Return a list of buildroots that reference the repo"""
@@ -12564,14 +12574,21 @@ class HostExports(object):
                 safer_move(filepath, dst)
 
     def repoDone(self, repo_id, data, expire=False):
-        """Move repo data into place, mark as ready, and expire earlier repos
+        """Finalize a repo
 
         repo_id: the id of the repo
-        data: a dictionary of the form { arch: (uploadpath, files), ...}
-        expire(optional): if set to true, mark the repo expired immediately*
+        data: a dictionary of repo files in the form:
+              { arch: [uploadpath, [file1, file2, ...]], ...}
+        expire: if set to true, mark the repo expired immediately [*]
+
+        Actions:
+        * Move uploaded repo files into place
+        * Mark repo ready
+        * Expire earlier repos
+        * Move/create 'latest' symlink
 
-        If this is a dist repo, also hardlink the rpms in the final
-        directory.
+        For dist repos, the move step is skipped (that is handled in
+        distRepoMove).
 
         * This is used when a repo from an older event is generated
         """
@@ -12605,7 +12622,7 @@ class HostExports(object):
             return
         #else:
         repo_ready(repo_id)
-        repo_expire_older(rinfo['tag_id'], rinfo['create_event'])
+        repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist'])
 
         #make a latest link
         if rinfo['dist']:
@@ -12623,25 +12640,22 @@ class HostExports(object):
         koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)
 
 
-    def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):
+    def distRepoMove(self, repo_id, uploadpath, arch):
         """
-        Move a dist repo into its final location
+        Move one arch of a dist repo into its final location
 
-
-        Unlike normal repos (which are moved into place by repoDone), dist
-        repos have all their content linked (or copied) into place.
+        Unlike normal repos, dist repos have all their content linked (or
+        copied) into place.
 
         repo_id - the repo to move
         uploadpath - where the uploaded files are
-        files - a list of the uploaded file names
         arch - the arch of the repo
-        sigmap - a list of [rpm_id, sig] pairs
 
-        The rpms from sigmap should match the contents of the uploaded pkglist
-        file.
+        uploadpath should contain a repo_manifest file
 
-        In sigmap, use sig=None to use the primary copy of the rpm instead of a
-        signed copy.
+        The uploaded files should include:
+            - kojipkgs: json file with information about the component rpms
+            - repo metadata files
         """
         host = Host()
         host.verify()
@@ -12651,33 +12665,48 @@ class HostExports(object):
         archdir = "%s/%s" % (repodir, koji.canonArch(arch))
         if not os.path.isdir(archdir):
             raise koji.GenericError("Repo arch directory missing: %s" % archdir)
-        datadir = "%s/repodata" % archdir
-        koji.ensuredir(datadir)
+        repo_state = koji.REPO_STATES[rinfo['state']]
+        if repo_state != 'INIT':
+            raise koji.GenericError('Repo is in state: %s' % repo_state)
 
-        pkglist = set()
-        for fn in files:
-            src = "%s/%s/%s" % (workdir, uploadpath, fn)
-            if fn.endswith('.drpm'):
-                koji.ensuredir(os.path.join(archdir, 'drpms'))
-                dst = "%s/drpms/%s" % (archdir, fn)
-            elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):
-                dst = '%s/%s' % (archdir, fn)
-            else:
-                dst = "%s/%s" % (datadir, fn)
+        # read manifest
+        fn = '%s/%s/repo_manifest' % (workdir, uploadpath)
+        if not os.path.isfile(fn):
+            raise koji.GenericError('Missing repo manifest')
+        with open(fn) as fp:
+            files = json.load(fp)
+
+        # Read package data
+        fn = '%s/%s/kojipkgs' % (workdir, uploadpath)
+        if not os.path.isfile(fn):
+            raise koji.GenericError('Missing kojipkgs file')
+        with open(fn) as fp:
+            kojipkgs = json.load(fp)
+
+        # Figure out where to send the uploaded files
+        file_moves = []
+        for relpath in files:
+            src = "%s/%s/%s" % (workdir, uploadpath, relpath)
+            dst = "%s/%s" % (archdir, relpath)
             if not os.path.exists(src):
                 raise koji.GenericError("uploaded file missing: %s" % src)
-            if fn.endswith('pkglist'):
-                with open(src) as pkgfile:
-                    for pkg in pkgfile:
-                        pkg = os.path.basename(pkg.strip())
-                        pkglist.add(pkg)
-            safer_move(src, dst)
+            file_moves.append([src, dst])
 
         # get rpms
         build_dirs = {}
         rpmdata = {}
-        for rpm_id, sigkey in sigmap:
-            rpminfo = get_rpm(rpm_id, strict=True)
+        rpm_check_keys = ['name', 'version', 'release', 'arch', 'epoch',
+                'size', 'payloadhash', 'build_id']
+        for bnp in kojipkgs:
+            rpminfo = kojipkgs[bnp]
+            rpm_id = rpminfo['id']
+            sigkey = rpminfo['sigkey']
+            _rpminfo = get_rpm(rpm_id, strict=True)
+            for key in rpm_check_keys:
+                if key not in rpminfo or rpminfo[key] != _rpminfo[key]:
+                    raise koji.GenericError(
+                            'kojipkgs entry does not match db: file %s, key %s'
+                            % (bnp, key))
             if sigkey is None or sigkey == '':
                 relpath = koji.pathinfo.rpm(rpminfo)
             else:
@@ -12690,26 +12719,25 @@ class HostExports(object):
                 builddir = koji.pathinfo.build(binfo)
                 build_dirs[rpminfo['build_id']] = builddir
             rpminfo['_fullpath'] = os.path.join(builddir, relpath)
-            basename = os.path.basename(relpath)
-            rpmdata[basename] = rpminfo
+            rpmdata[bnp] = rpminfo
 
-        # sanity check
-        for fn in rpmdata:
-            if fn not in pkglist:
-                raise koji.GenericError("No signature data for: %s" % fn)
-        for fn in pkglist:
-            if fn  not in rpmdata:
-                raise koji.GenericError("RPM missing from pkglist: %s" % fn)
+        # move the uploaded files
+        dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
+        for dirname in dirnames:
+            koji.ensuredir(dirname)
+        for src, dst in file_moves:
+            safer_move(src, dst)
 
+        # hardlink or copy the rpms into the final repodir
+        # TODO: properly consider split-volume functionality
         for fn in rpmdata:
-            # hardlink or copy the rpms into the final repodir
-            # TODO: properly consider split-volume functionality
             rpminfo = rpmdata[fn]
             rpmpath = rpminfo['_fullpath']
             bnp = fn
             bnplet = bnp[0].lower()
-            koji.ensuredir(os.path.join(archdir, bnplet))
-            l_dst = os.path.join(archdir, bnplet, bnp)
+            ddir = os.path.join(archdir, 'Packages', bnplet)
+            koji.ensuredir(ddir)
+            l_dst = os.path.join(ddir, bnp)
             if os.path.exists(l_dst):
                 raise koji.GenericError("File already in repo: %s", l_dst)
             logger.debug("os.link(%r, %r)", rpmpath, l_dst)
@@ -12717,8 +12745,7 @@ class HostExports(object):
                 os.link(rpmpath, l_dst)
             except OSError as ose:
                 if ose.errno == 18:
-                    shutil.copy2(
-                        rpmpath, os.path.join(archdir, bnplet, bnp))
+                    shutil.copy2(rpmpath, l_dst)
                 else:
                     raise
 
diff --git a/tests/test_cli/test_dist_repo.py b/tests/test_cli/test_dist_repo.py
index 9b176e5..527cd81 100644
--- a/tests/test_cli/test_dist_repo.py
+++ b/tests/test_cli/test_dist_repo.py
@@ -252,14 +252,16 @@ Options:
   --allow-missing-signatures
                         For RPMs not signed with a desired key, fall back to
                         the primary copy
-  --arch=ARCH           Indicate an architecture to consider. The default is
+  -a ARCH, --arch=ARCH  Indicate an architecture to consider. The default is
                         all architectures associated with the given tag. This
                         option may be specified multiple times.
+  --with-src            Also generate a src repo
+  --split-debuginfo     Split debuginfo info a separate repo for each arch
   --comps=COMPS         Include a comps file in the repodata
   --delta-rpms=REPO     Create delta rpms. REPO can be the id of another dist
                         repo or the name of a tag that has a dist repo. May be
                         specified multiple times.
-  --event=EVENT         create a dist repository based on a Brew event
+  --event=EVENT         Use tag content at event
   --non-latest          Include older builds, not just the latest
   --multilib=CONFIG     Include multilib packages in the repository using the
                         given config file
diff --git a/tests/test_hub/test_dist_repo.py b/tests/test_hub/test_dist_repo.py
index 33f1b6c..90f9d54 100644
--- a/tests/test_hub/test_dist_repo.py
+++ b/tests/test_hub/test_dist_repo.py
@@ -1,5 +1,6 @@
 
 import unittest
+import json
 import mock
 import os
 import shutil
@@ -106,7 +107,7 @@ class TestDistRepoMove(unittest.TestCase):
             'create_ts': 1487256924.72718,
             'creation_time': '2017-02-16 14:55:24.727181',
             'id': 47,
-            'state': 1,
+            'state': 0,  # INIT
             'tag_id': 2,
             'tag_name': 'my-tag'}
         self.arch = 'x86_64'
@@ -123,19 +124,18 @@ class TestDistRepoMove(unittest.TestCase):
         os.makedirs(uploaddir)
 
         # place some test files
-        self.files = ['foo.drpm', 'repomd.xml']
+        self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
         self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
         for fn in self.files:
             path = os.path.join(uploaddir, fn)
             koji.ensuredir(os.path.dirname(path))
             with open(path, 'w') as fo:
-                fo.write('%s' % fn)
+                fo.write('%s' % os.path.basename(fn))
 
-        # generate pkglist file and sigmap
+        # generate pkglist file
         self.files.append('pkglist')
         plist = os.path.join(uploaddir, 'pkglist')
         nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']
-        self.sigmap = []
         self.rpms = {}
         self.builds ={}
         self.key = '4c8da725'
@@ -153,15 +153,30 @@ class TestDistRepoMove(unittest.TestCase):
                     fo.write('%s' % basename)
                 f_pkglist.write(path)
                 f_pkglist.write('\n')
-                self.expected.append('x86_64/%s/%s' % (basename[0], basename))
+                self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))
                 build_id = len(self.builds) + 10000
                 rpm_id = len(self.rpms) + 20000
                 binfo['id'] = build_id
                 rpminfo['build_id'] = build_id
                 rpminfo['id'] = rpm_id
+                rpminfo['sigkey'] = self.key
+                rpminfo['size'] = 1024
+                rpminfo['payloadhash'] = 'helloworld'
                 self.builds[build_id] = binfo
                 self.rpms[rpm_id] = rpminfo
-                self.sigmap.append([rpm_id, self.key])
+
+        # write kojipkgs
+        kojipkgs = {}
+        for rpminfo in self.rpms.values():
+            bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
+            kojipkgs[bnp] = rpminfo
+        with open("%s/kojipkgs" % uploaddir, "w") as fp:
+            json.dump(kojipkgs, fp, indent=4)
+        self.files.append('kojipkgs')
+
+        # write manifest
+        with open("%s/repo_manifest" % uploaddir, "w") as fp:
+            json.dump(self.files, fp, indent=4)
 
         # mocks
         self.repo_info = mock.patch('kojihub.repo_info').start()
@@ -187,8 +202,7 @@ class TestDistRepoMove(unittest.TestCase):
 
     def test_distRepoMove(self):
         exports = kojihub.HostExports()
-        exports.distRepoMove(self.rinfo['id'], self.uploadpath,
-                list(self.files), self.arch, self.sigmap)
+        exports.distRepoMove(self.rinfo['id'], self.uploadpath, self.arch)
         # check result
         repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo
         for relpath in self.expected:
diff --git a/tests/test_hub/test_repos.py b/tests/test_hub/test_repos.py
new file mode 100644
index 0000000..8bda536
--- /dev/null
+++ b/tests/test_hub/test_repos.py
@@ -0,0 +1,75 @@
+import mock
+import unittest
+
+import koji
+import kojihub
+
+
+QP = kojihub.QueryProcessor
+IP = kojihub.InsertProcessor
+UP = kojihub.UpdateProcessor
+
+
+class TestRepoFunctions(unittest.TestCase):
+
+    def setUp(self):
+        self.QueryProcessor = mock.patch('kojihub.QueryProcessor',
+                side_effect=self.getQuery).start()
+        self.queries = []
+        self.InsertProcessor = mock.patch('kojihub.InsertProcessor',
+                side_effect=self.getInsert).start()
+        self.inserts = []
+        self.UpdateProcessor = mock.patch('kojihub.UpdateProcessor',
+                side_effect=self.getUpdate).start()
+        self.updates = []
+        self._dml = mock.patch('kojihub._dml').start()
+
+    def tearDown(self):
+        mock.patch.stopall()
+
+    def getQuery(self, *args, **kwargs):
+        query = QP(*args, **kwargs)
+        query.execute = mock.MagicMock()
+        self.queries.append(query)
+        return query
+
+    def getInsert(self, *args, **kwargs):
+        insert = IP(*args, **kwargs)
+        insert.execute = mock.MagicMock()
+        self.inserts.append(insert)
+        return insert
+
+    def getUpdate(self, *args, **kwargs):
+        update = UP(*args, **kwargs)
+        update.execute = mock.MagicMock()
+        self.updates.append(update)
+        return update
+
+    def test_repo_expire_older(self):
+        kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id)
+        self.assertEqual(len(self.updates), 1)
+        update = self.updates[0]
+        self.assertEqual(update.table, 'repo')
+        self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})
+        self.assertEqual(update.rawdata, {})
+        self.assertEqual(update.values['event_id'], mock.sentinel.event_id)
+        self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)
+        self.assertEqual(update.values['dist'], None)
+        if 'dist = %(dist)s' in update.clauses:
+            raise Exception('Unexpected dist condition')
+
+        # and with dist specified
+        for dist in True, False:
+            self.updates = []
+            kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id,
+                                      dist=dist)
+            self.assertEqual(len(self.updates), 1)
+            update = self.updates[0]
+            self.assertEqual(update.table, 'repo')
+            self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})
+            self.assertEqual(update.rawdata, {})
+            self.assertEqual(update.values['event_id'], mock.sentinel.event_id)
+            self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)
+            self.assertEqual(update.values['dist'], dist)
+            if 'dist = %(dist)s' not in update.clauses:
+                raise Exception('Missing dist condition')
diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml
index a4e050f..5ee634a 100644
--- a/www/kojiweb/taskinfo.chtml
+++ b/www/kojiweb/taskinfo.chtml
@@ -225,7 +225,7 @@ $value
         #end if
         #elif $task.method == 'distRepo'
         <strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>
-        <strong>Repo ID:</strong> $params[1]<br/>
+        <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>
         <strong>Keys:</strong> $printValue(0, $params[2])<br/>
         $printOpts($params[3])
         #elif $task.method == 'prepRepo'
@@ -243,7 +243,7 @@ $value
         #end if
         #elif $task.method == 'createdistrepo'
         <strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>
-        <strong>Repo ID:</strong> $params[1]<br/>
+        <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>
         <strong>Arch:</strong> $printValue(0, $params[2])<br/>
         <strong>Keys:</strong> $printValue(0, $params[3])<br/>
         <strong>Options:</strong> $printMap($params[4], '&nbsp;&nbsp;&nbsp;&nbsp;')