#914 dist repo updates
Merged a year ago by mikem. Opened a year ago by mikem.
mikem/koji dist-repo-updates  into  master

file modified
+190 -111

@@ -5055,7 +5055,7 @@ 

              self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)

          return [uploadpath, files]

  

-     def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None):

+     def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):

          koji.ensuredir(self.outdir)

          if self.options.use_createrepo_c:

              cmd = ['/usr/bin/createrepo_c']

@@ -5082,11 +5082,6 @@ 

                  cmd.append('--update')

                  if self.options.createrepo_skip_stat:

                      cmd.append('--skip-stat')

-         if oldpkgs:

-             # generate delta-rpms

-             cmd.append('--deltas')

-             for op_dir in oldpkgs:

-                 cmd.extend(['--oldpackagedirs', op_dir])

          # note: we can't easily use a cachedir because we do not have write

          # permission. The good news is that with --update we won't need to

          # be scanning many rpms.

@@ -5139,10 +5134,9 @@ 

  

      def handler(self, tag, repo_id, keys, task_opts):

          tinfo = self.session.getTag(tag, strict=True, event=task_opts['event'])

-         path = koji.pathinfo.distrepo(repo_id, tinfo['name'])

          if len(task_opts['arch']) == 0:

-              arches = tinfo['arches'] or ''

-              task_opts['arch'] = arches.split()

+             arches = tinfo['arches'] or ''

+             task_opts['arch'] = arches.split()

          if len(task_opts['arch']) == 0:

              raise koji.GenericError('No arches specified nor for the tag!')

          subtasks = {}

@@ -5162,13 +5156,12 @@ 

                  method='createdistrepo', arglist=arglist, label=arch,

                  parent=self.id, arch='noarch')

          if len(subtasks) > 0 and task_opts['multilib']:

-             results = self.wait(subtasks.values(), all=True, failany=True)

+             self.wait(subtasks.values(), all=True, failany=True)

              for arch in arch32s:

                  # move the 32-bit task output to the final resting place

                  # so the 64-bit arches can use it for multilib

-                 upload, files, sigmap = results[subtasks[arch]]

-                 self.session.host.distRepoMove(

-                     repo_id, upload, files, arch, sigmap)

+                 upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])

+                 self.session.host.distRepoMove(repo_id, upload_dir, arch)

          for arch in canonArches:

              # do the other arches

              if arch not in arch32s:

@@ -5177,23 +5170,18 @@ 

                      method='createdistrepo', arglist=arglist, label=arch,

                      parent=self.id, arch='noarch')

          # wait for 64-bit subtasks to finish

-         data = {}

-         results = self.wait(subtasks.values(), all=True, failany=True)

+         self.wait(subtasks.values(), all=True, failany=True)

          for (arch, task_id) in subtasks.iteritems():

-             data[arch] = results[task_id]

-             self.logger.debug("DEBUG: %r : %r " % (arch, data[arch]))

              if task_opts['multilib'] and arch in arch32s:

                  # already moved above

                  continue

-             #else

-             upload, files, sigmap = results[subtasks[arch]]

-             self.session.host.distRepoMove(

-                 repo_id, upload, files, arch, sigmap)

-         self.session.host.repoDone(repo_id, data, expire=False)

+             upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])

+             self.session.host.distRepoMove(repo_id, upload_dir, arch)

+         self.session.host.repoDone(repo_id, {}, expire=False)

          return 'Dist repository #%s successfully generated' % repo_id

  

  

- class createDistRepoTask(CreaterepoTask):

+ class createDistRepoTask(BaseTaskHandler):

      Methods = ['createdistrepo']

      _taskWeight = 1.5

  

@@ -5223,17 +5211,17 @@ 

          self.rinfo = self.session.repoInfo(repo_id, strict=True)

          if self.rinfo['state'] != koji.REPO_INIT:

              raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo)

-         self.repo_id = self.rinfo['id']

-         self.pathinfo = koji.PathInfo(self.options.topdir)

          groupdata = os.path.join(

-             self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),

+             koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),

              'groups', 'comps.xml')

-         #set up our output dir

+ 

+         # set up our output dir

          self.repodir = '%s/repo' % self.workdir

+         self.repo_files = []

          koji.ensuredir(self.repodir)

-         self.outdir = self.repodir # workaround create_local_repo use

-         self.datadir = '%s/repodata' % self.repodir

-         self.sigmap = {}

+         self.subrepos = set()

+ 

+         # gather oldpkgs data if delta option in use

          oldpkgs = []

          if opts.get('delta'):

              # should be a list of repo ids to delta against

@@ -5246,63 +5234,136 @@ 

                  path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name'])

                  if not os.path.exists(path):

                      raise koji.GenericError('Base drpm repo missing: %s' % path)

+                 # note: since we're using the top level dir, this will handle

+                 #       split repos as well

                  oldpkgs.append(path)

+ 

+         # sort out our package list(s)

          self.uploadpath = self.getUploadDir()

-         self.pkglist = self.make_pkglist(tag, arch, keys, opts)

+         self.get_rpms(tag, arch, keys, opts)

          if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):

              self.do_multilib(arch, self.archmap[arch], opts['multilib'])

+         self.split_pkgs(opts)

          self.write_kojipkgs()

-         self.logger.debug('package list is %s' % self.pkglist)

-         self.session.uploadWrapper(self.pkglist, self.uploadpath,

-             os.path.basename(self.pkglist))

-         if os.path.getsize(self.pkglist) == 0:

-             self.pkglist = None

-         self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs)

-         if self.pkglist is None:

-             fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')

-             fo.write("This repo is empty because its tag has no content for this arch\n")

-             fo.close()

-         files = ['pkglist', 'kojipkgs']

-         for f in os.listdir(self.datadir):

-             files.append(f)

-             self.session.uploadWrapper('%s/%s' % (self.datadir, f),

-                 self.uploadpath, f)

-         if opts['delta']:

-             ddir = os.path.join(self.repodir, 'drpms')

-             for f in os.listdir(ddir):

-                 files.append(f)

-                 self.session.uploadWrapper('%s/%s' % (ddir, f),

-                     self.uploadpath, f)

-         return [self.uploadpath, files, self.sigmap.items()]

+         self.write_pkglist()

+         self.link_pkgs()

+ 

+         # generate the repodata

+         self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,

+                     groupdata, oldpkgs=oldpkgs)

+         for subrepo in self.subrepos:

+             self.do_createrepo(

+                     '%s/%s' % (self.repodir, subrepo),

+                     '%s/%s/pkglist' % (self.repodir, subrepo),

+                     groupdata, oldpkgs=oldpkgs,

+                     logname='createrepo_%s' % subrepo)

+         if len(self.kojipkgs) == 0:

+             fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")

+             with open(fn, 'w') as fp:

+                 fp.write("This repo is empty because its tag has no content "

+                          "for this arch\n")

+ 

+         # upload repo files

+         self.upload_repo()

+         self.upload_repo_manifest()

+ 

+     def upload_repo_file(self, relpath):

+         """Upload a file from the repo

+ 

+         relpath should be relative to self.repodir

+         """

+ 

+         localpath = '%s/%s' % (self.repodir, relpath)

+         reldir = os.path.dirname(relpath)

+         if reldir:

+             uploadpath = "%s/%s" % (self.uploadpath, reldir)

+             fn = os.path.basename(relpath)

+         else:

+             uploadpath = self.uploadpath

+             fn = relpath

+         self.session.uploadWrapper(localpath, uploadpath, fn)

+         self.repo_files.append(relpath)

+ 

+     def upload_repo(self):

+         """Traverse the repo and upload needed files

+ 

+         We omit the symlinks we made for the rpms

+         """

+         for dirpath, dirs, files in os.walk(self.repodir):

+             reldir = os.path.relpath(dirpath, self.repodir)

+             for filename in files:

+                 path = "%s/%s" % (dirpath, filename)

+                 if os.path.islink(path):

+                     continue

+                 relpath = "%s/%s" % (reldir, filename)

+                 self.upload_repo_file(relpath)

+ 

+     def upload_repo_manifest(self):

+         """Upload a list of the repo files we've uploaded"""

+         fn = '%s/repo_manifest' % self.workdir

+         with open(fn, 'w') as fp:

+             json.dump(self.repo_files, fp, indent=4)

+         self.session.uploadWrapper(fn, self.uploadpath)

+ 

+     def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):

+         """Run createrepo

+ 

+         This is derived from CreaterepoTask.create_local_repo, but adapted to

+         our requirements here

+         """

+         koji.ensuredir(repodir)

+         if self.options.use_createrepo_c:

+             cmd = ['/usr/bin/createrepo_c']

+         else:

+             cmd = ['/usr/bin/createrepo']

+         cmd.extend(['-vd', '-i', pkglist])

+         if groupdata and os.path.isfile(groupdata):

+             cmd.extend(['-g', groupdata])

+         # TODO: can we recycle data (with --update) as in create_local_repo?

+         if oldpkgs:

+             # generate delta-rpms

+             cmd.append('--deltas')

+             for op_dir in oldpkgs:

+                 cmd.extend(['--oldpackagedirs', op_dir])

+         cmd.append(repodir)

+ 

+         if logname is None:

+             logname = 'createrepo'

+         logfile = '%s/%s.log' % (self.workdir, logname)

+         status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)

+         if not isSuccess(status):

+             raise koji.GenericError('failed to create repo: %s' \

+                     % parseStatus(status, ' '.join(cmd)))

  

      def do_multilib(self, arch, ml_arch, conf):

-         self.repo_id = self.rinfo['id']

-         pathinfo = koji.PathInfo(self.options.topdir)

-         repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])

+         repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])

          mldir = os.path.join(repodir, koji.canonArch(ml_arch))

-         ml_true = set() # multilib packages we need to include before depsolve

-         ml_conf = os.path.join(self.pathinfo.work(), conf)

+         ml_true = set()  # multilib packages we need to include before depsolve

+         ml_conf = os.path.join(koji.pathinfo.work(), conf)

+ 

+         # read pkgs data from multilib repo

+         ml_pkgfile = os.path.join(mldir, 'kojipkgs')

+         ml_pkgs = json.load(open(ml_pkgfile, 'r'))

  

          # step 1: figure out which packages are multilib (should already exist)

          mlm = multilib.DevelMultilibMethod(ml_conf)

          fs_missing = set()

-         with open(self.pkglist) as pkglist:

-             for pkg in pkglist:

-                 ppath = os.path.join(self.repodir, pkg.strip())

+         for bnp in self.kojipkgs:

+                 rpminfo = self.kojipkgs[bnp]

+                 ppath = rpminfo['_pkgpath']

                  po = yum.packages.YumLocalPackage(filename=ppath)

-                 if mlm.select(po) and arch in self.archmap:

+                 if mlm.select(po):

                      # we need a multilib package to be included

-                     # we assume the same signature level is available

-                     # XXX: what is a subarchitecture is the right answer?

-                     pl_path = pkg.replace(arch, self.archmap[arch]).strip()

-                     # assume this exists in the task results for the ml arch

-                     real_path = os.path.join(mldir, pl_path)

-                     if not os.path.exists(real_path):

-                         self.logger.error('%s (multilib) is not on the filesystem' % real_path)

-                         fs_missing.add(real_path)

+                     ml_bnp = bnp.replace(arch, self.archmap[arch])

+                     ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)

+                     # ^ XXX - should actually generate this

+                     if ml_bnp not in ml_pkgs:

+                         # not in our multilib repo

+                         self.logger.error('%s (multilib) is not on the filesystem' % ml_path)

+                         fs_missing.add(ml_path)

                          # we defer failure so can report all the missing deps

                          continue

-                     ml_true.add(real_path)

+                     ml_true.add(ml_path)

  

          # step 2: set up architectures for yum configuration

          self.logger.info("Resolving multilib for %s using method devel" % arch)

@@ -5392,29 +5453,22 @@ 

              raise koji.GenericError('multilib packages missing. '

                      'See missing_multilib.log')

  

-         # get rpm ids for ml pkgs

-         kpkgfile = os.path.join(mldir, 'kojipkgs')

-         kojipkgs = json.load(open(kpkgfile, 'r'))

- 

-         # step 5: add dependencies to our package list

-         pkgwriter = open(self.pkglist, 'a')

+         # step 5: update kojipkgs

          for dep_path in ml_needed:

              tspkg = ml_needed[dep_path]

              bnp = os.path.basename(dep_path)

-             bnplet = bnp[0].lower()

-             koji.ensuredir(os.path.join(self.repodir, bnplet))

-             dst = os.path.join(self.repodir, bnplet, bnp)

-             if os.path.exists(dst):

+             if bnp in self.kojipkgs:

                  # we expect duplication with noarch, but not other arches

                  if tspkg.arch != 'noarch':

-                     self.logger.warning("Path exists: %r", dst)

+                     self.logger.warning("Multilib duplicate: %s", bnp)

                  continue

-             pkgwriter.write(bnplet + '/' + bnp + '\n')

-             self.logger.debug("os.symlink(%r, %r)", dep_path, dst)

-             os.symlink(dep_path, dst)

-             rpminfo = kojipkgs[bnp]

-             self.sigmap[rpminfo['id']] = rpminfo['sigkey']

- 

+             rpminfo = ml_pkgs[bnp].copy()

+             # fix _pkgpath, which comes from another task and could be wrong

+             # for us

+             # TODO: would be better if we could use the proper path here

+             rpminfo['_pkgpath'] = dep_path

+             rpminfo['_multilib'] = True

+             self.kojipkgs[bnp] = rpminfo

  

      def pick_key(self, keys, avail_keys):

          best = None

@@ -5430,21 +5484,20 @@ 

                  best_idx = idx

          return best

  

- 

-     def make_pkglist(self, tag_id, arch, keys, opts):

+     def get_rpms(self, tag_id, arch, keys, opts):

          # get the rpm data

          rpms = []

          builddirs = {}

-         for a in self.compat[arch] + ('noarch',):

+         for a in self.compat[arch]:

+             # note: self.compat includes noarch for non-src already

              rpm_iter, builds = self.session.listTaggedRPMS(tag_id,

                  event=opts['event'], arch=a, latest=opts['latest'],

                  inherit=opts['inherit'], rpmsigs=True)

              for build in builds:

-                 builddirs[build['id']] = self.pathinfo.build(build)

+                 builddirs[build['id']] = koji.pathinfo.build(build)

              rpms += list(rpm_iter)

  

          # index by id and key

-         preferred = {}

          rpm_idx = {}

          for rpminfo in rpms:

              sigidx = rpm_idx.setdefault(rpminfo['id'], {})

@@ -5464,9 +5517,7 @@ 

              else:

                  selected[rpm_id] = rpm_idx[rpm_id][best_key]

  

-         #generate pkglist files

-         pkgfile = os.path.join(self.repodir, 'pkglist')

-         pkglist = file(pkgfile, 'w')

+         # generate kojipkgs data and note missing files

          fs_missing = []

          sig_missing = []

          kojipkgs = {}

@@ -5478,25 +5529,18 @@ 

                      continue

                  # use the primary copy, if allowed (checked below)

                  pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],

-                     self.pathinfo.rpm(rpminfo))

+                     koji.pathinfo.rpm(rpminfo))

              else:

                  # use the signed copy

                  pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],

-                     self.pathinfo.signed(rpminfo, rpminfo['sigkey']))

+                     koji.pathinfo.signed(rpminfo, rpminfo['sigkey']))

              if not os.path.exists(pkgpath):

                  fs_missing.append(pkgpath)

                  # we'll raise an error below

              else:

                  bnp = os.path.basename(pkgpath)

-                 bnplet = bnp[0].lower()

-                 pkglist.write(bnplet + '/' + bnp + '\n')

-                 koji.ensuredir(os.path.join(self.repodir, bnplet))

-                 self.sigmap[rpminfo['id']] = rpminfo['sigkey']

-                 dst = os.path.join(self.repodir, bnplet, bnp)

-                 self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)

-                 os.symlink(pkgpath, dst)

+                 rpminfo['_pkgpath'] = pkgpath

                  kojipkgs[bnp] = rpminfo

-         pkglist.close()

          self.kojipkgs = kojipkgs

  

          # report problems

@@ -5534,19 +5578,54 @@ 

                          and not opts['allow_missing_signatures']):

                  raise koji.GenericError('Unsigned packages found. See '

                          'missing_signatures.log')

-         return pkgfile

  

+     def link_pkgs(self):

+         for bnp in self.kojipkgs:

+             bnplet = bnp[0].lower()

+             ddir = os.path.join(self.repodir, 'Packages', bnplet)

+             koji.ensuredir(ddir)

+             dst = os.path.join(ddir, bnp)

+             pkgpath = self.kojipkgs[bnp]['_pkgpath']

+             self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)

+             os.symlink(pkgpath, dst)

+ 

+     def split_pkgs(self, opts):

+         '''Direct rpms to subrepos if needed'''

+         for rpminfo in self.kojipkgs.values():

+             if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']):

+                 rpminfo['_subrepo'] = 'debug'

+                 self.subrepos.add('debug')

+ 

+     def write_pkglist(self):

+         pkgs = []

+         subrepo_pkgs = {}

+         for bnp in self.kojipkgs:

+             rpminfo = self.kojipkgs[bnp]

+             bnplet = bnp[0].lower()

+             subrepo = rpminfo.get('_subrepo')

+             if subrepo:

+                 # note the ../

+                 subrepo_pkgs.setdefault(subrepo, []).append(

+                         '../Packages/%s/%s\n' % (bnplet, bnp))

+             else:

+                 pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))

+ 

+         with open('%s/pkglist' % self.repodir, 'w') as fo:

+             for line in pkgs:

+                 fo.write(line)

+         for subrepo in subrepo_pkgs:

+             koji.ensuredir('%s/%s' % (self.repodir, subrepo))

+             with open('%s/%s/pkglist' % (self.repodir, subrepo), 'w') as fo:

+                 for line in subrepo_pkgs[subrepo]:

+                     fo.write(line)

  

      def write_kojipkgs(self):

          filename = os.path.join(self.repodir, 'kojipkgs')

          datafile = file(filename, 'w')

          try:

-             json.dump(self.kojipkgs, datafile, indent=4)

+             json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True)

          finally:

              datafile.close()

-         # and upload too

-         self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')

- 

  

  

  class WaitrepoTask(BaseTaskHandler):

file modified
+10 -5

@@ -6911,10 +6911,13 @@ 

          default=False,

          help=_('For RPMs not signed with a desired key, fall back to the '

              'primary copy'))

-     parser.add_option("--arch", action='append', default=[],

+     parser.add_option("-a", "--arch", action='append', default=[],

          help=_("Indicate an architecture to consider. The default is all " +

              "architectures associated with the given tag. This option may " +

              "be specified multiple times."))

+     parser.add_option("--with-src", action='store_true', help='Also generate a src repo')

+     parser.add_option("--split-debuginfo", action='store_true', default=False,

+             help='Split debuginfo info a separate repo for each arch')

      parser.add_option('--comps', help='Include a comps file in the repodata')

      parser.add_option('--delta-rpms', metavar='REPO',default=[],

          action='append',

@@ -6922,7 +6925,7 @@ 

              'or the name of a tag that has a dist repo. May be specified '

              'multiple times.'))

      parser.add_option('--event', type='int',

-         help=_('create a dist repository based on a Brew event'))

+         help=_('Use tag content at event'))

      parser.add_option('--non-latest', dest='latest', default=True,

          action='store_false', help='Include older builds, not just the latest')

      parser.add_option('--multilib', default=None, metavar="CONFIG",

@@ -6996,9 +6999,10 @@ 

          task_opts.multilib = os.path.join(stuffdir,

              os.path.basename(task_opts.multilib))

          print('')

-     for f in ('noarch', 'src'):

-         if f in task_opts.arch:

-             task_opts.arch.remove(f)

+     if 'noarch' in task_opts.arch:

+         task_opts.arch.remove('noarch')

+     if task_opts.with_src and 'src' not in task_opts.arch:

+         task_opts.arch.append('src')

      opts = {

          'arch': task_opts.arch,

          'comps': task_opts.comps,

@@ -7007,6 +7011,7 @@ 

          'inherit': not task_opts.noinherit,

          'latest': task_opts.latest,

          'multilib': task_opts.multilib,

+         'split_debuginfo': task_opts.split_debuginfo,

          'skip_missing_signatures': task_opts.skip_missing_signatures,

          'allow_missing_signatures': task_opts.allow_missing_signatures

      }

file modified
+89 -62

@@ -2470,7 +2470,7 @@ 

      tinfo = get_tag(tag, strict=True)

      tag_id = tinfo['id']

      event = task_opts.get('event')

-     arches = set([koji.canonArch(a) for a in task_opts['arch']])

+     arches = list(set([koji.canonArch(a) for a in task_opts['arch']]))

      # note: we need to match args from the other preRepoInit callback

      koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,

              with_debuginfo=False, event=event, repo_id=None,

@@ -2493,7 +2493,9 @@ 

              task_opts['comps']), groupsdir + '/comps.xml')

      # note: we need to match args from the other postRepoInit callback

      koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,

-             with_debuginfo=False, event=event, repo_id=repo_id)

+             with_debuginfo=False, event=event, repo_id=repo_id,

+             dist=True, keys=keys, arches=arches, task_opts=task_opts,

+             repodir=repodir)

      return repo_id, event

  

  

@@ -2550,15 +2552,23 @@ 

          repo_set_state(repo_id, koji.REPO_DELETED)

      return len(references)

  

- def repo_expire_older(tag_id, event_id):

-     """Expire repos for tag older than event"""

+ 

+ def repo_expire_older(tag_id, event_id, dist=None):

+     """Expire repos for tag older than event

+ 

+     If dist is not None, then only expire repos with the given dist value

+     """

      st_ready = koji.REPO_READY

-     st_expired = koji.REPO_EXPIRED

-     q = """UPDATE repo SET state=%(st_expired)i

-     WHERE tag_id = %(tag_id)i

-         AND create_event < %(event_id)i

-         AND state = %(st_ready)i"""

-     _dml(q, locals())

+     clauses=['tag_id = %(tag_id)s',

+              'create_event < %(event_id)s',

+              'state = %(st_ready)s']

+     if dist is not None:

+         dist = bool(dist)

+         clauses.append('dist = %(dist)s')

+     update = UpdateProcessor('repo', values=locals(), clauses=clauses)

+     update.set(state=koji.REPO_EXPIRED)

+     update.execute()

+ 

  

  def repo_references(repo_id):

      """Return a list of buildroots that reference the repo"""

@@ -12564,14 +12574,21 @@ 

                  safer_move(filepath, dst)

  

      def repoDone(self, repo_id, data, expire=False):

-         """Move repo data into place, mark as ready, and expire earlier repos

+         """Finalize a repo

  

          repo_id: the id of the repo

-         data: a dictionary of the form { arch: (uploadpath, files), ...}

-         expire(optional): if set to true, mark the repo expired immediately*

+         data: a dictionary of repo files in the form:

+               { arch: [uploadpath, [file1, file2, ...]], ...}

+         expire: if set to true, mark the repo expired immediately [*]

+ 

+         Actions:

+         * Move uploaded repo files into place

+         * Mark repo ready

+         * Expire earlier repos

+         * Move/create 'latest' symlink

  

-         If this is a dist repo, also hardlink the rpms in the final

-         directory.

+         For dist repos, the move step is skipped (that is handled in

+         distRepoMove).

  

          * This is used when a repo from an older event is generated

          """

@@ -12605,7 +12622,7 @@ 

              return

          #else:

          repo_ready(repo_id)

-         repo_expire_older(rinfo['tag_id'], rinfo['create_event'])

+         repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist'])

  

          #make a latest link

          if rinfo['dist']:

@@ -12623,25 +12640,22 @@ 

          koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)

  

  

-     def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):

+     def distRepoMove(self, repo_id, uploadpath, arch):

          """

-         Move a dist repo into its final location

+         Move one arch of a dist repo into its final location

  

- 

-         Unlike normal repos (which are moved into place by repoDone), dist

-         repos have all their content linked (or copied) into place.

+         Unlike normal repos, dist repos have all their content linked (or

+         copied) into place.

  

          repo_id - the repo to move

          uploadpath - where the uploaded files are

-         files - a list of the uploaded file names

          arch - the arch of the repo

-         sigmap - a list of [rpm_id, sig] pairs

  

-         The rpms from sigmap should match the contents of the uploaded pkglist

-         file.

+         uploadpath should contain a repo_manifest file

  

-         In sigmap, use sig=None to use the primary copy of the rpm instead of a

-         signed copy.

+         The uploaded files should include:

+             - kojipkgs: json file with information about the component rpms

+             - repo metadata files

          """

          host = Host()

          host.verify()

@@ -12651,33 +12665,48 @@ 

          archdir = "%s/%s" % (repodir, koji.canonArch(arch))

          if not os.path.isdir(archdir):

              raise koji.GenericError("Repo arch directory missing: %s" % archdir)

-         datadir = "%s/repodata" % archdir

-         koji.ensuredir(datadir)

+         repo_state = koji.REPO_STATES[rinfo['state']]

+         if repo_state != 'INIT':

+             raise koji.GenericError('Repo is in state: %s' % repo_state)

  

-         pkglist = set()

-         for fn in files:

-             src = "%s/%s/%s" % (workdir, uploadpath, fn)

-             if fn.endswith('.drpm'):

-                 koji.ensuredir(os.path.join(archdir, 'drpms'))

-                 dst = "%s/drpms/%s" % (archdir, fn)

-             elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):

-                 dst = '%s/%s' % (archdir, fn)

-             else:

-                 dst = "%s/%s" % (datadir, fn)

+         # read manifest

+         fn = '%s/%s/repo_manifest' % (workdir, uploadpath)

+         if not os.path.isfile(fn):

+             raise koji.GenericError('Missing repo manifest')

+         with open(fn) as fp:

+             files = json.load(fp)

+ 

+         # Read package data

+         fn = '%s/%s/kojipkgs' % (workdir, uploadpath)

+         if not os.path.isfile(fn):

+             raise koji.GenericError('Missing kojipkgs file')

+         with open(fn) as fp:

+             kojipkgs = json.load(fp)

+ 

+         # Figure out where to send the uploaded files

+         file_moves = []

+         for relpath in files:

+             src = "%s/%s/%s" % (workdir, uploadpath, relpath)

+             dst = "%s/%s" % (archdir, relpath)

              if not os.path.exists(src):

                  raise koji.GenericError("uploaded file missing: %s" % src)

-             if fn.endswith('pkglist'):

-                 with open(src) as pkgfile:

-                     for pkg in pkgfile:

-                         pkg = os.path.basename(pkg.strip())

-                         pkglist.add(pkg)

-             safer_move(src, dst)

+             file_moves.append([src, dst])

  

          # get rpms

          build_dirs = {}

          rpmdata = {}

-         for rpm_id, sigkey in sigmap:

-             rpminfo = get_rpm(rpm_id, strict=True)

+         rpm_check_keys = ['name', 'version', 'release', 'arch', 'epoch',

+                 'size', 'payloadhash', 'build_id']

+         for bnp in kojipkgs:

+             rpminfo = kojipkgs[bnp]

+             rpm_id = rpminfo['id']

+             sigkey = rpminfo['sigkey']

+             _rpminfo = get_rpm(rpm_id, strict=True)

+             for key in rpm_check_keys:

+                 if key not in rpminfo or rpminfo[key] != _rpminfo[key]:

+                     raise koji.GenericError(

+                             'kojipkgs entry does not match db: file %s, key %s'

+                             % (bnp, key))

              if sigkey is None or sigkey == '':

                  relpath = koji.pathinfo.rpm(rpminfo)

              else:

@@ -12690,26 +12719,25 @@ 

                  builddir = koji.pathinfo.build(binfo)

                  build_dirs[rpminfo['build_id']] = builddir

              rpminfo['_fullpath'] = os.path.join(builddir, relpath)

-             basename = os.path.basename(relpath)

-             rpmdata[basename] = rpminfo

+             rpmdata[bnp] = rpminfo

  

-         # sanity check

-         for fn in rpmdata:

-             if fn not in pkglist:

-                 raise koji.GenericError("No signature data for: %s" % fn)

-         for fn in pkglist:

-             if fn  not in rpmdata:

-                 raise koji.GenericError("RPM missing from pkglist: %s" % fn)

+         # move the uploaded files

+         dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])

+         for dirname in dirnames:

+             koji.ensuredir(dirname)

+         for src, dst in file_moves:

+             safer_move(src, dst)

  

+         # hardlink or copy the rpms into the final repodir

+         # TODO: properly consider split-volume functionality

          for fn in rpmdata:

-             # hardlink or copy the rpms into the final repodir

-             # TODO: properly consider split-volume functionality

              rpminfo = rpmdata[fn]

              rpmpath = rpminfo['_fullpath']

              bnp = fn

              bnplet = bnp[0].lower()

-             koji.ensuredir(os.path.join(archdir, bnplet))

-             l_dst = os.path.join(archdir, bnplet, bnp)

+             ddir = os.path.join(archdir, 'Packages', bnplet)

+             koji.ensuredir(ddir)

+             l_dst = os.path.join(ddir, bnp)

              if os.path.exists(l_dst):

                  raise koji.GenericError("File already in repo: %s", l_dst)

              logger.debug("os.link(%r, %r)", rpmpath, l_dst)

@@ -12717,8 +12745,7 @@ 

                  os.link(rpmpath, l_dst)

              except OSError as ose:

                  if ose.errno == 18:

-                     shutil.copy2(

-                         rpmpath, os.path.join(archdir, bnplet, bnp))

+                     shutil.copy2(rpmpath, l_dst)

                  else:

                      raise

  

@@ -252,14 +252,16 @@ 

    --allow-missing-signatures

                          For RPMs not signed with a desired key, fall back to

                          the primary copy

-   --arch=ARCH           Indicate an architecture to consider. The default is

+   -a ARCH, --arch=ARCH  Indicate an architecture to consider. The default is

                          all architectures associated with the given tag. This

                          option may be specified multiple times.

+   --with-src            Also generate a src repo

+   --split-debuginfo     Split debuginfo info a separate repo for each arch

    --comps=COMPS         Include a comps file in the repodata

    --delta-rpms=REPO     Create delta rpms. REPO can be the id of another dist

                          repo or the name of a tag that has a dist repo. May be

                          specified multiple times.

-   --event=EVENT         create a dist repository based on a Brew event

+   --event=EVENT         Use tag content at event

    --non-latest          Include older builds, not just the latest

    --multilib=CONFIG     Include multilib packages in the repository using the

                          given config file

@@ -1,5 +1,6 @@ 

  

  import unittest

+ import json

  import mock

  import os

  import shutil

@@ -106,7 +107,7 @@ 

              'create_ts': 1487256924.72718,

              'creation_time': '2017-02-16 14:55:24.727181',

              'id': 47,

-             'state': 1,

+             'state': 0,  # INIT

              'tag_id': 2,

              'tag_name': 'my-tag'}

          self.arch = 'x86_64'

@@ -123,19 +124,18 @@ 

          os.makedirs(uploaddir)

  

          # place some test files

-         self.files = ['foo.drpm', 'repomd.xml']

+         self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']

          self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']

          for fn in self.files:

              path = os.path.join(uploaddir, fn)

              koji.ensuredir(os.path.dirname(path))

              with open(path, 'w') as fo:

-                 fo.write('%s' % fn)

+                 fo.write('%s' % os.path.basename(fn))

  

-         # generate pkglist file and sigmap

+         # generate pkglist file

          self.files.append('pkglist')

          plist = os.path.join(uploaddir, 'pkglist')

          nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']

-         self.sigmap = []

          self.rpms = {}

          self.builds ={}

          self.key = '4c8da725'

@@ -153,15 +153,30 @@ 

                      fo.write('%s' % basename)

                  f_pkglist.write(path)

                  f_pkglist.write('\n')

-                 self.expected.append('x86_64/%s/%s' % (basename[0], basename))

+                 self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))

                  build_id = len(self.builds) + 10000

                  rpm_id = len(self.rpms) + 20000

                  binfo['id'] = build_id

                  rpminfo['build_id'] = build_id

                  rpminfo['id'] = rpm_id

+                 rpminfo['sigkey'] = self.key

+                 rpminfo['size'] = 1024

+                 rpminfo['payloadhash'] = 'helloworld'

                  self.builds[build_id] = binfo

                  self.rpms[rpm_id] = rpminfo

-                 self.sigmap.append([rpm_id, self.key])

+ 

+         # write kojipkgs

+         kojipkgs = {}

+         for rpminfo in self.rpms.values():

+             bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo

+             kojipkgs[bnp] = rpminfo

+         with open("%s/kojipkgs" % uploaddir, "w") as fp:

+             json.dump(kojipkgs, fp, indent=4)

+         self.files.append('kojipkgs')

+ 

+         # write manifest

+         with open("%s/repo_manifest" % uploaddir, "w") as fp:

+             json.dump(self.files, fp, indent=4)

  

          # mocks

          self.repo_info = mock.patch('kojihub.repo_info').start()

@@ -187,8 +202,7 @@ 

  

      def test_distRepoMove(self):

          exports = kojihub.HostExports()

-         exports.distRepoMove(self.rinfo['id'], self.uploadpath,

-                 list(self.files), self.arch, self.sigmap)

+         exports.distRepoMove(self.rinfo['id'], self.uploadpath, self.arch)

          # check result

          repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo

          for relpath in self.expected:

@@ -0,0 +1,75 @@ 

+ import mock

+ import unittest

+ 

+ import koji

+ import kojihub

+ 

+ 

+ QP = kojihub.QueryProcessor

+ IP = kojihub.InsertProcessor

+ UP = kojihub.UpdateProcessor

+ 

+ 

+ class TestRepoFunctions(unittest.TestCase):

+ 

+     def setUp(self):

+         self.QueryProcessor = mock.patch('kojihub.QueryProcessor',

+                 side_effect=self.getQuery).start()

+         self.queries = []

+         self.InsertProcessor = mock.patch('kojihub.InsertProcessor',

+                 side_effect=self.getInsert).start()

+         self.inserts = []

+         self.UpdateProcessor = mock.patch('kojihub.UpdateProcessor',

+                 side_effect=self.getUpdate).start()

+         self.updates = []

+         self._dml = mock.patch('kojihub._dml').start()

+ 

+     def tearDown(self):

+         mock.patch.stopall()

+ 

+     def getQuery(self, *args, **kwargs):

+         query = QP(*args, **kwargs)

+         query.execute = mock.MagicMock()

+         self.queries.append(query)

+         return query

+ 

+     def getInsert(self, *args, **kwargs):

+         insert = IP(*args, **kwargs)

+         insert.execute = mock.MagicMock()

+         self.inserts.append(insert)

+         return insert

+ 

+     def getUpdate(self, *args, **kwargs):

+         update = UP(*args, **kwargs)

+         update.execute = mock.MagicMock()

+         self.updates.append(update)

+         return update

+ 

+     def test_repo_expire_older(self):

+         kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id)

+         self.assertEqual(len(self.updates), 1)

+         update = self.updates[0]

+         self.assertEqual(update.table, 'repo')

+         self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})

+         self.assertEqual(update.rawdata, {})

+         self.assertEqual(update.values['event_id'], mock.sentinel.event_id)

+         self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)

+         self.assertEqual(update.values['dist'], None)

+         if 'dist = %(dist)s' in update.clauses:

+             raise Exception('Unexpected dist condition')

+ 

+         # and with dist specified

+         for dist in True, False:

+             self.updates = []

+             kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id,

+                                       dist=dist)

+             self.assertEqual(len(self.updates), 1)

+             update = self.updates[0]

+             self.assertEqual(update.table, 'repo')

+             self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})

+             self.assertEqual(update.rawdata, {})

+             self.assertEqual(update.values['event_id'], mock.sentinel.event_id)

+             self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)

+             self.assertEqual(update.values['dist'], dist)

+             if 'dist = %(dist)s' not in update.clauses:

+                 raise Exception('Missing dist condition')

file modified
+2 -2

@@ -225,7 +225,7 @@ 

          #end if

          #elif $task.method == 'distRepo'

          <strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>

-         <strong>Repo ID:</strong> $params[1]<br/>

+         <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>

          <strong>Keys:</strong> $printValue(0, $params[2])<br/>

          $printOpts($params[3])

          #elif $task.method == 'prepRepo'

@@ -243,7 +243,7 @@ 

          #end if

          #elif $task.method == 'createdistrepo'

          <strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>

-         <strong>Repo ID:</strong> $params[1]<br/>

+         <strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>

          <strong>Arch:</strong> $printValue(0, $params[2])<br/>

          <strong>Keys:</strong> $printValue(0, $params[3])<br/>

          <strong>Options:</strong> $printMap($params[4], '&nbsp;&nbsp;&nbsp;&nbsp;')

  • support split debuginfo
  • avoid expiring not-dist repos
  • general cleanup

Fixes #409
Fixes #457

What about os.path.join(self.repodir, 'Packages', bnplet, bnp)? It is more close to current repos (at least in Fedora). Or maybe make it configurable?

Hmm, also unexpected structure. What about placing 'normal' packages to 'os', so all subrepos are on same (highest) level?

What about os.path.join(self.repodir, 'Packages', bnplet, bnp)? It is more close to current repos (at least in Fedora). Or maybe make it configurable?

This is the layout dist repos had before. Changing that wasn't on the list, though certainly we could.

Are we talking about a layout like so?

dist-arch
|
|---- Packages
|---- repodata
|---- debug
     |
     |---- Packages
     |---- repodata

Hmm, also unexpected structure. What about placing 'normal' packages to 'os', so all subrepos are on same (highest) level?

Trying not to break existing paths, but maybe that doesn't matter. I'm not sure how many folks are relying on the feature as-is, or how much trouble it would be for them to change their urls.

'os' might not make sense for all tags, though.

Current PR creates following:

tag/repo_id
├── src
│   ├── a
│   ├── repodata
│   └── z
└── x86_64
    ├── a
    ├── debug
    │   └── repodata
    ├── repodata
    └── z

I've suggested:

tag/repo_id
├── src
│   ├── Packages
│   │   ├── a
│   │   └── z
│   └── repodata
└── x86_64
    ├── debug
    │   ├── Packages       
    │   │   ├── a
    │   │   └── z
    │   └── repodata
    └── os
        ├── Packages       
        │   ├── a
        │   └── z
        └── repodata

But yes, it is not compatible with current layout. On the other hand, I find it little messy to have all these data mixed in one level (especially letter-based dirs with its repodata and debug directory).

About 'os' - do you mean just the name doesn't make sense, or even location when everything is put together (default behaviour)?

@tkopecek I actually prefer your suggested layout, but the name os is definitely wrong. Maybe make that name configurable, so that it can be release, updates, testing, etc.?

The reason I like your layout is that it makes it easy to merge tag exports together easily on the filesystem.

That said, the current layout proposed probably makes sense too, and is less difficult for naming. And the tag/repo-id already exports out as a top level directory...

About 'os' - do you mean just the name doesn't make sense, or even location when everything is put together (default behaviour)?

I mean the name. The content of a tag might not constitute an os. It could just be a small set of packages.

So, we're inflating the feature quite a lot here....

It'd probably be easier to go without the os directory. The current proposed layout works, though I like the idea of the [a-z] directory being under a Packages directory.

Yep, I think we can just store rpms in Packages dirs, so there is a little cleanup and no change to current repomd path.

Ok, that much is easy enough and not a compat problem

Will update PR shortly

@mikem: Would it also be possible to consider lib{,x32,64}* as a separate prefix to l? This would be similar to what Debian/Ubuntu does.

For example: http://archive.ubuntu.com/ubuntu/pool/main/

Oh, because of dir size? Interesting, but I don't know if we need to rush that in right now.

@mikem Yes. This gets really painful in Mageia because we do lib packages for every library, so the enumeration of the l directory is obnoxious.

For repos, traversing the directory should rarely be done. The whole point of a repo is to use the repodata as an index.

@mikem It's mainly for partial syncs and a few other things done by people in our community...

In Fedora, l is one of the bad ones, but not nearly as bad as p.

$ cat rpms | cut -c 1|sort|uniq -c|sort -n|tail
   2128 a
   2427 c
   3007 s
   3021 n
   3039 r
   3646 m
   6316 l
   6458 g
   7576 t
  13858 p

So, I'm open to making an adjustment here, but probably not for 1.16, which we are currently trying to freeze for.

that's f28 x86_64 data btw. Looks like p accounts for almost 20% of rpm names there. l is only 8.9%. For reference 1/26 is about 3.8%.

For current Mageia Cauldron, this is what it looks like:

$ cat cauldron-rpms | cut -c 1|sort|uniq -c|sort -n|tail
    715 o
    833 j
    899 n
    954 a
   1008 r
   1356 s
   1452 g
   1500 m
   6859 l
   7340 p

p is a bit higher than l, but they're both up there...

This probably needs to be either 1) a configurable regex, or 2) dynamically split based on the content in the repo. Either way, I don't think we need to hold up 1.16 over it. Do you?

Maybe you could file this is a separate rfe?

@mikem No, I don't think it needs to be held up, but I wanted to give that initial feedback for consideration.

@mikem For what it's worth, a configurable regex makes sense, because we may want to also split on perl- and python[23]- prefixes, too...

1 new commit added

  • put dist repo rpms under Packages/
a year ago

rebased onto 9c910eff6450119bb123228a277579acfe3f746c

a year ago

@mikem, I've filed the request as #915.

rebased onto 2b7447e

a year ago

Commit 18600b2 fixes this pull-request

Pull-Request has been merged by mikem

a year ago