From 5216c545721b9b97d32f21b195badd42c50b12fe Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 12 2019 21:09:19 +0000 Subject: distrepo via dnf --- diff --git a/builder/kojid b/builder/kojid index ddd1c00..3fed2e1 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5288,7 +5288,10 @@ class createDistRepoTask(BaseTaskHandler): self.uploadpath = self.getUploadDir() self.get_rpms(tag, arch, keys, opts) if opts['multilib'] and koji.arch.isMultiLibArch(arch): - self.do_multilib(arch, self.archmap[arch], opts['multilib']) + if dnf is not None: + self.do_multilib_dnf(arch, self.archmap[arch], opts['multilib']) + else: + self.do_multilib_yum(arch, self.archmap[arch], opts['multilib']) self.split_pkgs(opts) self.write_kojipkgs() self.write_pkglist() @@ -5381,7 +5384,135 @@ class createDistRepoTask(BaseTaskHandler): raise koji.GenericError('failed to create repo: %s' \ % parseStatus(status, ' '.join(cmd))) - def do_multilib(self, arch, ml_arch, conf): + + def do_multilib_dnf(self, arch, ml_arch, conf): + repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) + mldir = os.path.join(repodir, koji.canonArch(ml_arch)) + ml_true = set() # multilib packages we need to include before depsolve + ml_conf = os.path.join(koji.pathinfo.work(), conf) + + # read pkgs data from multilib repo + ml_pkgfile = os.path.join(mldir, 'kojipkgs') + ml_pkgs = json.load(open(ml_pkgfile, 'r')) + + # step 1: figure out which packages are multilib (should already exist) + dnfbase = dnf.Base() + mlm = multilib.DevelMultilibMethod(ml_conf) + fs_missing = set() + for bnp in self.kojipkgs: + rpminfo = self.kojipkgs[bnp] + ppath = rpminfo['_pkgpath'] + dnfbase.fill_sack(load_system_repo=False, load_available_repos=False) + po = dnfbase.sack.add_cmdline_package(ppath) + if mlm.select(po): + # we need a multilib package to be included + ml_bnp = bnp.replace(arch, self.archmap[arch]) + ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp) + # ^ XXX - should actually generate this + if ml_bnp not in ml_pkgs: + # not in our multilib repo + self.logger.error('%s (multilib) is not on the filesystem' % ml_path) + fs_missing.add(ml_path) + # we defer failure so can report all the missing deps + continue + ml_true.add(ml_path) + + # step 2: set up architectures for dnf configuration + self.logger.info("Resolving multilib for %s using method devel" % arch) + dnfdir = os.path.join(self.workdir, 'dnf') + # TODO: unwind this arch mess + archlist = (arch, 'noarch') + transaction_arch = arch + archlist = archlist + self.compat[self.biarch[arch]] + best_compat = self.compat[self.biarch[arch]][0] + if koji.arch.archDifference(best_compat, arch) > 0: + transaction_arch = best_compat + dnfconfig = """ +[main] +debuglevel=2 +#pkgpolicy=newest +#exactarch=1 +gpgcheck=0 +#reposdir=/dev/null +#cachedir=/dnfcache +installroot=%s +#logfile=/dnf.log + +[koji-%s] +name=koji multilib task +baseurl=file://%s +enabled=1 + +""" % (dnfdir, self.id, mldir) + os.makedirs(os.path.join(dnfdir, "dnfcache")) + os.makedirs(os.path.join(dnfdir, 'var/lib/rpm')) + + # step 3: proceed with dnf config and set up + yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch) + with open(yconfig_path, 'w') as f: + f.write(dnfconfig) + self.session.uploadWrapper(yconfig_path, self.uploadpath, + os.path.basename(yconfig_path)) + conf = dnf.conf.Conf() + conf.reposdir = [] # don't use system repos at all + conf.read(yconfig_path) + dnfbase = dnf.Base(conf) + if hasattr(koji.arch, 'ArchStorage'): + dnfbase.conf.arch = transaction_arch + else: + koji.arch.canonArch = transaction_arch + dnfbase.read_all_repos() + dnfbase.fill_sack(load_system_repo=False, load_available_repos=True) + for pkg in ml_true: + dnfbase.install(pkg) + + # step 4: execute dnf transaction to get dependencies + self.logger.info("Resolving dependencies for arch %s" % arch) + + ml_needed = {} + try: + dnfbase.resolve() + self.logger.info('dnf depsolve successfully finished') + for po in dnfbase.transaction.install_set: + bnp = os.path.basename(po.localPkg()) + dep_path = os.path.join(mldir, bnp[0].lower(), bnp) + ml_needed[dep_path] = po + if not os.path.exists(dep_path): + self.logger.error('%s (multilib dep) not on filesystem' % dep_path) + fs_missing.add(dep_path) + except dnf.exceptions.DepsolveError: + self.logger.error('dnf depsolve was unsuccessful') + raise + + if len(fs_missing) > 0: + missing_log = os.path.join(self.workdir, 'missing_multilib.log') + with open(missing_log, 'w') as outfile: + outfile.write('The following multilib files were missing:\n') + for ml_path in fs_missing: + outfile.write(ml_path + '\n') + self.session.uploadWrapper(missing_log, self.uploadpath) + raise koji.GenericError('multilib packages missing. ' + 'See missing_multilib.log') + + # step 5: update kojipkgs + for dep_path in ml_needed: + tspkg = ml_needed[dep_path] + bnp = os.path.basename(dep_path) + if bnp in self.kojipkgs: + # we expect duplication with noarch, but not other arches + if tspkg.arch != 'noarch': + self.logger.warning("Multilib duplicate: %s", bnp) + continue + rpminfo = ml_pkgs[bnp].copy() + # fix _pkgpath, which comes from another task and could be wrong + # for us + # TODO: would be better if we could use the proper path here + rpminfo['_pkgpath'] = dep_path + rpminfo['_multilib'] = True + self.kojipkgs[bnp] = rpminfo + + + def do_multilib_yum(self, arch, ml_arch, conf): repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) ml_true = set() # multilib packages we need to include before depsolve @@ -5392,12 +5523,15 @@ class createDistRepoTask(BaseTaskHandler): ml_pkgs = json.load(open(ml_pkgfile, 'r')) # step 1: figure out which packages are multilib (should already exist) + dnfbase = dnf.Base() mlm = multilib.DevelMultilibMethod(ml_conf) fs_missing = set() for bnp in self.kojipkgs: rpminfo = self.kojipkgs[bnp] ppath = rpminfo['_pkgpath'] - po = yum.packages.YumLocalPackage(filename=ppath) + #po = yum.packages.YumLocalPackage(filename=ppath) + dnfbase.fill_sack(load_system_repo=False, load_available_repos=False) + po = dnfbase.sack.add_cmdline_package(ppath) if mlm.select(po): # we need a multilib package to be included ml_bnp = bnp.replace(arch, self.archmap[arch])