pungi

Clone

#919 gather: Use another variant as lookaside
Merged 6 years ago by lsedlar. Opened 6 years ago by lsedlar.
lsedlar/pungi variant-lookaside  into  master

file modified
+18 
@@ -179,6 +179,17 @@ 
 

              makedirs(path)
 

          return path
 

  

+     def lookaside_repo(self, arch, variant, create_dir=True):
 

+         """
 

+         Examples:
 

+             work/x86_64/Server/lookaside_repo
 

+         """
 

+         path = os.path.join(self.topdir(arch, create_dir=create_dir),
 

+                             variant.uid, "lookaside_repo")
 

+         if create_dir:
 

+             makedirs(path)
 

+         return path
 

+ 

      def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
 

          """
 

          Examples:
 
@@ -200,6 +211,13 @@ 
 

          path = os.path.join(path, file_name)
 

          return path
 

  

+     def lookaside_package_list(self, arch, variant, create_dir=True):
 

+         """
 

+         Examples:
 

+             work/x86_64/package_list/Server.x86_64.lookaside.conf
 

+         """
 

+         return self.package_list(arch, variant, pkg_type='lookaside', create_dir=create_dir)
 

+ 

      def pungi_download_dir(self, arch, create_dir=True):
 

          """
 

          Examples:

file modified
+82 -1 
@@ -19,16 +19,20 @@ 
 

  import shutil
 

  

  from kobo.rpmlib import parse_nvra
 

+ from kobo.shortcuts import run
 

  from productmd.rpms import Rpms
 

  

  from pungi.wrappers.scm import get_file_from_scm
 

  from .link import link_files
 

+ from ...wrappers.createrepo import CreaterepoWrapper
 

+ import pungi.wrappers.kojiwrapper
 

  

  from pungi import Modulemd
 

  from pungi.arch import get_compatible_arches, split_name_arch
 

  from pungi.graph import SimpleAcyclicOrientedGraph
 

  from pungi.phases.base import PhaseBase
 

- from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
 

+ from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
 

+                         makedirs)
 

  

  

  def get_gather_source(name):
 
@@ -308,6 +312,77 @@ 
 

      return list(variant_processing_order)
 

  

  

+ def _make_lookaside_repo(compose, variant, arch, pkg_map):
 

+     """
 

+     Create variant lookaside repo for given variant and architecture with
 

+     packages from the map. If the repo repo already exists, then nothing will
 

+     happen. This could happen if multiple variants depend on this one.
 

+     """
 

+     repo = compose.paths.work.lookaside_repo(arch, variant, create_dir=False)
 

+     if os.path.exists(repo):
 

+         # We have already generated this, nothing to do.
 

+         return repo
 

+ 

+     makedirs(repo)
 

+     msg = 'Generating lookaside repo from %s.%s' % (variant.uid, arch)
 

+     compose.log_info('[BEGIN] %s', msg)
 

+ 

+     prefixes = {
 

+         'repos': lambda: os.path.join(compose.paths.work.topdir(
 

+             arch="global"), "download") + "/",
 

+         'koji': lambda: pungi.wrappers.kojiwrapper.KojiWrapper(
 

+             compose.conf['koji_profile']).koji_module.config.topdir.rstrip("/") + "/"
 

+     }
 

+     path_prefix = prefixes[compose.conf['pkgset_source']]()
 

+     pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant)
 

+     with open(pkglist, 'w') as f:
 

+         for packages in pkg_map[arch][variant.uid].values():
 

+             for pkg in packages:
 

+                 pkg = pkg['path']
 

+                 if path_prefix and pkg.startswith(path_prefix):
 

+                     pkg = pkg[len(path_prefix):]
 

+                 f.write('%s\n' % pkg)
 

+ 

+     cr = CreaterepoWrapper(compose.conf['createrepo_c'])
 

+     cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
 

+                                 pkglist=pkglist,
 

+                                 outputdir=repo,
 

+                                 baseurl="file://%s" % path_prefix,
 

+                                 workers=compose.conf["createrepo_num_workers"],
 

+                                 update_md_path=compose.paths.work.arch_repo(arch))
 

+     run(cmd,
 

+         logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)),
 

+         show_cmd=True)
 

+     compose.log_info('[DONE ] %s', msg)
 

+ 

+     return repo
 

+ 

+ 

+ def _update_config(compose, variant_uid, arch, repo):
 

+     """
 

+     Add the variant lookaside repository into the configuration.
 

+     """
 

+     lookasides = compose.conf.setdefault('gather_lookaside_repos', [])
 

+     lookasides.append(('^%s$' % variant_uid, {arch: repo}))
 

+ 

+ 

+ def _update_lookaside_config(compose, variant, arch, pkg_map):
 

+     """
 

+     Make sure lookaside repo for all variants that the given one depends on
 

+     exist, and that configuration is updated to use those repos.
 

+     """
 

+     for dest, lookaside_variant_uid in compose.conf.get('variant_as_lookaside', []):
 

+         lookaside_variant = compose.all_variants[lookaside_variant_uid]
 

+         if dest != variant.uid:
 

+             continue
 

+         if arch not in lookaside_variant.arches:
 

+             compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch',
 

+                                 lookaside_variant.uid, variant.uid, arch)
 

+             continue
 

+         repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map)
 

+         _update_config(compose, variant.uid, arch, repo)
 

+ 

+ 

  def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
 

      """Run gathering on all arches of all variants of given type.
 

  
@@ -330,6 +405,12 @@ 
 

              if exclude_fulltree:
 

                  for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:
 

                      fulltree_excludes.add(pkg_name)
 

+ 

+             # Get lookaside repos for this variant from other variants. Based
 

+             # on the ordering we already know that we have the packages from
 

+             # there.
 

+             _update_lookaside_config(compose, variant, arch, result)
 

+ 

              pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes)
 

              result.setdefault(arch, {})[variant.uid] = pkg_map

file modified
+173 
@@ -901,3 +901,176 @@ 
 

  

          self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
 

          self.assertItemsEqual(groups, ["core"])
 

+ 

+ 

+ class TestUpdateConfig(unittest.TestCase):
 

+ 

+     def test_add_to_empty(self):
 

+         compose = mock.Mock(conf={})
 

+         gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
 

+         self.assertEqual(compose.conf,
 

+                          {'gather_lookaside_repos': [
 

+                              ('^Server$', {'x86_64': '/tmp/foo'})
 

+                          ]})
 

+ 

+     def test_add_to_existing(self):
 

+         compose = mock.Mock(conf={'gather_lookaside_repos': [
 

+             ('^Server$', {'x86_64': '/tmp/bar'}),
 

+         ]})
 

+         gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
 

+         self.assertEqual(compose.conf,
 

+                          {'gather_lookaside_repos': [
 

+                              ('^Server$', {'x86_64': '/tmp/bar'}),
 

+                              ('^Server$', {'x86_64': '/tmp/foo'})
 

+                          ]})
 

+ 

+ 

+ class TestUpdateLookasideConfig(helpers.PungiTestCase):
 

+ 

+     def setUp(self):
 

+         super(TestUpdateLookasideConfig, self).setUp()
 

+         self.compose = helpers.DummyCompose(self.topdir, {})
 

+         self.pkg_map = mock.Mock()
 

+ 

+     @mock.patch('pungi.phases.gather._update_config')
 

+     @mock.patch('pungi.phases.gather._make_lookaside_repo')
 

+     def test_no_config(self, mock_make_repo, mock_update_config):
 

+         gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
 

+                                         'x86_64', self.pkg_map)
 

+         self.assertEqual(mock_make_repo.call_args_list, [])
 

+         self.assertEqual(mock_update_config.call_args_list, [])
 

+ 

+     @mock.patch('pungi.phases.gather._update_config')
 

+     @mock.patch('pungi.phases.gather._make_lookaside_repo')
 

+     def test_no_matching_config(self, mock_make_repo, mock_update_config):
 

+         self.compose.conf['variant_as_lookaside'] = [('Everything', 'Client')]
 

+         gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
 

+                                         'x86_64', self.pkg_map)
 

+         self.assertEqual(mock_make_repo.call_args_list, [])
 

+         self.assertEqual(mock_update_config.call_args_list, [])
 

+ 

+     @mock.patch('pungi.phases.gather._update_config')
 

+     @mock.patch('pungi.phases.gather._make_lookaside_repo')
 

+     def test_missing_arch(self, mock_make_repo, mock_update_config):
 

+         # Client only has amd64
 

+         self.compose.conf['variant_as_lookaside'] = [('Server', 'Client')]
 

+         gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
 

+                                         'x86_64', self.pkg_map)
 

+         self.assertEqual(len(self.compose.log_warning.call_args_list), 1)
 

+         self.assertEqual(mock_make_repo.call_args_list, [])
 

+         self.assertEqual(mock_update_config.call_args_list, [])
 

+ 

+     @mock.patch('pungi.phases.gather._update_config')
 

+     @mock.patch('pungi.phases.gather._make_lookaside_repo')
 

+     def test_match(self, mock_make_repo, mock_update_config):
 

+         self.compose.conf['variant_as_lookaside'] = [('Server', 'Everything')]
 

+         gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
 

+                                         'x86_64', self.pkg_map)
 

+         self.assertEqual(len(self.compose.log_warning.call_args_list), 0)
 

+         self.assertEqual(mock_make_repo.call_args_list,
 

+                          [mock.call(self.compose,
 

+                                     self.compose.variants['Everything'],
 

+                                     'x86_64',
 

+                                     self.pkg_map)])
 

+         self.assertEqual(mock_update_config.call_args_list,
 

+                          [mock.call(self.compose, 'Server', 'x86_64',
 

+                                     mock_make_repo.return_value)])
 

+ 

+ 

+ class TestMakeLookasideRepo(helpers.PungiTestCase):
 

+ 

+     def setUp(self):
 

+         super(TestMakeLookasideRepo, self).setUp()
 

+         self.compose = helpers.DummyCompose(self.topdir, {})
 

+         self.variant = self.compose.variants['Server']
 

+         self.arch = 'x86_64'
 

+         self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False)
 

+         self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant)
 

+ 

+     @mock.patch('pungi.phases.gather.run')
 

+     def test_existing_repo(self, mock_run):
 

+         helpers.touch(os.path.join(self.repodir, 'repodata', 'primary.xml'))
 

+         repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, {})
 

+         self.assertEqual(self.repodir, repopath)
 

+         self.assertFalse(os.path.exists(self.pkglist))
 

+         self.assertEqual(mock_run.call_args_list, [])
 

+ 

+     def assertCorrect(self, repopath, path_prefix, MockCR, mock_run):
 

+         with open(self.pkglist) as f:
 

+             packages = f.read().splitlines()
 

+         self.assertItemsEqual(packages,
 

+                               ['pkg/pkg-1.0-1.x86_64.rpm',
 

+                                'pkg/pkg-debuginfo-1.0-1.x86_64.rpm',
 

+                                'pkg/pkg-1.0-1.src.rpm'])
 

+ 

+         self.assertEqual(self.repodir, repopath)
 

+         print(MockCR.return_value.get_createrepo_cmd.call_args_list)
 

+         print([mock.call(path_prefix, update=True, database=True, skip_stat=True,
 

+                          pkglist=self.pkglist, outputdir=repopath,
 

+                          baseurl="file://%s" % path_prefix, workers=3,
 

+                          update_md_path=self.compose.paths.work.arch_repo(self.arch))])
 

+         self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list,
 

+                          [mock.call(path_prefix, update=True, database=True, skip_stat=True,
 

+                                     pkglist=self.pkglist, outputdir=repopath,
 

+                                     baseurl="file://%s" % path_prefix, workers=3,
 

+                                     update_md_path=self.compose.paths.work.arch_repo(self.arch))])
 

+         self.assertEqual(mock_run.call_args_list,
 

+                          [mock.call(MockCR.return_value.get_createrepo_cmd.return_value,
 

+                                     logfile=os.path.join(
 

+                                         self.topdir, 'logs', self.arch,
 

+                                         'lookaside_repo_Server.%s.log' % self.arch),
 

+                                     show_cmd=True)])
 

+ 

+     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
 

+     @mock.patch('pungi.phases.gather.CreaterepoWrapper')
 

+     @mock.patch('pungi.phases.gather.run')
 

+     def test_create_repo_koji_pkgset(self, mock_run, MockCR, MockKW):
 

+         self.compose.conf.update({
 

+             'pkgset_source': 'koji',
 

+             'koji_profile': 'koji',
 

+         })
 

+ 

+         pkg_map = {
 

+             self.arch: {
 

+                 self.variant.uid: {
 

+                     'rpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.x86_64.rpm'}],
 

+                     'debuginfo': [{'path': '/tmp/packages/pkg/pkg-debuginfo-1.0-1.x86_64.rpm'}],
 

+                     'srpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.src.rpm'}],
 

+                 }
 

+             }
 

+         }
 

+ 

+         MockKW.return_value.koji_module.config.topdir = '/tmp/packages'
 

+ 

+         repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
 

+ 

+         self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run)
 

+ 

+     @mock.patch('pungi.phases.gather.CreaterepoWrapper')
 

+     @mock.patch('pungi.phases.gather.run')
 

+     def test_create_repo_repos_pkgset(self, mock_run, MockCR):
 

+         self.compose.conf.update({
 

+             'pkgset_source': 'repos',
 

+         })
 

+ 

+         dl_dir = self.compose.paths.work.topdir('global')
 

+ 

+         pkg_map = {
 

+             self.arch: {
 

+                 self.variant.uid: {
 

+                     'rpm': [
 

+                         {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.x86_64.rpm')}
 

+                     ],
 

+                     'debuginfo': [
 

+                         {'path': os.path.join(dl_dir, 'download/pkg/pkg-debuginfo-1.0-1.x86_64.rpm')}
 

+                     ],
 

+                     'srpm': [
 

+                         {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.src.rpm')}
 

+                     ],
 

+                 }
 

+             }
 

+         }
 

+ 

+         repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
 

+ 

+         self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)

Create a temporary repository and add it as another lookaside in the compose.

This should have no effect on composes that do not use this new option (other than depsolving possibly running in different order).

rebased onto c5c27a8d40be5e1a53bbbc747bc2a4b1b6f9d246
6 years ago

rebased onto ffd0c18707da12b88a599cee06545e48e64e87db
6 years ago

rebased onto eaf58f7
6 years ago

Pull-Request has been merged by lsedlar
6 years ago
Changes Summary 3
+18 -0
file changed
pungi/paths.py
+82 -1
file changed
pungi/phases/gather/__init__.py
+173 -0
file changed
tests/test_gather_phase.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.