From bb4afea4f16c593d693a6c17949ca0cc18de2f6a Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: May 23 2016 16:30:14 +0000 Subject: [PATCH 1/2] [image-build] Allow using external install trees This is needed for the twoweek Fedora atomic release to point to regular nightly composes. Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index b45dae3..dcd74d3 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -57,6 +57,8 @@ class ImageBuildPhase(base.ImageConfigMixin, base.ConfigGuardedPhase): dict. """ install_tree_from = image_conf.pop('install_tree_from', variant.uid) + if '://' in install_tree_from: + return install_tree_from install_tree_source = self.compose.variants.get(install_tree_from) if not install_tree_source: raise RuntimeError( diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index 87fa84b..33933c3 100755 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -310,6 +310,66 @@ class TestImageBuildPhase(PungiTestCase): }) @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build_set_external_install_tree(self, ThreadPool): + compose = DummyCompose(self.topdir, { + 'image_build': { + '^Server$': [ + { + 'image-build': { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3, + 'arches': ['x86_64'], + 'install_tree_from': 'http://example.com/install-tree/', + } + } + ] + }, + 'koji_profile': 'koji', + }) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + + self.assertTrue(phase.pool.queue_put.called_once) + args, kwargs = phase.pool.queue_put.call_args + self.assertEqual(args[0][0], compose) + self.maxDiff = None + self.assertDictEqual(args[0][1], { + "format": [('docker', 'tar.xz')], + "image_conf": { + 'image-build': { + 'install_tree': 'http://example.com/install-tree/', + 'kickstart': 'fedora-docker-base.ks', + 'format': 'docker', + 'repo': ','.join([self.topdir + '/compose/Server/$arch/os']), + 'variant': compose.variants['Server'], + 'target': 'f24', + 'disk_size': 3, + 'name': 'Fedora-Docker-Base', + 'arches': 'x86_64', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'distro': 'Fedora-20', + } + }, + "conf_file": self.topdir + '/work/image-build/Server/docker_Fedora-Docker-Base.cfg', + "image_dir": self.topdir + '/compose/Server/%(arch)s/images', + "relative_image_dir": 'Server/%(arch)s/images', + "link_type": 'hardlink-or-copy', + "scratch": False, + }) + + @mock.patch('pungi.phases.image_build.ThreadPool') def test_image_build_create_release(self, ThreadPool): compose = DummyCompose(self.topdir, { 'image_build': { From c8341e180672669d8041ecba33b320882df72ec0 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: May 24 2016 08:06:06 +0000 Subject: [PATCH 2/2] [ostree-installer] Allow using external repos as source Signed-off-by: Lubomír Sedlář --- diff --git a/doc/configuration.rst b/doc/configuration.rst index fb450c0..e6ab404 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1033,7 +1033,7 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. The configuration dict for each variant arch pair must have this key: * ``source_repo_from`` -- (*str*) Name of variant serving as source - repository. + repository or a URL pointing the the repo. These keys are optional: diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index f7ede1f..5d8a762 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -50,9 +50,7 @@ class OstreeInstallerThread(WorkerThread): self.pool.log_info('[BEGIN] %s' % msg) self.logdir = compose.paths.log.topdir('{}/ostree_installer'.format(arch)) - source_variant = compose.variants[config['source_repo_from']] - source_repo = translate_path( - compose, compose.paths.compose.repository(arch, source_variant, create_dir=False)) + source_repo = self._get_source_repo(compose, arch, config['source_repo_from']) output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') util.makedirs(os.path.dirname(output_dir)) @@ -67,6 +65,18 @@ class OstreeInstallerThread(WorkerThread): self._add_to_manifest(compose, variant, arch, filename) self.pool.log_info('[DONE ] %s' % msg) + def _get_source_repo(self, compose, arch, source): + """ + If `source` is a URL, return it as-is (possibly replacing $arch with + actual arch. Otherwise treat is a a variant name and return path to + repo in that variant. + """ + if '://' in source: + return source.replace('$arch', arch) + source_variant = compose.variants[source] + return translate_path( + compose, compose.paths.compose.repository(arch, source_variant, create_dir=False)) + def _clone_templates(self, url, branch='master'): if not url: self.template_dir = None diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 6162904..d8a2832 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -169,6 +169,69 @@ class OstreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.wrappers.iso.IsoWrapper') @mock.patch('os.link') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run(self, KojiWrapper, link, IsoWrapper, + get_file_size, get_mtime, ImageCls, run): + compose = helpers.DummyCompose(self.topdir, { + 'release_name': 'Fedora', + 'release_version': 'Rawhide', + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + }) + pool = mock.Mock() + cfg = { + 'source_repo_from': 'http://example.com/repo/$arch/', + 'release': '20160321.n.0', + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + get_file_size.return_value = 1024 + get_mtime.return_value = 13579 + final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name' + + t = ostree.OstreeInstallerThread(pool) + + t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertEqual(koji.get_runroot_cmd.call_args_list, + [mock.call('rrt', 'x86_64', + ['lorax', + '--product=Fedora', + '--version=Rawhide', + '--release=20160321.n.0', + '--source=http://example.com/repo/x86_64/', + '--variant=Everything', + '--nomacboot', + self.topdir + '/work/x86_64/Everything/ostree_installer'], + channel=None, mounts=[self.topdir], + packages=['pungi', 'lorax', 'ostree'], + task_id=True, use_shell=True)]) + self.assertEqual(koji.run_runroot_cmd.call_args_list, + [mock.call(koji.get_runroot_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')]) + self.assertEqual(link.call_args_list, + [mock.call(self.topdir + '/work/x86_64/Everything/ostree_installer/images/boot.iso', + final_iso_path)]) + self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)]) + self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)]) + self.assertImageAdded(compose, ImageCls, IsoWrapper) + self.assertEqual(compose.get_image_name.call_args_list, + [mock.call('x86_64', compose.variants['Everything'], disc_type='dvd')]) + self.assertTrue(os.path.isdir(self.topdir + '/work/x86_64/Everything/')) + self.assertFalse(os.path.isdir(self.topdir + '/work/x86_64/Everything/ostree_installer')) + self.assertEqual(run.call_args_list, + [mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/iso/'.format(self.topdir))]) + + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.wrappers.iso.IsoWrapper') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link, IsoWrapper, get_file_size, get_mtime, ImageCls, run):