From 9b0edaafe902a0885d2e9d35986784053269d8d1 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Apr 11 2016 17:19:15 +0000 Subject: [ostree-installer] Clone repo with templates We can't assume the templates will just be available. This patch adds a configuration option to point to the git repo with them. It will be cloned at compose box and relative paths to templates will be made absolute respective to this clone. Signed-off-by: Lubomír Sedlář --- diff --git a/doc/configuration.rst b/doc/configuration.rst index ff26b44..94704ae 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1005,6 +1005,13 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. * ``add_arch_template`` -- (*[str]*) * ``add_template_var`` -- (*[str]*) * ``add_arch_template_var`` -- (*[str]*) + * ``template_repo`` -- (*str*) Git repository with extra templates. + * ``template_branch`` -- (*str*) Branch to use from ``template_repo``. + + The templates can either be absolute paths, in which case they will be used + as configured; or they can be relative paths, in which case + ``template_repo`` needs to point to a Git repository from which to take the + templates. Example config @@ -1018,17 +1025,19 @@ Example config "release": None, "filename": "%(release_short)s-%(variant)s-%(arch)s-%(version)s-%(compose_date)s.iso", "installpkgs": ["fedora-productimg-atomic"], - "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], + "add_template": ["atomic-installer/lorax-configure-repo.tmpl"], "add_template_var": [ "ostree_osname=fedora-atomic", "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", ], - "add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"], + "add_arch_template": ["atomic-installer/lorax-embed-repo.tmpl"], "add_arch_template_var": [ "ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/", "ostree_osname=fedora-atomic", "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", ] + 'template_repo': 'https://git.fedorahosted.org/git/spin-kickstarts.git', + 'template_branch': 'f24', } }) ] diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index c73bbea..7399874 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -10,7 +10,7 @@ from kobo import shortcuts from .base import ConfigGuardedPhase from .. import util from ..paths import translate_path -from ..wrappers import kojiwrapper, iso, lorax +from ..wrappers import kojiwrapper, iso, lorax, scm class OstreeInstallerPhase(ConfigGuardedPhase): @@ -56,6 +56,9 @@ class OstreeInstallerThread(WorkerThread): output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') util.makedirs(os.path.dirname(output_dir)) + self.template_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'lorax_templates') + self._clone_templates(config.get('template_repo'), config.get('template_branch')) + self._run_ostree_cmd(compose, variant, arch, config, source_repo, output_dir) disc_type = compose.conf.get('disc_types', {}).get('dvd', 'dvd') @@ -65,6 +68,13 @@ class OstreeInstallerThread(WorkerThread): self._add_to_manifest(compose, variant, arch, filename) self.pool.log_info('[DONE ] %s' % msg) + def _clone_templates(self, url, branch='master'): + if not url: + self.template_dir = None + return + scm.get_dir_from_scm({'scm': 'git', 'repo': url, 'branch': branch, 'dir': '.'}, + self.template_dir, logger=self.pool._logger) + def _get_release(self, compose, config): if 'release' in config and config['release'] is None: return compose.image_release @@ -105,6 +115,20 @@ class OstreeInstallerThread(WorkerThread): pass compose.im.add(variant.uid, arch, img) + def _get_templates(self, config, key): + """Retrieve all templates from configuration and make sure the paths + are absolute. Raises RuntimeError if template repo is needed but not + configured. + """ + templates = [] + for template in config.get(key, []): + if template[0] != '/': + if not self.template_dir: + raise RuntimeError('Relative path to template without setting template_repo.') + template = os.path.join(self.template_dir, template) + templates.append(template) + return templates + def _run_ostree_cmd(self, compose, variant, arch, config, source_repo, output_dir): lorax_wrapper = lorax.LoraxWrapper() cmd = lorax_wrapper.get_lorax_cmd( @@ -116,8 +140,8 @@ class OstreeInstallerThread(WorkerThread): variant=variant.uid, nomacboot=True, buildinstallpackages=config.get('installpkgs'), - add_template=config.get('add_template'), - add_arch_template=config.get('add_arch_template'), + add_template=self._get_templates(config, 'add_template'), + add_arch_template=self._get_templates(config, 'add_arch_template'), add_template_var=config.get('add_template_var'), add_arch_template_var=config.get('add_arch_template_var') ) diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 6a8205c..7007a3f 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -172,6 +172,123 @@ class OstreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.wrappers.iso.IsoWrapper') @mock.patch('os.link') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_fail_with_relative_template_path_but_no_repo(self, KojiWrapper, link, + IsoWrapper, get_file_size, + get_mtime, ImageCls, run): + compose = helpers.DummyCompose(self.topdir, { + 'release_name': 'Fedora', + 'release_version': 'Rawhide', + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + }) + pool = mock.Mock() + cfg = { + 'source_repo_from': 'Everything', + 'release': '20160321.n.0', + 'filename': 'Fedora-Atomic.iso', + 'add_template': ['some-file.txt'], + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + get_file_size.return_value = 1024 + get_mtime.return_value = 13579 + + t = ostree.OstreeInstallerThread(pool) + + with self.assertRaises(RuntimeError) as ctx: + t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertIn('template_repo', str(ctx.exception)) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.wrappers.iso.IsoWrapper') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_clone_templates(self, KojiWrapper, link, IsoWrapper, + get_file_size, get_mtime, ImageCls, run, + get_dir_from_scm): + compose = helpers.DummyCompose(self.topdir, { + 'release_name': 'Fedora', + 'release_version': 'Rawhide', + 'koji_profile': 'koji', + 'runroot_tag': 'rrt', + }) + pool = mock.Mock() + cfg = { + 'source_repo_from': 'Everything', + 'release': '20160321.n.0', + 'filename': 'Fedora-Atomic.iso', + 'add_template': ['some_file.txt'], + 'add_arch_template': ['other_file.txt'], + 'template_repo': 'git://example.com/templates.git', + 'template_branch': 'f24', + } + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + get_file_size.return_value = 1024 + get_mtime.return_value = 13579 + final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name' + templ_dir = self.topdir + '/work/x86_64/Everything/lorax_templates' + + t = ostree.OstreeInstallerThread(pool) + + t.process((compose, compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertEqual(get_dir_from_scm.call_args_list, + [mock.call({'scm': 'git', 'repo': 'git://example.com/templates.git', + 'branch': 'f24', 'dir': '.'}, + templ_dir, logger=pool._logger)]) + self.assertEqual(koji.get_runroot_cmd.call_args_list, + [mock.call('rrt', 'x86_64', + ['lorax', + '--product=Fedora', + '--version=Rawhide', + '--release=20160321.n.0', + '--source=file://{}/compose/Everything/x86_64/os'.format(self.topdir), + '--variant=Everything', + '--nomacboot', + '--add-template={}/some_file.txt'.format(templ_dir), + '--add-arch-template={}/other_file.txt'.format(templ_dir), + self.topdir + '/work/x86_64/Everything/ostree_installer'], + channel=None, mounts=[self.topdir], + packages=['pungi', 'lorax'], + task_id=True, use_shell=True)]) + self.assertEqual(koji.run_runroot_cmd.call_args_list, + [mock.call(koji.get_runroot_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/ostree_installer/runroot.log')]) + self.assertEqual(link.call_args_list, + [mock.call(self.topdir + '/work/x86_64/Everything/ostree_installer/images/boot.iso', + final_iso_path)]) + self.assertEqual(get_file_size.call_args_list, [mock.call(final_iso_path)]) + self.assertEqual(get_mtime.call_args_list, [mock.call(final_iso_path)]) + self.assertImageAdded(compose, ImageCls, IsoWrapper) + self.assertEqual(compose.get_image_name.call_args_list, + [mock.call('x86_64', compose.variants['Everything'], + disc_type='dvd', format='Fedora-Atomic.iso')]) + self.assertTrue(os.path.isdir(self.topdir + '/work/x86_64/Everything/')) + self.assertFalse(os.path.isdir(self.topdir + '/work/x86_64/Everything/ostree_installer')) + self.assertEqual(run.call_args_list, + [mock.call('cp -av {0}/work/x86_64/Everything/ostree_installer/* {0}/compose/Everything/x86_64/iso/'.format(self.topdir))]) + + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.wrappers.iso.IsoWrapper') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_run_with_implicit_release(self, KojiWrapper, link, IsoWrapper, get_file_size, get_mtime, ImageCls, run): compose = helpers.DummyCompose(self.topdir, {