From 5c081cb545715c2a912ff50fa57554e89d905868 Mon Sep 17 00:00:00 2001 From: Ondrej Nosek Date: Feb 26 2018 12:10:36 +0000 Subject: Other repo for OstreeInstaller OstreeInstaller phase will be moved to a different timeslot and therefore needs different repo not to depend on Gather phase which runs at the same time. Related: https://pagure.io/pungi/issue/778 Signed-off-by: Ondrej Nosek --- diff --git a/pungi/checks.py b/pungi/checks.py index 9b317ab..c6082ea 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -935,7 +935,7 @@ def make_schema(): "tag_ref": {"type": "boolean"}, "ostree_ref": {"type": "string"}, }, - "required": ["treefile", "config_url", "repo", "ostree_repo"], + "required": ["treefile", "config_url", "ostree_repo"], "additionalProperties": False, }), ] @@ -956,7 +956,6 @@ def make_schema(): "template_repo": {"type": "string"}, "template_branch": {"type": "string"}, }, - "required": ["repo"], "additionalProperties": False, }), diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 9bf5e25..b784319 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -9,7 +9,7 @@ from kobo import shortcuts from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util -from ..util import get_volid, get_repo_urls, version_generator +from ..util import get_volid, get_repo_urls, version_generator, translate_path from ..wrappers import kojiwrapper, iso, lorax, scm @@ -64,7 +64,11 @@ class OstreeInstallerThread(WorkerThread): self.pool.log_info('[BEGIN] %s' % msg) self.logdir = compose.paths.log.topdir('%s/%s/ostree_installer-%s' % (arch, variant, self.num)) - repos = get_repo_urls(compose, shortcuts.force_list(config['repo']), arch=arch) + repo_baseurl = compose.paths.work.arch_repo('$basearch', create_dir=False) + repos = get_repo_urls(None, # compose==None. Special value says that method should ignore deprecated variant-type repo + shortcuts.force_list(config['repo']) + shortcuts.force_list(translate_path(compose, repo_baseurl)), + arch=arch, + logger=self.pool) repos = [url.replace('$arch', arch) for url in repos] output_dir = os.path.join(compose.paths.work.topdir(arch), variant.uid, 'ostree_installer') util.makedirs(os.path.dirname(output_dir)) diff --git a/pungi/util.py b/pungi/util.py index 8ea945d..261f3f2 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -678,6 +678,8 @@ def get_repo_url(compose, repo, arch='$basearch'): Convert repo to repo URL. @param compose - required for access to variants + special value compose==None determines that method is called during + OSTreeInstaller phase where variant-type source repository is deprecated @param repo - string or a dict which at least contains 'baseurl' key @param arch - string to be used as arch in repo url """ @@ -688,14 +690,17 @@ def get_repo_url(compose, repo, arch='$basearch'): raise RuntimeError('Baseurl is required in repo dict %s' % str(repo)) if '://' not in repo: # this is a variant name - v = compose.all_variants.get(repo) - if not v: - raise RuntimeError('There is no variant %s to get repo from.' % repo) + if compose is not None: + v = compose.all_variants.get(repo) + if not v: + raise RuntimeError('There is no variant %s to get repo from.' % repo) + else: + return None repo = translate_path(compose, compose.paths.compose.repository(arch, v, create_dir=False)) return repo -def get_repo_urls(compose, repos, arch='$basearch'): +def get_repo_urls(compose, repos, arch='$basearch', logger=None): """ Convert repos to a list of repo URLs. @@ -706,7 +711,11 @@ def get_repo_urls(compose, repos, arch='$basearch'): urls = [] for repo in repos: repo = get_repo_url(compose, repo, arch=arch) - urls.append(repo) + if repo is None: + if logger: + logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'OSTreeInstaller' phase: %s" % (repo)) + else: + urls.append(repo) return urls @@ -764,7 +773,7 @@ def get_repo_dicts(repos, logger=None): repo_dict = get_repo_dict(repo) if repo_dict == {}: if logger: - logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'ostree' phase: %s" % (repo)) + logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'OSTree' phase: %s" % (repo)) else: repo_dicts.append(repo_dict) return repo_dicts diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index ba56d8c..1c7af1a 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -14,6 +14,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from tests import helpers from pungi.phases import ostree_installer as ostree +from six.moves import shlex_quote LOG_PATH = 'logs/x86_64/Everything/ostree_installer-1' @@ -58,6 +59,9 @@ class OstreeThreadTest(helpers.PungiTestCase): 'koji_profile': 'koji', 'runroot_tag': 'rrt', 'image_volid_formats': ['{release_short}-{variant}-{arch}'], + 'translate_paths': [ + (self.topdir + '/work', 'http://example.com/work') + ], }) def assertImageAdded(self, compose, ImageCls, iso): @@ -84,7 +88,7 @@ class OstreeThreadTest(helpers.PungiTestCase): ] for s in force_list(sources): - lorax_cmd.append('--source=%s' % s) + lorax_cmd.append(shlex_quote('--source=%s' % s)) lorax_cmd.append('--variant=Everything') lorax_cmd.append('--nomacboot') @@ -137,7 +141,7 @@ class OstreeThreadTest(helpers.PungiTestCase): self.compose.supported = False pool = mock.Mock() cfg = { - 'repo': 'Everything', + 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo 'release': '20160321.n.0', } koji = KojiWrapper.return_value @@ -155,7 +159,7 @@ class OstreeThreadTest(helpers.PungiTestCase): t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) self.assertRunrootCall(koji, - 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'http://example.com/work/$basearch/repo', cfg['release'], extra=['--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)]) self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) @@ -190,7 +194,11 @@ class OstreeThreadTest(helpers.PungiTestCase): t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) - self.assertRunrootCall(koji, 'http://example.com/repo/x86_64/', cfg['release'], isfinal=True, + self.assertRunrootCall(koji, + ('http://example.com/repo/x86_64/', + 'http://example.com/work/$basearch/repo'), + cfg['release'], + isfinal=True, extra=['--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)]) self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) self.assertImageAdded(self.compose, ImageCls, iso) @@ -209,7 +217,7 @@ class OstreeThreadTest(helpers.PungiTestCase): cfg = { 'release': '20160321.n.0', 'repo': [ - 'Everything', + 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo 'https://example.com/extra-repo1.repo', 'https://example.com/extra-repo2.repo', ], @@ -226,9 +234,9 @@ class OstreeThreadTest(helpers.PungiTestCase): t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) sources = [ - 'file://%s/compose/Everything/x86_64/os' % self.topdir, 'https://example.com/extra-repo1.repo', - 'https://example.com/extra-repo2.repo' + 'https://example.com/extra-repo2.repo', + 'http://example.com/work/$basearch/repo', ] self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, @@ -247,8 +255,8 @@ class OstreeThreadTest(helpers.PungiTestCase): cfg = { 'release': '20160321.n.0', 'repo': [ - 'Everything', - 'Server', + 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo + 'Server', # this variant-type repo is deprecated, in result will be replaced with default repo 'https://example.com/extra-repo1.repo', 'https://example.com/extra-repo2.repo', ], @@ -265,10 +273,9 @@ class OstreeThreadTest(helpers.PungiTestCase): t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) sources = [ - 'file://%s/compose/Everything/x86_64/os' % self.topdir, - 'file://%s/compose/Server/x86_64/os' % self.topdir, 'https://example.com/extra-repo1.repo', - 'https://example.com/extra-repo2.repo' + 'https://example.com/extra-repo2.repo', + 'http://example.com/work/$basearch/repo', ] self.assertRunrootCall(koji, sources, cfg['release'], isfinal=True, @@ -319,7 +326,7 @@ class OstreeThreadTest(helpers.PungiTestCase): get_dir_from_scm): pool = mock.Mock() cfg = { - 'repo': 'Everything', + 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo 'release': '20160321.n.0', 'add_template': ['some_file.txt'], 'add_arch_template': ['other_file.txt'], @@ -346,7 +353,7 @@ class OstreeThreadTest(helpers.PungiTestCase): 'branch': 'f24', 'dir': '.'}, templ_dir, logger=pool._logger)]) self.assertRunrootCall(koji, - 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'http://example.com/work/$basearch/repo', cfg['release'], isfinal=True, extra=['--add-template=%s/some_file.txt' % templ_dir, @@ -367,7 +374,7 @@ class OstreeThreadTest(helpers.PungiTestCase): get_file_size, get_mtime, ImageCls, run): pool = mock.Mock() cfg = { - 'repo': 'Everything', + 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo 'release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', "installpkgs": ["fedora-productimg-atomic"], "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], @@ -399,7 +406,7 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertRunrootCall( koji, - 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'http://example.com/work/$basearch/repo', '20151203.t.0', isfinal=True, extra=['--installpkgs=fedora-productimg-atomic', @@ -428,7 +435,7 @@ class OstreeThreadTest(helpers.PungiTestCase): get_file_size, get_mtime, ImageCls, run): pool = mock.Mock() cfg = { - 'repo': 'Everything', + 'repo': 'Everything', # this variant-type repo is deprecated, in result will be replaced with default repo 'release': None, "installpkgs": ["fedora-productimg-atomic"], "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], @@ -460,7 +467,7 @@ class OstreeThreadTest(helpers.PungiTestCase): self.assertRunrootCall( koji, - 'file://%s/compose/Everything/x86_64/os' % self.topdir, + 'http://example.com/work/$basearch/repo', '20151203.t.0', isfinal=True, extra=['--installpkgs=fedora-productimg-atomic',