From 98f40f6138658afb94515d7b37cb83edceb72751 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Apr 13 2017 08:17:59 +0000 Subject: [PATCH 1/3] ostree: Autogenerate a version If the value is a particular magic string, we will expand it to a proper value. Signed-off-by: Lubomír Sedlář --- diff --git a/pungi/phases/ostree.py b/pungi/phases/ostree.py index 2abecf7..42a0b8c 100644 --- a/pungi/phases/ostree.py +++ b/pungi/phases/ostree.py @@ -100,7 +100,7 @@ class OSTreeThread(WorkerThread): '--treefile=%s' % os.path.join(config_repo, config['treefile']), ] - version = config.get('version', None) + version = util.version_generator(compose, config.get('version')) if version: cmd.append('--version=%s' % version) diff --git a/pungi/util.py b/pungi/util.py index de6effb..c44059f 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -761,3 +761,12 @@ def get_repo_dicts(compose, repos, arch='$basearch'): repo_dict = get_repo_dict(compose, repo, arch=arch) repo_dicts.append(repo_dict) return repo_dicts + + +def version_generator(compose, gen): + """If ``gen`` is a known generator, create a value. Otherwise return + the argument value unchanged. + """ + if gen == '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN': + return '%s.%s' % (compose.image_version, compose.image_release) + return gen diff --git a/tests/test_config.py b/tests/test_config.py index 769cb8d..2d9af3f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -274,7 +274,8 @@ class OstreeConfigTestCase(ConfigTestCase): "treefile": "fedora-atomic-docker-host.json", "config_url": "https://git.fedorahosted.org/git/fedora-atomic.git", "repo": "Everything", - "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/" + "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", + "version": '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN', } }) ] diff --git a/tests/test_ostree_phase.py b/tests/test_ostree_phase.py index 15ebf39..3361353 100644 --- a/tests/test_ostree_phase.py +++ b/tests/test_ostree_phase.py @@ -298,6 +298,41 @@ class OSTreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_generated_versioning_metadata(self, KojiWrapper, get_dir_from_scm): + self.cfg['version'] = '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN' + + get_dir_from_scm.side_effect = self._dummy_config_repo + + koji = KojiWrapper.return_value + koji.run_runroot_cmd.side_effect = self._mock_runroot(0) + + t = ostree.OSTreeThread(self.pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1) + + self.assertEqual(get_dir_from_scm.call_args_list, + [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git', + 'branch': 'f24', 'dir': '.'}, + self.topdir + '/work/ostree-1/config_repo', logger=self.pool._logger)]) + self.assertEqual(koji.get_runroot_cmd.call_args_list, + [mock.call('rrt', 'x86_64', + ['pungi-make-ostree', + 'tree', + '--repo=%s' % self.repo, + '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir, + '--treefile=%s/fedora-atomic-docker-host.json' % ( + self.topdir + '/work/ostree-1/config_repo'), + '--version=25.20151203.t.0', + '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir], + channel=None, mounts=[self.topdir, self.repo], + packages=['pungi', 'ostree', 'rpm-ostree'], + task_id=True, use_shell=True, new_chroot=True, weight=None)]) + self.assertEqual(koji.run_runroot_cmd.call_args_list, + [mock.call(koji.get_runroot_cmd.return_value, + log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')]) + + @mock.patch('pungi.wrappers.scm.get_dir_from_scm') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_write_extra_config_file(self, KojiWrapper, get_dir_from_scm): get_dir_from_scm.side_effect = self._dummy_config_repo From 0f4b6b1947e58852b511fe3fd62ffe3caa661bc0 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Apr 13 2017 08:17:59 +0000 Subject: [PATCH 2/3] docs: Add examples for generated versions Signed-off-by: Lubomír Sedlář --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 7381c22..532f070 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -871,6 +871,31 @@ Example ] +.. _auto-version: + +Automatic generation of version and release +=========================================== + +Version and release values for certain artifacts can be generated automatically +based on release version, compose label, date, type and respin. This can be +used to shorten the config and keep it the same for multiple uses. + ++----------------------------+-------------------+--------------+------------------+ +| Compose ID | Label | Version | Release | ++============================+===================+==============+==================+ +| ``F-Rawhide-20170406.n.0`` | ``-`` | ``Rawhide`` | ``20170406.n.0`` | ++----------------------------+-------------------+--------------+------------------+ +| ``F-26-20170329.1`` | ``Alpha-1.6`` | ``26_Alpha`` | ``1.6`` | ++----------------------------+-------------------+--------------+------------------+ +| ``F-Atomic-25-20170407.0`` | ``RC-20170407.0`` | ``25`` | ``20170407.0`` | ++----------------------------+-------------------+--------------+------------------+ +| ``F-Atomic-25-20170407.0`` | ``-`` | ``25`` | ``20170407.0`` | ++----------------------------+-------------------+--------------+------------------+ + +All non-``RC`` milestones from label get appended to the version. For release +either label is used or date, type and respin. + + Common options for Live Images, Live Media and Image Build ========================================================== @@ -896,22 +921,16 @@ Target is specified by these settings. For live images refer to ``live_target``. * ``live_media_target`` * ``image_build_target`` -.. _auto_version: - Version is specified by these options. If no version is set, a default value -will be provided based on product version. If label is used (and is not -``RC``), the milestone will be appended to the version with an underscore. +will be provided according to :ref:`automatic versioning `. * ``global_version`` -- global fallback setting * ``live_media_version`` * ``image_build_version`` * ``live_images_version`` -.. _auto_release: - Release is specified by these options. If set explicitly to ``None``, a value -will be generated based on compose label, and when compose label is not -provided; date, compose type and respin will be used. +will be generated according to :ref:`automatic versioning `. * ``global_release`` -- global fallback setting * ``live_media_release`` @@ -983,8 +1002,8 @@ Live Media Settings * ``ksversion`` (*str*) * ``scratch`` (*bool*) * ``release`` (*str*) -- a string with the release, or explicit ``None`` - for automatically generating one. See :ref:`common options - ` for details. + for automatically generating one. See :ref:`automatic versioning + ` for details. * ``skip_tag`` (*bool*) * ``repo`` (*str|[str]*) -- repos specified by URL or variant UID * ``title`` (*str*) @@ -1036,7 +1055,7 @@ Image Build Settings currently built variant will be added as well. If you explicitly set ``release`` to ``None``, it will be replaced with - a value generated as described in :ref:`common options `. + a value generated as described in :ref:`automatic versioning `. Please don't set ``install_tree``. This gets automatically set by *pungi* based on current variant. You can use ``install_tree_from`` key to use @@ -1157,6 +1176,9 @@ a new commit. * ``update_summary`` -- (*bool*) Update summary metadata after tree composing. Defaults to ``False``. * ``version`` -- (*str*) Version string to be added as versioning metadata. + If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``, + a value will be generated automatically as ``$VERSION.$RELEASE``. + :ref:`See how those values are created `. * ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git reference will not be created. @@ -1186,7 +1208,8 @@ Example config "keep_original_sources": True, "ostree_repo": "/mnt/koji/compose/atomic/Rawhide/", "update_summary": True, - "version": "24" + # Automatically generate a reasonable version + "version": "!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN", } }) ] @@ -1208,7 +1231,7 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. * ``repo`` -- (*str|[str]*) repos specified by URL or variant UID * ``release`` -- (*str*) Release value to set for the installer image. Set - to ``None`` to generate the value :ref:`automatically `. + to ``None`` to generate the value :ref:`automatically `. * ``failable`` -- (*[str]*) List of architectures for which this deliverable is not release blocking. From f6d07c1651c5eabc40c9757b9076c25fef223120 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Apr 13 2017 08:17:59 +0000 Subject: [PATCH 3/3] Be explicit about generating release for images The config now uses similar logic what previous commit did for OSTree. Also we should report error when an unknown generator is used. Signed-off-by: Lubomír Sedlář --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 532f070..9d1f479 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -929,8 +929,9 @@ will be provided according to :ref:`automatic versioning `. * ``image_build_version`` * ``live_images_version`` -Release is specified by these options. If set explicitly to ``None``, a value -will be generated according to :ref:`automatic versioning `. +Release is specified by these options. If set to a magic value to +``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated according +to :ref:`automatic versioning `. * ``global_release`` -- global fallback setting * ``live_media_release`` @@ -1001,9 +1002,10 @@ Live Media Settings * ``ksurl`` (*str*) * ``ksversion`` (*str*) * ``scratch`` (*bool*) - * ``release`` (*str*) -- a string with the release, or explicit ``None`` - for automatically generating one. See :ref:`automatic versioning - ` for details. + * ``release`` (*str*) -- a string with the release, or + ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN`` to automatically generate a + suitable value. See :ref:`automatic versioning ` for + details. * ``skip_tag`` (*bool*) * ``repo`` (*str|[str]*) -- repos specified by URL or variant UID * ``title`` (*str*) @@ -1013,29 +1015,6 @@ Live Media Settings * ``repo_from`` -- deprecated, use ``repo`` instead -If many of your media use the same value for one of ``ksurl``, ``release``, -``target`` or ``version``, consider using these options to set the value in one -place and have all media inherit it. - -**live_media_ksurl** - (*str*) -- Provides a fallback for media that do not specify ``ksurl`` in - the ``live_media`` block. - -**live_media_release** - (*str*) -- Provides a fallback for media that do not specify ``release`` in - the ``live_media`` block. Please note that if you set this, there is no way - to unset it for a particular media. This is important if you want the - release generated by Koji. - -**live_media_target** - (*str*) -- Provides a fallback for media that do not specify ``target`` in - the ``live_media`` block. - -**live_media_version** - (*str*) -- Provides a fallback for media that do not specify ``version`` in - the ``live_media`` block. - - Image Build Settings ==================== @@ -1054,8 +1033,9 @@ Image Build Settings automatically transformed into format suitable for ``koji``. A repo for the currently built variant will be added as well. - If you explicitly set ``release`` to ``None``, it will be replaced with - a value generated as described in :ref:`automatic versioning `. + If you explicitly set ``release`` to + ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, it will be replaced with a value + generated as described in :ref:`automatic versioning `. Please don't set ``install_tree``. This gets automatically set by *pungi* based on current variant. You can use ``install_tree_from`` key to use @@ -1136,7 +1116,7 @@ Example 'repo': ['Everything'], # Set release automatically. - 'release': None, + 'release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', } } ] @@ -1231,7 +1211,8 @@ an OSTree repository. This always runs in Koji as a ``runroot`` task. * ``repo`` -- (*str|[str]*) repos specified by URL or variant UID * ``release`` -- (*str*) Release value to set for the installer image. Set - to ``None`` to generate the value :ref:`automatically `. + to ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN`` to generate the value + :ref:`automatically `. * ``failable`` -- (*[str]*) List of architectures for which this deliverable is not release blocking. @@ -1268,7 +1249,7 @@ Example config "https://example.com/extra-repo1.repo", "https://example.com/extra-repo2.repo", ], - "release": None, + "release": "!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN", "installpkgs": ["fedora-productimg-atomic"], "add_template": ["atomic-installer/lorax-configure-repo.tmpl"], "add_template_var": [ diff --git a/pungi/phases/base.py b/pungi/phases/base.py index 6c83fa3..bdfd7af 100644 --- a/pungi/phases/base.py +++ b/pungi/phases/base.py @@ -112,15 +112,15 @@ class ImageConfigMixin(object): def get_release(self, cfg): """ - If release is set explicitly to None, replace it with date and respin. - Uses configuration passed as argument, phase specific settings and - global settings. + If release is set to a magic string (or explicitly to None - + deprecated), replace it with a generated value. Uses configuration + passed as argument, phase specific settings and global settings. """ for key, conf in [('release', cfg), ('%s_release' % self.name, self.compose.conf), ('global_release', self.compose.conf)]: if key in conf: - return conf[key] or self.compose.image_release + return util.version_generator(self.compose, conf[key]) or self.compose.image_release return None def get_ksurl(self, cfg): diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 4b03ce9..5dd7ce5 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -6,7 +6,7 @@ import time from kobo import shortcuts from pungi.util import get_variant_data, makedirs, get_mtime, get_file_size, failable -from pungi.util import translate_path, get_repo_urls +from pungi.util import translate_path, get_repo_urls, version_generator from pungi.phases import base from pungi.linker import Linker from pungi.wrappers.kojiwrapper import KojiWrapper @@ -66,8 +66,9 @@ class ImageBuildPhase(base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigG def _set_release(self, image_conf): """If release is set explicitly to None, replace it with date and respin.""" - if 'release' in image_conf and image_conf['release'] is None: - image_conf['release'] = self.compose.image_release + if 'release' in image_conf: + image_conf['release'] = (version_generator(self.compose, image_conf['release']) or + self.compose.image_release) def run(self): for variant in self.compose.get_variants(): diff --git a/pungi/phases/ostree_installer.py b/pungi/phases/ostree_installer.py index 6d5bdea..895827e 100644 --- a/pungi/phases/ostree_installer.py +++ b/pungi/phases/ostree_installer.py @@ -9,7 +9,7 @@ from kobo import shortcuts from .base import ConfigGuardedPhase, PhaseLoggerMixin from .. import util -from ..util import get_volid, get_repo_urls +from ..util import get_volid, get_repo_urls, version_generator from ..wrappers import kojiwrapper, iso, lorax, scm @@ -70,8 +70,8 @@ class OstreeInstallerThread(WorkerThread): self.template_dir, logger=self.pool._logger) def _get_release(self, compose, config): - if 'release' in config and config['release'] is None: - return compose.image_release + if 'release' in config: + return version_generator(compose, config['release']) or compose.image_release return config.get('release', None) def _copy_image(self, compose, variant, arch, filename, output_dir): diff --git a/pungi/util.py b/pungi/util.py index c44059f..43953a5 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -769,4 +769,8 @@ def version_generator(compose, gen): """ if gen == '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN': return '%s.%s' % (compose.image_version, compose.image_release) + if gen == '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN': + return compose.image_release + if gen and gen[0] == '!': + raise RuntimeError("Unknown version generator '%s'" % gen) return gen diff --git a/tests/test_imagebuildphase.py b/tests/test_imagebuildphase.py index d3185e2..07fa15a 100644 --- a/tests/test_imagebuildphase.py +++ b/tests/test_imagebuildphase.py @@ -112,7 +112,7 @@ class TestImageBuildPhase(PungiTestCase): def test_image_build_phase_global_options(self, ThreadPool): compose = DummyCompose(self.topdir, { 'image_build_ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', - 'image_build_release': None, + 'image_build_release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', 'image_build_target': 'f24', 'image_build_version': 'Rawhide', 'image_build': { @@ -171,7 +171,7 @@ class TestImageBuildPhase(PungiTestCase): def test_image_build_phase_missing_version(self, ThreadPool): compose = DummyCompose(self.topdir, { 'image_build_ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', - 'image_build_release': None, + 'image_build_release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', 'image_build_target': 'f24', 'image_build': { '^Server$': [ @@ -482,6 +482,44 @@ class TestImageBuildPhase(PungiTestCase): '20151203.t.0') @mock.patch('pungi.phases.image_build.ThreadPool') + def test_image_build_create_release_with_explicit_config(self, ThreadPool): + compose = DummyCompose(self.topdir, { + 'image_build': { + '^Server$': [ + { + 'image-build': { + 'format': [('docker', 'tar.xz')], + 'name': 'Fedora-Docker-Base', + 'target': 'f24', + 'version': 'Rawhide', + 'ksurl': 'git://git.fedorahosted.org/git/spin-kickstarts.git', + 'kickstart': "fedora-docker-base.ks", + 'distro': 'Fedora-20', + 'disk_size': 3, + 'arches': ['x86_64'], + 'release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', + } + } + ] + }, + 'koji_profile': 'koji', + }) + + self.assertValidConfig(compose.conf) + + phase = ImageBuildPhase(compose) + + phase.run() + + # assert at least one thread was started + self.assertTrue(phase.pool.add.called) + + self.assertTrue(phase.pool.queue_put.called_once) + args, kwargs = phase.pool.queue_put.call_args + self.assertEqual(args[0][1].get('image_conf', {}).get('image-build', {}).get('release'), + '20151203.t.0') + + @mock.patch('pungi.phases.image_build.ThreadPool') def test_image_build_scratch_build(self, ThreadPool): compose = DummyCompose(self.topdir, { 'image_build': { diff --git a/tests/test_ostree_installer_phase.py b/tests/test_ostree_installer_phase.py index 73eab8f..b0daeee 100644 --- a/tests/test_ostree_installer_phase.py +++ b/tests/test_ostree_installer_phase.py @@ -362,6 +362,67 @@ class OstreeThreadTest(helpers.PungiTestCase): @mock.patch('pungi.phases.ostree_installer.iso') @mock.patch('os.link') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') + def test_run_with_explicitly_generated_release(self, KojiWrapper, link, iso, + get_file_size, get_mtime, ImageCls, run): + pool = mock.Mock() + cfg = { + 'repo': 'Everything', + 'release': '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN', + "installpkgs": ["fedora-productimg-atomic"], + "add_template": ["/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl"], + "add_template_var": [ + "ostree_osname=fedora-atomic", + "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", + ], + "add_arch_template": ["/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl"], + "add_arch_template_var": [ + "ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/", + "ostree_osname=fedora-atomic", + "ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host", + ], + } + self.compose.conf['runroot_weights'] = {'ostree_installer': 123} + koji = KojiWrapper.return_value + koji.run_runroot_cmd.return_value = { + 'task_id': 1234, + 'retcode': 0, + 'output': 'Foo bar\n', + } + get_file_size.return_value = 1024 + get_mtime.return_value = 13579 + final_iso_path = self.topdir + '/compose/Everything/x86_64/iso/image-name' + + t = ostree.OstreeInstallerThread(pool) + + t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1) + + self.assertRunrootCall( + koji, + 'file://%s/compose/Everything/x86_64/os' % self.topdir, + '20151203.t.0', + isfinal=True, + extra=['--installpkgs=fedora-productimg-atomic', + '--add-template=/spin-kickstarts/atomic-installer/lorax-configure-repo.tmpl', + '--add-arch-template=/spin-kickstarts/atomic-installer/lorax-embed-repo.tmpl', + '--add-template-var=ostree_osname=fedora-atomic', + '--add-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host', + '--add-arch-template-var=ostree_repo=https://kojipkgs.fedoraproject.org/compose/atomic/Rawhide/', + '--add-arch-template-var=ostree_osname=fedora-atomic', + '--add-arch-template-var=ostree_ref=fedora-atomic/Rawhide/x86_64/docker-host', + '--logfile=%s/%s/lorax.log' % (self.topdir, LOG_PATH)], + weight=123, + ) + self.assertIsoLinked(link, get_file_size, get_mtime, final_iso_path) + self.assertImageAdded(self.compose, ImageCls, iso) + self.assertAllCopied(run) + + @mock.patch('kobo.shortcuts.run') + @mock.patch('productmd.images.Image') + @mock.patch('pungi.util.get_mtime') + @mock.patch('pungi.util.get_file_size') + @mock.patch('pungi.phases.ostree_installer.iso') + @mock.patch('os.link') + @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') def test_run_with_implicit_release(self, KojiWrapper, link, iso, get_file_size, get_mtime, ImageCls, run): pool = mock.Mock() diff --git a/tests/test_util.py b/tests/test_util.py index a85c6a3..d6c5d0c 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -642,5 +642,23 @@ class GetRepoFuncsTestCase(unittest.TestCase): self.assertEqual(repos, expect) +class TestVersionGenerator(unittest.TestCase): + def test_unknown_generator(self): + compose = mock.Mock() + with self.assertRaises(RuntimeError) as ctx: + util.version_generator(compose, '!GIMME_VERSION') + + self.assertEqual(str(ctx.exception), + "Unknown version generator '!GIMME_VERSION'") + + def test_passthrough_value(self): + compose = mock.Mock() + self.assertEqual(util.version_generator(compose, '1.2.3'), '1.2.3') + + def test_passthrough_none(self): + compose = mock.Mock() + self.assertEqual(util.version_generator(compose, None), None) + + if __name__ == "__main__": unittest.main()