From a45f4969f35827764799c1f20f43adb15a69fc07 Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Nov 04 2020 12:04:43 +0000 Subject: Add phase for building images with osbuild This is similar to image-build in terms of what it does, and somewhat similar to OSBS phase in how it's implemented. The phase reads configuration, submits the build via XMLRPC call and waits for the task to finish. Then it downloads the built image and includes it in the compose metadata. JIRA: RHELCMP-315 Signed-off-by: Lubomír Sedlář --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 73af4fb..e89dc6e 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -1305,6 +1305,7 @@ Target is specified by these settings. * ``live_media_target`` * ``image_build_target`` * ``live_images_target`` + * ``osbuild_target`` Version is specified by these options. If no version is set, a default value will be provided according to :ref:`automatic versioning `. @@ -1313,6 +1314,7 @@ will be provided according to :ref:`automatic versioning `. * ``live_media_version`` * ``image_build_version`` * ``live_images_version`` + * ``osbuild_version`` Release is specified by these options. If set to a magic value to ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated according @@ -1322,6 +1324,7 @@ to :ref:`automatic versioning `. * ``live_media_release`` * ``image_build_release`` * ``live_images_release`` + * ``osbuild_release`` Each configuration block can also optionally specify a ``failable`` key. For live images it should have a boolean value. For live media and image build it @@ -1512,6 +1515,45 @@ Example } +OSBuild Composer for building images +==================================== + +**osbuild** + (*dict*) -- configuration for building images in OSBuild Composer service + fronted by a Koji plugin. Pungi will trigger a Koji task delegating to the + OSBuild Composer, which will build the image, import it to Koji via content + generators. + + Format: ``{variant_uid_regex: [{...}]}``. + + Required keys in the configuration dict: + + * ``name`` -- name of the Koji package + * ``distro`` -- image for which distribution should be build TODO examples + * ``image_type`` -- a list of image types to build (e.g. ``qcow2``) + + Optional keys: + + * ``target`` -- which build target to use for the task. Either this option + or the global ``osbuild_target`` is required. + * ``version`` -- version for the final build (as a string). This option is + required if the global ``osbuild_version`` is not specified. + * ``release`` -- release part of the final NVR. If neither this option nor + the global ``osbuild_release`` is set, Koji will automatically generate a + value. + * ``repo`` -- a list of repository URLs from which to consume packages for + building the image. By default only the variant repository is used. + * ``arches`` -- list of architectures for which to build the image. By + default, the variant arches are used. This option can only restrict it, + not add a new one. + +.. note:: + There is initial support for having this task as failable without aborting + the whole compose. This can be enabled by setting ``"failable": ["*"]`` in + the config for the image. It is an on/off switch without granularity per + arch. + + OSTree Settings =============== diff --git a/pungi/checks.py b/pungi/checks.py index 6f190c8..9aec2c4 100644 --- a/pungi/checks.py +++ b/pungi/checks.py @@ -1111,6 +1111,37 @@ def make_schema(): }, "additionalProperties": False, }, + "osbuild_target": {"type": "string"}, + "osbuild_release": {"$ref": "#/definitions/optional_string"}, + "osbuild_version": {"type": "string"}, + "osbuild": { + "type": "object", + "patternProperties": { + # Warning: this pattern is a variant uid regex, but the + # format does not let us validate it as there is no regular + # expression to describe all regular expressions. + ".+": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "version": {"type": "string"}, + "distro": {"type": "string"}, + "target": {"type": "string"}, + "image_type": {"$ref": "#/definitions/strings"}, + "arches": {"$ref": "#/definitions/list_of_strings"}, + "release": {"type": "string"}, + "repo": {"$ref": "#/definitions/list_of_strings"}, + "failable": {"$ref": "#/definitions/list_of_strings"}, + }, + "subvariant": {"type": "string"}, + }, + "required": ["name", "distro", "image_type"], + "additionalProperties": False, + }, + }, + }, "lorax_options": _variant_arch_mapping( { "type": "object", diff --git a/pungi/phases/__init__.py b/pungi/phases/__init__.py index d99f878..7b28e4e 100644 --- a/pungi/phases/__init__.py +++ b/pungi/phases/__init__.py @@ -27,6 +27,7 @@ from .createiso import CreateisoPhase # noqa from .extra_isos import ExtraIsosPhase # noqa from .live_images import LiveImagesPhase # noqa from .image_build import ImageBuildPhase # noqa +from .osbuild import OSBuildPhase # noqa from .repoclosure import RepoclosurePhase # noqa from .test import TestPhase # noqa from .image_checksum import ImageChecksumPhase # noqa diff --git a/pungi/phases/osbuild.py b/pungi/phases/osbuild.py new file mode 100644 index 0000000..ddd7053 --- /dev/null +++ b/pungi/phases/osbuild.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +import os +import re +from kobo.threads import ThreadPool, WorkerThread +from kobo import shortcuts +from productmd.images import Image + +from . import base +from .. import util +from ..linker import Linker +from ..wrappers import kojiwrapper +from .image_build import EXTENSIONS + + +class OSBuildPhase( + base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase +): + name = "osbuild" + + def __init__(self, compose): + super(OSBuildPhase, self).__init__(compose) + self.pool = ThreadPool(logger=self.logger) + + def _get_arches(self, image_conf, arches): + """Get an intersection of arches in the config dict and the given ones.""" + if "arches" in image_conf: + arches = set(image_conf["arches"]) & arches + return sorted(arches) + + def _get_repo(self, image_conf, variant): + """ + Get a list of repos. First included are those explicitly listed in + config, followed by by repo for current variant if it's not included in + the list already. + """ + repos = shortcuts.force_list(image_conf.get("repo", [])) + + if not variant.is_empty and variant.uid not in repos: + repos.append(variant.uid) + + return util.get_repo_urls(self.compose, repos, arch="$arch") + + def run(self): + for variant in self.compose.get_variants(): + arches = set([x for x in variant.arches if x != "src"]) + + for image_conf in self.get_config_block(variant): + build_arches = self._get_arches(image_conf, arches) + if not build_arches: + self.log_debug("skip: no arches") + continue + + release = self.get_release(image_conf) + version = self.get_version(image_conf) + target = self.get_config(image_conf, "target") + + repo = self._get_repo(image_conf, variant) + + can_fail = image_conf.pop("failable", []) + if can_fail == ["*"]: + can_fail = image_conf["arches"] + if can_fail: + can_fail = sorted(can_fail) + + self.pool.add(RunOSBuildThread(self.pool)) + self.pool.queue_put( + ( + self.compose, + variant, + image_conf, + build_arches, + version, + release, + target, + repo, + can_fail, + ) + ) + + self.pool.start() + + +class RunOSBuildThread(WorkerThread): + def process(self, item, num): + ( + compose, + variant, + config, + arches, + version, + release, + target, + repo, + can_fail, + ) = item + self.can_fail = can_fail + self.num = num + with util.failable( + compose, + bool(config.get("failable")), + variant, + "*", + "osbuild", + logger=self.pool._logger, + ): + self.worker( + compose, variant, config, arches, version, release, target, repo + ) + + def worker(self, compose, variant, config, arches, version, release, target, repo): + msg = "OSBuild task for variant %s" % variant.uid + self.pool.log_info("[BEGIN] %s" % msg) + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + koji.login() + + # Start task + opts = {"repo": repo, "release": release} + task_id = koji.koji_proxy.osbuildImage( + config["name"], + version, + config["distro"], + config["image_types"], + target, + arches, + opts=opts, + ) + + # Wait for it to finish and capture the output into log file. + log_dir = os.path.join(compose.paths.log.topdir(), "osbuild") + util.makedirs(log_dir) + log_file = os.path.join( + log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) + ) + if koji.watch_task(task_id, log_file) != 0: + raise RuntimeError( + "OSBuild: task %s failed: see %s for details" % (task_id, log_file) + ) + + # Parse NVR from the task output. If release part of NVR was generated + # by Koji, we don't have enough information in the configuration. + nvr = get_nvr(log_file) + + # Refresh koji session which may have timed out while the task was + # running. Watching is done via a subprocess, so the session is + # inactive. + koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"]) + + linker = Linker(logger=self.pool._logger) + + # Process all images in the build. There should be one for each + # architecture, but we don't verify that. + build_info = koji.koji_proxy.getBuild(nvr) + for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]): + if archive["type_name"] not in config["image_types"]: + # Ignore values that are not of required types. + continue + + # Get architecture of the image from extra data. + try: + arch = archive["extra"]["image"]["arch"] + except KeyError: + raise RuntimeError("Image doesn't have any architecture!") + + # image_dir is absolute path to which the image should be copied. + # We also need the same path as relative to compose directory for + # including in the metadata. + image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch} + rel_image_dir = compose.paths.compose.image_dir(variant, relative=True) % { + "arch": arch + } + util.makedirs(image_dir) + + image_dest = os.path.join(image_dir, archive["filename"]) + + src_file = os.path.join( + koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"] + ) + + linker.link(src_file, image_dest, link_type=compose.conf["link_type"]) + + suffix = archive["filename"].rsplit(".", 1)[-1] + if suffix not in EXTENSIONS[archive["type_name"]]: + raise RuntimeError( + "Failed to generate metadata. Format %s doesn't match type %s" + % (suffix, archive["type_name"]) + ) + + # Update image manifest + img = Image(compose.im) + img.type = archive["type_name"] + img.format = suffix + img.path = os.path.join(rel_image_dir, archive["filename"]) + img.mtime = util.get_mtime(image_dest) + img.size = util.get_file_size(image_dest) + img.arch = arch + img.disc_number = 1 # We don't expect multiple disks + img.disc_count = 1 + img.bootable = False + img.subvariant = config.get("subvariant", variant.uid) + setattr(img, "can_fail", self.can_fail) + setattr(img, "deliverable", "image-build") + compose.im.add(variant=variant.uid, arch=arch, image=img) + + self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id)) + + +def get_nvr(log_file): + with open(log_file) as f: + for line in f: + match = re.search("Creating compose: ([^ ]+) ", line) + if match: + return match.group(1) + raise RuntimeError("Failed to find image NVR in the output") diff --git a/pungi/scripts/pungi_koji.py b/pungi/scripts/pungi_koji.py index 2ff89e6..6fec647 100644 --- a/pungi/scripts/pungi_koji.py +++ b/pungi/scripts/pungi_koji.py @@ -374,6 +374,7 @@ def run_compose( liveimages_phase = pungi.phases.LiveImagesPhase(compose) livemedia_phase = pungi.phases.LiveMediaPhase(compose) image_build_phase = pungi.phases.ImageBuildPhase(compose) + osbuild_phase = pungi.phases.OSBuildPhase(compose) osbs_phase = pungi.phases.OSBSPhase(compose) image_checksum_phase = pungi.phases.ImageChecksumPhase(compose) repoclosure_phase = pungi.phases.RepoclosurePhase(compose) @@ -397,6 +398,7 @@ def run_compose( ostree_installer_phase, extra_isos_phase, osbs_phase, + osbuild_phase, ): if phase.skip(): continue @@ -494,6 +496,7 @@ def run_compose( liveimages_phase, image_build_phase, livemedia_phase, + osbuild_phase, ) compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema) extra_phase_schema = ( @@ -514,6 +517,7 @@ def run_compose( and liveimages_phase.skip() and livemedia_phase.skip() and image_build_phase.skip() + and osbuild_phase.skip() ): compose.im.dump(compose.paths.compose.metadata("images.json")) osbs_phase.dump_metadata() diff --git a/tests/test_osbuild_phase.py b/tests/test_osbuild_phase.py new file mode 100644 index 0000000..9fd04b4 --- /dev/null +++ b/tests/test_osbuild_phase.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- + +import mock + +import os + +import koji as orig_koji + +from tests import helpers +from pungi.phases import osbuild + + +class OSBuildPhaseTest(helpers.PungiTestCase): + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_run(self, ThreadPool): + cfg = { + "name": "test-image", + "version": "1", + "target": "image-target", + "arches": ["x86_64"], + "failable": ["x86_64"], + } + compose = helpers.DummyCompose( + self.topdir, {"osbuild": {"^Everything$": [cfg]}} + ) + + pool = ThreadPool.return_value + + phase = osbuild.OSBuildPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [ + mock.call( + ( + compose, + compose.variants["Everything"], + cfg, + ["x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + ), + ], + ) + + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_run_with_global_options(self, ThreadPool): + cfg = {"name": "test-image"} + compose = helpers.DummyCompose( + self.topdir, + { + "osbuild": {"^Everything$": [cfg]}, + "osbuild_target": "image-target", + "osbuild_version": "1", + "osbuild_release": "2", + }, + ) + + pool = ThreadPool.return_value + + phase = osbuild.OSBuildPhase(compose) + phase.run() + + self.assertEqual(len(pool.add.call_args_list), 1) + self.assertEqual( + pool.queue_put.call_args_list, + [ + mock.call( + ( + compose, + compose.variants["Everything"], + cfg, + sorted(compose.variants["Everything"].arches), + "1", + "2", + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + [], + ), + ), + ], + ) + + @mock.patch("pungi.phases.osbuild.ThreadPool") + def test_skip_without_config(self, ThreadPool): + compose = helpers.DummyCompose(self.topdir, {}) + compose.just_phases = None + compose.skip_phases = [] + phase = osbuild.OSBuildPhase(compose) + self.assertTrue(phase.skip()) + + +class RunOSBuildThreadTest(helpers.PungiTestCase): + def setUp(self): + super(RunOSBuildThreadTest, self).setUp() + self.pool = mock.Mock() + self.t = osbuild.RunOSBuildThread(self.pool) + self.compose = helpers.DummyCompose( + self.topdir, + { + "koji_profile": "koji", + "translate_paths": [(self.topdir, "http://root")], + }, + ) + + def make_fake_watch(self, retval): + def inner(task_id, log_file): + with open(log_file, "w") as f: + f.write("Creating compose: test-image-1-1 1234\n") + return retval + + return inner + + @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536) + @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024) + @mock.patch("pungi.phases.osbuild.Linker") + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_process(self, KojiWrapper, Linker): + cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]} + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(0) + koji.koji_proxy.osbuildImage.return_value = 1234 + koji.koji_proxy.getBuild.return_value = { + "build_id": 5678, + "name": "test-image", + "version": "1", + "release": "1", + } + koji.koji_proxy.listArchives.return_value = [ + { + "extra": {"image": {"arch": "aarch64"}}, + "filename": "disk.aarch64.qcow2", + "type_name": "qcow2", + }, + { + "extra": {"image": {"arch": "x86_64"}}, + "filename": "disk.x86_64.qcow2", + "type_name": "qcow2", + }, + ] + koji.koji_module.pathinfo = orig_koji.pathinfo + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + # Verify two Koji instances were created. + self.assertEqual(len(KojiWrapper.call_args), 2) + # Verify correct calls to Koji + self.assertEqual( + koji.mock_calls, + [ + mock.call.login(), + mock.call.koji_proxy.osbuildImage( + "test-image", + "1", + "rhel8", + ["qcow2"], + "image-target", + ["aarch64", "x86_64"], + opts={ + "release": None, + "repo": [self.topdir + "/compose/Everything/$arch/os"], + }, + ), + mock.call.watch_task(1234, mock.ANY), + mock.call.koji_proxy.getBuild("test-image-1-1"), + mock.call.koji_proxy.listArchives(buildID=5678), + ], + ) + + # Assert there are 2 images added to manifest and the arguments are sane + self.assertEqual( + self.compose.im.add.call_args_list, + [ + mock.call(arch="aarch64", variant="Everything", image=mock.ANY), + mock.call(arch="x86_64", variant="Everything", image=mock.ANY), + ], + ) + for call in self.compose.im.add.call_args_list: + _, kwargs = call + image = kwargs["image"] + self.assertEqual(kwargs["variant"], "Everything") + self.assertIn(kwargs["arch"], ("aarch64", "x86_64")) + self.assertEqual(kwargs["arch"], image.arch) + self.assertEqual( + "Everything/%(arch)s/images/disk.%(arch)s.qcow2" % {"arch": image.arch}, + image.path, + ) + self.assertEqual("qcow2", image.format) + self.assertEqual("qcow2", image.type) + self.assertEqual("Everything", image.subvariant) + + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/aarch64/images") + ) + self.assertTrue( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + + self.assertEqual( + Linker.return_value.mock_calls, + [ + mock.call.link( + "/mnt/koji/packages/test-image/1/1/images/disk.%(arch)s.qcow2" + % {"arch": arch}, + self.topdir + + "/compose/Everything/%(arch)s/images/disk.%(arch)s.qcow2" + % {"arch": arch}, + link_type="hardlink-or-copy", + ) + for arch in ["aarch64", "x86_64"] + ], + ) + + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_task_fails(self, KojiWrapper): + cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]} + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(1) + koji.koji_proxy.osbuildImage.return_value = 1234 + + with self.assertRaises(RuntimeError): + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper") + def test_task_fails_but_is_failable(self, KojiWrapper): + cfg = { + "name": "test-image", + "distro": "rhel8", + "image_types": ["qcow2"], + "failable": ["x86_65"], + } + koji = KojiWrapper.return_value + koji.watch_task.side_effect = self.make_fake_watch(1) + koji.koji_proxy.osbuildImage.return_value = 1234 + + self.t.process( + ( + self.compose, + self.compose.variants["Everything"], + cfg, + ["aarch64", "x86_64"], + "1", + None, + "image-target", + [self.topdir + "/compose/Everything/$arch/os"], + ["x86_64"], + ), + 1, + ) + + self.assertFalse( + os.path.isdir(self.topdir + "/compose/Everything/aarch64/images") + ) + self.assertFalse( + os.path.isdir(self.topdir + "/compose/Everything/x86_64/images") + ) + self.assertEqual(len(self.compose.im.add.call_args_list), 0)