#1458 Add phase for building images with osbuild
Merged 19 days ago by lsedlar. Opened a month ago by lsedlar.
lsedlar/pungi osbuild  into  master

file modified
+42
@@ -1305,6 +1305,7 @@ 

   * ``live_media_target``

   * ``image_build_target``

   * ``live_images_target``

+  * ``osbuild_target``

  

  Version is specified by these options. If no version is set, a default value

  will be provided according to :ref:`automatic versioning <auto-version>`.
@@ -1313,6 +1314,7 @@ 

   * ``live_media_version``

   * ``image_build_version``

   * ``live_images_version``

+  * ``osbuild_version``

  

  Release is specified by these options. If set to a magic value to

  ``!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN``, a value will be generated according
@@ -1322,6 +1324,7 @@ 

   * ``live_media_release``

   * ``image_build_release``

   * ``live_images_release``

+  * ``osbuild_release``

  

  Each configuration block can also optionally specify a ``failable`` key. For

  live images it should have a boolean value. For live media and image build it
@@ -1512,6 +1515,45 @@ 

      }

  

  

+ OSBuild Composer for building images

+ ====================================

+ 

+ **osbuild**

+     (*dict*) -- configuration for building images in OSBuild Composer service

+     fronted by a Koji plugin. Pungi will trigger a Koji task delegating to the

+     OSBuild Composer, which will build the image, import it to Koji via content

+     generators.

+ 

+     Format: ``{variant_uid_regex: [{...}]}``.

+ 

+     Required keys in the configuration dict:

+ 

+     * ``name`` -- name of the Koji package

+     * ``distro`` -- image for which distribution should be build TODO examples

+     * ``image_type`` -- a list of image types to build (e.g. ``qcow2``)

+ 

+     Optional keys:

+ 

+     * ``target`` -- which build target to use for the task. Either this option

+       or the global ``osbuild_target`` is required.

+     * ``version`` -- version for the final build (as a string). This option is

+       required if the global ``osbuild_version`` is not specified.

+     * ``release`` -- release part of the final NVR. If neither this option nor

+       the global ``osbuild_release`` is set, Koji will automatically generate a

+       value.

+     * ``repo`` -- a list of repository URLs from which to consume packages for

+       building the image. By default only the variant repository is used.

+     * ``arches`` -- list of architectures for which to build the image. By

+       default, the variant arches are used. This option can only restrict it,

+       not add a new one.

+ 

+ .. note::

+    There is initial support for having this task as failable without aborting

+    the whole compose. This can be enabled by setting ``"failable": ["*"]`` in

+    the config for the image. It is an on/off switch without granularity per

+    arch.

+ 

+ 

  OSTree Settings

  ===============

  

file modified
+31
@@ -1111,6 +1111,37 @@ 

                  },

                  "additionalProperties": False,

              },

+             "osbuild_target": {"type": "string"},

+             "osbuild_release": {"$ref": "#/definitions/optional_string"},

+             "osbuild_version": {"type": "string"},

+             "osbuild": {

+                 "type": "object",

+                 "patternProperties": {

+                     # Warning: this pattern is a variant uid regex, but the

+                     # format does not let us validate it as there is no regular

+                     # expression to describe all regular expressions.

+                     ".+": {

+                         "type": "array",

+                         "items": {

+                             "type": "object",

+                             "properties": {

+                                 "name": {"type": "string"},

+                                 "version": {"type": "string"},

+                                 "distro": {"type": "string"},

+                                 "target": {"type": "string"},

+                                 "image_type": {"$ref": "#/definitions/strings"},

+                                 "arches": {"$ref": "#/definitions/list_of_strings"},

+                                 "release": {"type": "string"},

+                                 "repo": {"$ref": "#/definitions/list_of_strings"},

+                                 "failable": {"$ref": "#/definitions/list_of_strings"},

+                             },

+                             "subvariant": {"type": "string"},

+                         },

+                         "required": ["name", "distro", "image_type"],

+                         "additionalProperties": False,

+                     },

+                 },

+             },

              "lorax_options": _variant_arch_mapping(

                  {

                      "type": "object",

@@ -27,6 +27,7 @@ 

  from .extra_isos import ExtraIsosPhase  # noqa

  from .live_images import LiveImagesPhase  # noqa

  from .image_build import ImageBuildPhase  # noqa

+ from .osbuild import OSBuildPhase  # noqa

  from .repoclosure import RepoclosurePhase  # noqa

  from .test import TestPhase  # noqa

  from .image_checksum import ImageChecksumPhase  # noqa

@@ -0,0 +1,214 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ import os

+ import re

+ from kobo.threads import ThreadPool, WorkerThread

+ from kobo import shortcuts

+ from productmd.images import Image

+ 

+ from . import base

+ from .. import util

+ from ..linker import Linker

+ from ..wrappers import kojiwrapper

+ from .image_build import EXTENSIONS

+ 

+ 

+ class OSBuildPhase(

+     base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase

+ ):

+     name = "osbuild"

+ 

+     def __init__(self, compose):

+         super(OSBuildPhase, self).__init__(compose)

+         self.pool = ThreadPool(logger=self.logger)

+ 

+     def _get_arches(self, image_conf, arches):

+         """Get an intersection of arches in the config dict and the given ones."""

+         if "arches" in image_conf:

+             arches = set(image_conf["arches"]) & arches

+         return sorted(arches)

+ 

+     def _get_repo(self, image_conf, variant):

+         """

+         Get a list of repos. First included are those explicitly listed in

+         config, followed by by repo for current variant if it's not included in

+         the list already.

+         """

+         repos = shortcuts.force_list(image_conf.get("repo", []))

+ 

+         if not variant.is_empty and variant.uid not in repos:

+             repos.append(variant.uid)

+ 

+         return util.get_repo_urls(self.compose, repos, arch="$arch")

+ 

+     def run(self):

+         for variant in self.compose.get_variants():

+             arches = set([x for x in variant.arches if x != "src"])

+ 

+             for image_conf in self.get_config_block(variant):

+                 build_arches = self._get_arches(image_conf, arches)

+                 if not build_arches:

+                     self.log_debug("skip: no arches")

+                     continue

+ 

+                 release = self.get_release(image_conf)

+                 version = self.get_version(image_conf)

+                 target = self.get_config(image_conf, "target")

+ 

+                 repo = self._get_repo(image_conf, variant)

+ 

+                 can_fail = image_conf.pop("failable", [])

+                 if can_fail == ["*"]:

+                     can_fail = image_conf["arches"]

+                 if can_fail:

+                     can_fail = sorted(can_fail)

+ 

+                 self.pool.add(RunOSBuildThread(self.pool))

+                 self.pool.queue_put(

+                     (

+                         self.compose,

+                         variant,

+                         image_conf,

+                         build_arches,

+                         version,

+                         release,

+                         target,

+                         repo,

+                         can_fail,

+                     )

+                 )

+ 

+         self.pool.start()

+ 

+ 

+ class RunOSBuildThread(WorkerThread):

+     def process(self, item, num):

+         (

+             compose,

+             variant,

+             config,

+             arches,

+             version,

+             release,

+             target,

+             repo,

+             can_fail,

+         ) = item

+         self.can_fail = can_fail

+         self.num = num

+         with util.failable(

+             compose,

+             bool(config.get("failable")),

+             variant,

+             "*",

+             "osbuild",

+             logger=self.pool._logger,

+         ):

+             self.worker(

+                 compose, variant, config, arches, version, release, target, repo

+             )

+ 

+     def worker(self, compose, variant, config, arches, version, release, target, repo):

+         msg = "OSBuild task for variant %s" % variant.uid

+         self.pool.log_info("[BEGIN] %s" % msg)

+         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+         koji.login()

+ 

+         # Start task

+         opts = {"repo": repo, "release": release}

+         task_id = koji.koji_proxy.osbuildImage(

+             config["name"],

+             version,

+             config["distro"],

+             config["image_types"],

+             target,

+             arches,

+             opts=opts,

+         )

+ 

+         # Wait for it to finish and capture the output into log file.

+         log_dir = os.path.join(compose.paths.log.topdir(), "osbuild")

+         util.makedirs(log_dir)

+         log_file = os.path.join(

+             log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)

+         )

+         if koji.watch_task(task_id, log_file) != 0:

+             raise RuntimeError(

+                 "OSBuild: task %s failed: see %s for details" % (task_id, log_file)

+             )

+ 

+         # Parse NVR from the task output. If release part of NVR was generated

+         # by Koji, we don't have enough information in the configuration.

+         nvr = get_nvr(log_file)

+ 

+         # Refresh koji session which may have timed out while the task was

+         # running. Watching is done via a subprocess, so the session is

+         # inactive.

+         koji = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])

+ 

+         linker = Linker(logger=self.pool._logger)

+ 

+         # Process all images in the build. There should be one for each

+         # architecture, but we don't verify that.

+         build_info = koji.koji_proxy.getBuild(nvr)

+         for archive in koji.koji_proxy.listArchives(buildID=build_info["build_id"]):

+             if archive["type_name"] not in config["image_types"]:

+                 # Ignore values that are not of required types.

+                 continue

+ 

+             # Get architecture of the image from extra data.

+             try:

+                 arch = archive["extra"]["image"]["arch"]

+             except KeyError:

+                 raise RuntimeError("Image doesn't have any architecture!")

+ 

+             # image_dir is absolute path to which the image should be copied.

+             # We also need the same path as relative to compose directory for

+             # including in the metadata.

+             image_dir = compose.paths.compose.image_dir(variant) % {"arch": arch}

+             rel_image_dir = compose.paths.compose.image_dir(variant, relative=True) % {

+                 "arch": arch

+             }

+             util.makedirs(image_dir)

+ 

+             image_dest = os.path.join(image_dir, archive["filename"])

+ 

+             src_file = os.path.join(

+                 koji.koji_module.pathinfo.imagebuild(build_info), archive["filename"]

+             )

+ 

+             linker.link(src_file, image_dest, link_type=compose.conf["link_type"])

+ 

+             suffix = archive["filename"].rsplit(".", 1)[-1]

+             if suffix not in EXTENSIONS[archive["type_name"]]:

+                 raise RuntimeError(

+                     "Failed to generate metadata. Format %s doesn't match type %s"

+                     % (suffix, archive["type_name"])

+                 )

+ 

+             # Update image manifest

+             img = Image(compose.im)

+             img.type = archive["type_name"]

+             img.format = suffix

+             img.path = os.path.join(rel_image_dir, archive["filename"])

+             img.mtime = util.get_mtime(image_dest)

+             img.size = util.get_file_size(image_dest)

+             img.arch = arch

+             img.disc_number = 1  # We don't expect multiple disks

+             img.disc_count = 1

+             img.bootable = False

+             img.subvariant = config.get("subvariant", variant.uid)

+             setattr(img, "can_fail", self.can_fail)

+             setattr(img, "deliverable", "image-build")

+             compose.im.add(variant=variant.uid, arch=arch, image=img)

+ 

+         self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))

+ 

+ 

+ def get_nvr(log_file):

+     with open(log_file) as f:

+         for line in f:

+             match = re.search("Creating compose: ([^ ]+) ", line)

+             if match:

+                 return match.group(1)

+     raise RuntimeError("Failed to find image NVR in the output")

@@ -374,6 +374,7 @@ 

      liveimages_phase = pungi.phases.LiveImagesPhase(compose)

      livemedia_phase = pungi.phases.LiveMediaPhase(compose)

      image_build_phase = pungi.phases.ImageBuildPhase(compose)

+     osbuild_phase = pungi.phases.OSBuildPhase(compose)

      osbs_phase = pungi.phases.OSBSPhase(compose)

      image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)

      repoclosure_phase = pungi.phases.RepoclosurePhase(compose)
@@ -397,6 +398,7 @@ 

          ostree_installer_phase,

          extra_isos_phase,

          osbs_phase,

+         osbuild_phase,

      ):

          if phase.skip():

              continue
@@ -494,6 +496,7 @@ 

          liveimages_phase,

          image_build_phase,

          livemedia_phase,

+         osbuild_phase,

      )

      compose_images_phase = pungi.phases.WeaverPhase(compose, compose_images_schema)

      extra_phase_schema = (
@@ -514,6 +517,7 @@ 

          and liveimages_phase.skip()

          and livemedia_phase.skip()

          and image_build_phase.skip()

+         and osbuild_phase.skip()

      ):

          compose.im.dump(compose.paths.compose.metadata("images.json"))

      osbs_phase.dump_metadata()

@@ -0,0 +1,289 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ import mock

+ 

+ import os

+ 

+ import koji as orig_koji

+ 

+ from tests import helpers

+ from pungi.phases import osbuild

+ 

+ 

+ class OSBuildPhaseTest(helpers.PungiTestCase):

+     @mock.patch("pungi.phases.osbuild.ThreadPool")

+     def test_run(self, ThreadPool):

+         cfg = {

+             "name": "test-image",

+             "version": "1",

+             "target": "image-target",

+             "arches": ["x86_64"],

+             "failable": ["x86_64"],

+         }

+         compose = helpers.DummyCompose(

+             self.topdir, {"osbuild": {"^Everything$": [cfg]}}

+         )

+ 

+         pool = ThreadPool.return_value

+ 

+         phase = osbuild.OSBuildPhase(compose)

+         phase.run()

+ 

+         self.assertEqual(len(pool.add.call_args_list), 1)

+         self.assertEqual(

+             pool.queue_put.call_args_list,

+             [

+                 mock.call(

+                     (

+                         compose,

+                         compose.variants["Everything"],

+                         cfg,

+                         ["x86_64"],

+                         "1",

+                         None,

+                         "image-target",

+                         [self.topdir + "/compose/Everything/$arch/os"],

+                         ["x86_64"],

+                     ),

+                 ),

+             ],

+         )

+ 

+     @mock.patch("pungi.phases.osbuild.ThreadPool")

+     def test_run_with_global_options(self, ThreadPool):

+         cfg = {"name": "test-image"}

+         compose = helpers.DummyCompose(

+             self.topdir,

+             {

+                 "osbuild": {"^Everything$": [cfg]},

+                 "osbuild_target": "image-target",

+                 "osbuild_version": "1",

+                 "osbuild_release": "2",

+             },

+         )

+ 

+         pool = ThreadPool.return_value

+ 

+         phase = osbuild.OSBuildPhase(compose)

+         phase.run()

+ 

+         self.assertEqual(len(pool.add.call_args_list), 1)

+         self.assertEqual(

+             pool.queue_put.call_args_list,

+             [

+                 mock.call(

+                     (

+                         compose,

+                         compose.variants["Everything"],

+                         cfg,

+                         sorted(compose.variants["Everything"].arches),

+                         "1",

+                         "2",

+                         "image-target",

+                         [self.topdir + "/compose/Everything/$arch/os"],

+                         [],

+                     ),

+                 ),

+             ],

+         )

+ 

+     @mock.patch("pungi.phases.osbuild.ThreadPool")

+     def test_skip_without_config(self, ThreadPool):

+         compose = helpers.DummyCompose(self.topdir, {})

+         compose.just_phases = None

+         compose.skip_phases = []

+         phase = osbuild.OSBuildPhase(compose)

+         self.assertTrue(phase.skip())

+ 

+ 

+ class RunOSBuildThreadTest(helpers.PungiTestCase):

+     def setUp(self):

+         super(RunOSBuildThreadTest, self).setUp()

+         self.pool = mock.Mock()

+         self.t = osbuild.RunOSBuildThread(self.pool)

+         self.compose = helpers.DummyCompose(

+             self.topdir,

+             {

+                 "koji_profile": "koji",

+                 "translate_paths": [(self.topdir, "http://root")],

+             },

+         )

+ 

+     def make_fake_watch(self, retval):

+         def inner(task_id, log_file):

+             with open(log_file, "w") as f:

+                 f.write("Creating compose: test-image-1-1 1234\n")

+             return retval

+ 

+         return inner

+ 

+     @mock.patch("pungi.util.get_file_size", new=lambda fp: 65536)

+     @mock.patch("pungi.util.get_mtime", new=lambda fp: 1024)

+     @mock.patch("pungi.phases.osbuild.Linker")

+     @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")

+     def test_process(self, KojiWrapper, Linker):

+         cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]}

+         koji = KojiWrapper.return_value

+         koji.watch_task.side_effect = self.make_fake_watch(0)

+         koji.koji_proxy.osbuildImage.return_value = 1234

+         koji.koji_proxy.getBuild.return_value = {

+             "build_id": 5678,

+             "name": "test-image",

+             "version": "1",

+             "release": "1",

+         }

+         koji.koji_proxy.listArchives.return_value = [

+             {

+                 "extra": {"image": {"arch": "aarch64"}},

+                 "filename": "disk.aarch64.qcow2",

+                 "type_name": "qcow2",

+             },

+             {

+                 "extra": {"image": {"arch": "x86_64"}},

+                 "filename": "disk.x86_64.qcow2",

+                 "type_name": "qcow2",

+             },

+         ]

+         koji.koji_module.pathinfo = orig_koji.pathinfo

+ 

+         self.t.process(

+             (

+                 self.compose,

+                 self.compose.variants["Everything"],

+                 cfg,

+                 ["aarch64", "x86_64"],

+                 "1",

+                 None,

+                 "image-target",

+                 [self.topdir + "/compose/Everything/$arch/os"],

+                 ["x86_64"],

+             ),

+             1,

+         )

+ 

+         # Verify two Koji instances were created.

+         self.assertEqual(len(KojiWrapper.call_args), 2)

+         # Verify correct calls to Koji

+         self.assertEqual(

+             koji.mock_calls,

+             [

+                 mock.call.login(),

+                 mock.call.koji_proxy.osbuildImage(

+                     "test-image",

+                     "1",

+                     "rhel8",

+                     ["qcow2"],

+                     "image-target",

+                     ["aarch64", "x86_64"],

+                     opts={

+                         "release": None,

+                         "repo": [self.topdir + "/compose/Everything/$arch/os"],

+                     },

+                 ),

+                 mock.call.watch_task(1234, mock.ANY),

+                 mock.call.koji_proxy.getBuild("test-image-1-1"),

+                 mock.call.koji_proxy.listArchives(buildID=5678),

+             ],

+         )

+ 

+         # Assert there are 2 images added to manifest and the arguments are sane

+         self.assertEqual(

+             self.compose.im.add.call_args_list,

+             [

+                 mock.call(arch="aarch64", variant="Everything", image=mock.ANY),

+                 mock.call(arch="x86_64", variant="Everything", image=mock.ANY),

+             ],

+         )

+         for call in self.compose.im.add.call_args_list:

+             _, kwargs = call

+             image = kwargs["image"]

+             self.assertEqual(kwargs["variant"], "Everything")

+             self.assertIn(kwargs["arch"], ("aarch64", "x86_64"))

+             self.assertEqual(kwargs["arch"], image.arch)

+             self.assertEqual(

+                 "Everything/%(arch)s/images/disk.%(arch)s.qcow2" % {"arch": image.arch},

+                 image.path,

+             )

+             self.assertEqual("qcow2", image.format)

+             self.assertEqual("qcow2", image.type)

+             self.assertEqual("Everything", image.subvariant)

+ 

+         self.assertTrue(

+             os.path.isdir(self.topdir + "/compose/Everything/aarch64/images")

+         )

+         self.assertTrue(

+             os.path.isdir(self.topdir + "/compose/Everything/x86_64/images")

+         )

+ 

+         self.assertEqual(

+             Linker.return_value.mock_calls,

+             [

+                 mock.call.link(

+                     "/mnt/koji/packages/test-image/1/1/images/disk.%(arch)s.qcow2"

+                     % {"arch": arch},

+                     self.topdir

+                     + "/compose/Everything/%(arch)s/images/disk.%(arch)s.qcow2"

+                     % {"arch": arch},

+                     link_type="hardlink-or-copy",

+                 )

+                 for arch in ["aarch64", "x86_64"]

+             ],

+         )

+ 

+     @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")

+     def test_task_fails(self, KojiWrapper):

+         cfg = {"name": "test-image", "distro": "rhel8", "image_types": ["qcow2"]}

+         koji = KojiWrapper.return_value

+         koji.watch_task.side_effect = self.make_fake_watch(1)

+         koji.koji_proxy.osbuildImage.return_value = 1234

+ 

+         with self.assertRaises(RuntimeError):

+             self.t.process(

+                 (

+                     self.compose,

+                     self.compose.variants["Everything"],

+                     cfg,

+                     ["aarch64", "x86_64"],

+                     "1",

+                     None,

+                     "image-target",

+                     [self.topdir + "/compose/Everything/$arch/os"],

+                     ["x86_64"],

+                 ),

+                 1,

+             )

+ 

+     @mock.patch("pungi.phases.osbuild.kojiwrapper.KojiWrapper")

+     def test_task_fails_but_is_failable(self, KojiWrapper):

+         cfg = {

+             "name": "test-image",

+             "distro": "rhel8",

+             "image_types": ["qcow2"],

+             "failable": ["x86_65"],

+         }

+         koji = KojiWrapper.return_value

+         koji.watch_task.side_effect = self.make_fake_watch(1)

+         koji.koji_proxy.osbuildImage.return_value = 1234

+ 

+         self.t.process(

+             (

+                 self.compose,

+                 self.compose.variants["Everything"],

+                 cfg,

+                 ["aarch64", "x86_64"],

+                 "1",

+                 None,

+                 "image-target",

+                 [self.topdir + "/compose/Everything/$arch/os"],

+                 ["x86_64"],

+             ),

+             1,

+         )

+ 

+         self.assertFalse(

+             os.path.isdir(self.topdir + "/compose/Everything/aarch64/images")

+         )

+         self.assertFalse(

+             os.path.isdir(self.topdir + "/compose/Everything/x86_64/images")

+         )

+         self.assertEqual(len(self.compose.im.add.call_args_list), 0)

There are unit tests and basic documentation. It has not actually been tested against a Koji instance with the required plugin.

rebased onto 9d2ba00a60a0fe82d0a6e81c622d62742bc3b00c

a month ago

rebased onto f73e5b4

19 days ago

rebased onto 97cbd5d

19 days ago

rebased onto a45f496

19 days ago

This patch should not break anything (unless it's configured), so it should be safe to merge. I'll do that to unblock the release.

Pull-Request has been merged by lsedlar

19 days ago