-     This format will be used for all phases generating images. Currently that

-     means ``createiso``, ``live_images`` and ``buildinstall``.

-      * ``live`` -- for images created by *live_images* phase

-         'live': 'Live',

- Common options for Live Images, Live Media and Image Build

- ==========================================================

-  * ``live_images_ksurl``

-  * ``live_images_target``

-  * ``live_images_version``

-  * ``live_images_release``

- Each configuration block can also optionally specify a ``failable`` key. For

- live images it should have a boolean value. For live media and image build it

- Live Images Settings

- ====================

- 

- **live_images**

-     (*list*) -- Configuration for the particular image. The elements of the

-     list should be tuples ``(variant_uid_regex, {arch|*: config})``. The config

-     should be a dict with these keys:

- 

-       * ``kickstart`` (*str*)

-       * ``ksurl`` (*str*) [optional] -- where to get the kickstart from

-       * ``name`` (*str*)

-       * ``version`` (*str*)

-       * ``target`` (*str*)

-       * ``repo`` (*str|[str]*) -- repos specified by URL or variant UID

-       * ``specfile`` (*str*) -- for images wrapped in RPM

-       * ``scratch`` (*bool*) -- only RPM-wrapped images can use scratch builds,

-         but by default this is turned off

-       * ``type`` (*str*) -- what kind of task to start in Koji. Defaults to

-         ``live`` meaning ``koji spin-livecd`` will be used. Alternative option

-         is ``appliance`` corresponding to ``koji spin-appliance``.

-       * ``sign`` (*bool*) -- only RPM-wrapped images can be signed

- 

- **live_images_no_rename**

-     (*bool*) -- When set to ``True``, filenames generated by Koji will be used.

-     When ``False``, filenames will be generated based on ``image_name_format``

-     configuration option.

- 

- 

-     (*str*) -- If set, the ISO files from ``buildinstall``, ``createiso`` and

-     ``live_images`` phases will be put into this destination, and a symlink

-     pointing to this location will be created in actual compose directory.

-     live_target = 'f32'

-     live_images_no_rename = True

-     live_images = [

-         ('^Workstation$', {

-             'armhfp': {

-                 'kickstart': 'fedora-arm-workstation.ks',

-                 'name': 'Fedora-Workstation-armhfp',

-                 # Again workstation takes packages from Everything.

-                 'repo': 'Everything',

-                 'type': 'appliance',

-                 'failable': True,

-             }

-         }),

-         ('^Server$', {

-             # But Server has its own repo.

-             'armhfp': {

-                 'kickstart': 'fedora-arm-server.ks',

-                 'name': 'Fedora-Server-armhfp',

-                 'type': 'appliance',

-                 'failable': True,

-             }

-         }),

-     ]

- 

-             "live_image_config": {

-                 "type": "object",

-                 "properties": {

-                     "kickstart": {"type": "string"},

-                     "ksurl": {"type": "url"},

-                     "name": {"type": "string"},

-                     "subvariant": {"type": "string"},

-                     "target": {"type": "string"},

-                     "version": {"type": "string"},

-                     "repo": {"$ref": "#/definitions/repos"},

-                     "specfile": {"type": "string"},

-                     "scratch": {"type": "boolean"},

-                     "type": {"type": "string"},

-                     "sign": {"type": "boolean"},

-                     "failable": {"type": "boolean"},

-                     "release": {"$ref": "#/definitions/optional_string"},

-                 },

-                 "required": ["kickstart"],

-                 "additionalProperties": False,

-             },

-             "live_images_no_rename": {"type": "boolean", "default": False},

-             "live_images_ksurl": {"type": "url"},

-             "live_images_target": {"type": "string"},

-             "live_images_release": {"$ref": "#/definitions/optional_string"},

-             "live_images_version": {"type": "string"},

-             # Deprecated in favour of regular local/phase/global setting.

-             "live_target": {"type": "string"},

-             "live_images": _variant_arch_mapping(

-                 _one_or_list({"$ref": "#/definitions/live_image_config"})

-             ),

- from .live_images import LiveImagesPhase  # noqa

- # -*- coding: utf-8 -*-

- 

- 

- # This program is free software; you can redistribute it and/or modify

- # it under the terms of the GNU General Public License as published by

- # the Free Software Foundation; version 2 of the License.

- #

- # This program is distributed in the hope that it will be useful,

- # but WITHOUT ANY WARRANTY; without even the implied warranty of

- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

- # GNU Library General Public License for more details.

- #

- # You should have received a copy of the GNU General Public License

- # along with this program; if not, see <https://gnu.org/licenses/>.

- 

- 

- import os

- import sys

- import time

- import shutil

- 

- from kobo.threads import ThreadPool, WorkerThread

- from kobo.shortcuts import run, save_to_file, force_list

- from productmd.images import Image

- from six.moves import shlex_quote

- 

- from pungi.wrappers.kojiwrapper import KojiWrapper

- from pungi.wrappers import iso

- from pungi.phases import base

- from pungi.util import makedirs, get_mtime, get_file_size, failable

- from pungi.util import get_repo_urls

- 

- 

- # HACK: define cmp in python3

- if sys.version_info[0] == 3:

- 

-     def cmp(a, b):

-         return (a > b) - (a < b)

- 

- 

- class LiveImagesPhase(

-     base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase

- ):

-     name = "live_images"

- 

-     def __init__(self, compose):

-         super(LiveImagesPhase, self).__init__(compose)

-         self.pool = ThreadPool(logger=self.logger)

- 

-     def _get_repos(self, arch, variant, data):

-         repos = []

-         if not variant.is_empty:

-             repos.append(variant.uid)

-         repos.extend(force_list(data.get("repo", [])))

-         return get_repo_urls(self.compose, repos, arch=arch)

- 

-     def run(self):

-         symlink_isos_to = self.compose.conf.get("symlink_isos_to")

-         commands = []

- 

-         for variant in self.compose.all_variants.values():

-             for arch in variant.arches + ["src"]:

-                 for data in self.get_config_block(variant, arch):

-                     subvariant = data.get("subvariant", variant.uid)

-                     type = data.get("type", "live")

- 

-                     if type == "live":

-                         dest_dir = self.compose.paths.compose.iso_dir(

-                             arch, variant, symlink_to=symlink_isos_to

-                         )

-                     elif type == "appliance":

-                         dest_dir = self.compose.paths.compose.image_dir(

-                             variant, symlink_to=symlink_isos_to

-                         )

-                         dest_dir = dest_dir % {"arch": arch}

-                         makedirs(dest_dir)

-                     else:

-                         raise RuntimeError("Unknown live image type %s" % type)

-                     if not dest_dir:

-                         continue

- 

-                     cmd = {

-                         "name": data.get("name"),

-                         "version": self.get_version(data),

-                         "release": self.get_release(data),

-                         "dest_dir": dest_dir,

-                         "build_arch": arch,

-                         "ks_file": data["kickstart"],

-                         "ksurl": self.get_ksurl(data),

-                         # Used for images wrapped in RPM

-                         "specfile": data.get("specfile", None),

-                         # Scratch (only taken in consideration if specfile

-                         # specified) For images wrapped in rpm is scratch

-                         # disabled by default For other images is scratch

-                         # always on

-                         "scratch": data.get("scratch", False),

-                         "sign": False,

-                         "type": type,

-                         "label": "",  # currently not used

-                         "subvariant": subvariant,

-                         "failable_arches": data.get("failable", []),

-                         # First see if live_target is specified, then fall back

-                         # to regular setup of local, phase and global setting.

-                         "target": self.compose.conf.get("live_target")

-                         or self.get_config(data, "target"),

-                     }

- 

-                     cmd["repos"] = self._get_repos(arch, variant, data)

- 

-                     # Signing of the rpm wrapped image

-                     if not cmd["scratch"] and data.get("sign"):

-                         cmd["sign"] = True

- 

-                     cmd["filename"] = self._get_file_name(

-                         arch, variant, cmd["name"], cmd["version"]

-                     )

- 

-                     commands.append((cmd, variant, arch))

- 

-         for cmd, variant, arch in commands:

-             self.pool.add(CreateLiveImageThread(self.pool))

-             self.pool.queue_put((self.compose, cmd, variant, arch))

- 

-         self.pool.start()

- 

-     def _get_file_name(self, arch, variant, name=None, version=None):

-         if self.compose.conf["live_images_no_rename"]:

-             return None

- 

-         disc_type = self.compose.conf["disc_types"].get("live", "live")

- 

-         format = (

-             "%(compose_id)s-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"

-         )

-         # Custom name (prefix)

-         if name:

-             custom_iso_name = name

-             if version:

-                 custom_iso_name += "-%s" % version

-             format = (

-                 custom_iso_name

-                 + "-%(variant)s-%(arch)s-%(disc_type)s%(disc_num)s%(suffix)s"

-             )

- 

-         # XXX: hardcoded disc_num

-         return self.compose.get_image_name(

-             arch, variant, disc_type=disc_type, disc_num=None, format=format

-         )

- 

- 

- class CreateLiveImageThread(WorkerThread):

-     EXTS = (".iso", ".raw.xz")

- 

-     def process(self, item, num):

-         compose, cmd, variant, arch = item

-         self.failable_arches = cmd.get("failable_arches", [])

-         self.can_fail = bool(self.failable_arches)

-         with failable(

-             compose,

-             self.can_fail,

-             variant,

-             arch,

-             "live",

-             cmd.get("subvariant"),

-             logger=self.pool._logger,

-         ):

-             self.worker(compose, cmd, variant, arch, num)

- 

-     def worker(self, compose, cmd, variant, arch, num):

-         self.basename = "%(name)s-%(version)s-%(release)s" % cmd

-         log_file = compose.paths.log.log_file(arch, "liveimage-%s" % self.basename)

- 

-         subvariant = cmd.pop("subvariant")

- 

-         imgname = "%s-%s-%s-%s" % (

-             compose.ci_base.release.short,

-             subvariant,

-             "Live" if cmd["type"] == "live" else "Disk",

-             arch,

-         )

- 

-         msg = "Creating ISO (arch: %s, variant: %s): %s" % (

-             arch,

-             variant,

-             self.basename,

-         )

-         self.pool.log_info("[BEGIN] %s" % msg)

- 

-         koji_wrapper = KojiWrapper(compose)

-         _, version = compose.compose_id.rsplit("-", 1)

-         name = cmd["name"] or imgname

-         version = cmd["version"] or version

-         archive = False

-         if cmd["specfile"] and not cmd["scratch"]:

-             # Non scratch build are allowed only for rpm wrapped images

-             archive = True

-         koji_cmd = koji_wrapper.get_create_image_cmd(

-             name,

-             version,

-             cmd["target"],

-             cmd["build_arch"],

-             cmd["ks_file"],

-             cmd["repos"],

-             image_type=cmd["type"],

-             wait=True,

-             archive=archive,

-             specfile=cmd["specfile"],

-             release=cmd["release"],

-             ksurl=cmd["ksurl"],

-         )

- 

-         # avoid race conditions?

-         # Kerberos authentication failed:

-         #   Permission denied in replay cache code (-1765328215)

-         time.sleep(num * 3)

- 

-         output = koji_wrapper.run_blocking_cmd(koji_cmd, log_file=log_file)

-         if output["retcode"] != 0:

-             raise RuntimeError(

-                 "LiveImage task failed: %s. See %s for more details."

-                 % (output["task_id"], log_file)

-             )

- 

-         # copy finished image to isos/

-         image_path = [

-             path

-             for path in koji_wrapper.get_image_path(output["task_id"])

-             if self._is_image(path)

-         ]

-         if len(image_path) != 1:

-             raise RuntimeError(

-                 "Got %d images from task %d, expected 1."

-                 % (len(image_path), output["task_id"])

-             )

-         image_path = compose.koji_downloader.get_file(image_path[0])

-         filename = cmd.get("filename") or os.path.basename(image_path)

-         destination = os.path.join(cmd["dest_dir"], filename)

-         shutil.copy2(image_path, destination)

- 

-         # copy finished rpm to isos/ (if rpm wrapped ISO was built)

-         if cmd["specfile"]:

-             rpm_paths = koji_wrapper.get_wrapped_rpm_path(output["task_id"])

- 

-             if cmd["sign"]:

-                 # Sign the rpm wrapped images and get their paths

-                 self.pool.log_info(

-                     "Signing rpm wrapped images in task_id: %s (expected key ID: %s)"

-                     % (output["task_id"], compose.conf.get("signing_key_id"))

-                 )

-                 signed_rpm_paths = self._sign_image(

-                     koji_wrapper, compose, cmd, output["task_id"]

-                 )

-                 if signed_rpm_paths:

-                     rpm_paths = signed_rpm_paths

- 

-             for rpm_path in rpm_paths:

-                 shutil.copy2(rpm_path, cmd["dest_dir"])

- 

-         if cmd["type"] == "live":

-             # ISO manifest only makes sense for live images

-             self._write_manifest(destination)

- 

-         self._add_to_images(

-             compose,

-             variant,

-             subvariant,

-             arch,

-             cmd["type"],

-             self._get_format(image_path),

-             destination,

-         )

- 

-         self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, output["task_id"]))

- 

-     def _add_to_images(self, compose, variant, subvariant, arch, type, format, path):

-         """Adds the image to images.json"""

-         img = Image(compose.im)

-         img.type = "raw-xz" if type == "appliance" else type

-         img.format = format

-         img.path = os.path.relpath(path, compose.paths.compose.topdir())

-         img.mtime = get_mtime(path)

-         img.size = get_file_size(path)

-         img.arch = arch

-         img.disc_number = 1  # We don't expect multiple disks

-         img.disc_count = 1

-         img.bootable = True

-         img.subvariant = subvariant

-         setattr(img, "can_fail", self.can_fail)

-         setattr(img, "deliverable", "live")

-         compose.im.add(variant=variant.uid, arch=arch, image=img)

- 

-     def _is_image(self, path):

-         for ext in self.EXTS:

-             if path.endswith(ext):

-                 return True

-         return False

- 

-     def _get_format(self, path):

-         """Get format based on extension."""

-         for ext in self.EXTS:

-             if path.endswith(ext):

-                 return ext[1:]

-         raise RuntimeError("Getting format for unknown image %s" % path)

- 

-     def _write_manifest(self, iso_path):

-         """Generate manifest for ISO at given path.

- 

-         :param iso_path: (str) absolute path to the ISO

-         """

-         dir, filename = os.path.split(iso_path)

-         run("cd %s && %s" % (shlex_quote(dir), iso.get_manifest_cmd(filename)))

- 

-     def _sign_image(self, koji_wrapper, compose, cmd, koji_task_id):

-         signing_key_id = compose.conf.get("signing_key_id")

-         signing_command = compose.conf.get("signing_command")

- 

-         if not signing_key_id:

-             self.pool.log_warning(

-                 "Signing is enabled but signing_key_id is not specified"

-             )

-             self.pool.log_warning("Signing skipped")

-             return None

-         if not signing_command:

-             self.pool.log_warning(

-                 "Signing is enabled but signing_command is not specified"

-             )

-             self.pool.log_warning("Signing skipped")

-             return None

- 

-         # Prepare signing log file

-         signing_log_file = compose.paths.log.log_file(

-             cmd["build_arch"], "live_images-signing-%s" % self.basename

-         )

- 

-         # Sign the rpm wrapped images

-         try:

-             sign_builds_in_task(

-                 koji_wrapper,

-                 koji_task_id,

-                 signing_command,

-                 log_file=signing_log_file,

-                 signing_key_password=compose.conf.get("signing_key_password"),

-             )

-         except RuntimeError:

-             self.pool.log_error(

-                 "Error while signing rpm wrapped images. See log: %s" % signing_log_file

-             )

-             raise

- 

-         # Get pats to the signed rpms

-         signing_key_id = signing_key_id.lower()  # Koji uses lowercase in paths

-         rpm_paths = koji_wrapper.get_signed_wrapped_rpms_paths(

-             koji_task_id, signing_key_id

-         )

- 

-         # Wait until files are available

-         if wait_paths(rpm_paths, 60 * 15):

-             # Files are ready

-             return rpm_paths

- 

-         # Signed RPMs are not available

-         self.pool.log_warning("Signed files are not available: %s" % rpm_paths)

-         self.pool.log_warning("Unsigned files will be used")

-         return None

- 

- 

- def wait_paths(paths, timeout=60):

-     started = time.time()

-     remaining = paths[:]

-     while True:

-         for path in remaining[:]:

-             if os.path.exists(path):

-                 remaining.remove(path)

-         if not remaining:

-             break

-         time.sleep(1)

-         if timeout >= 0 and (time.time() - started) > timeout:

-             return False

-     return True

- 

- 

- def sign_builds_in_task(

-     koji_wrapper, task_id, signing_command, log_file=None, signing_key_password=None

- ):

-     # Get list of nvrs that should be signed

-     nvrs = koji_wrapper.get_build_nvrs(task_id)

-     if not nvrs:

-         # No builds are available (scratch build, etc.?)

-         return

- 

-     # Append builds to sign_cmd

-     for nvr in nvrs:

-         signing_command += " '%s'" % nvr

- 

-     # Log signing command before password is filled in it

-     if log_file:

-         save_to_file(log_file, signing_command, append=True)

- 

-     # Fill password into the signing command

-     if signing_key_password:

-         signing_command = signing_command % {

-             "signing_key_password": signing_key_password

-         }

- 

-     # Sign the builds

-     run(signing_command, can_fail=False, show_cmd=False, logfile=log_file)

-         pungi.phases.LiveImagesPhase(compose),

-     liveimages_phase = pungi.phases.LiveImagesPhase(compose)

-     def get_create_image_cmd(

-         self,

-         name,

-         version,

-         target,

-         arch,

-         ks_file,

-         repos,

-         image_type="live",

-         image_format=None,

-         release=None,

-         wait=True,

-         archive=False,

-         specfile=None,

-         ksurl=None,

-     ):

-         # Usage: koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>  # noqa: E501

-         # Usage: koji spin-appliance [options] <name> <version> <target> <arch> <kickstart-file>  # noqa: E501

-         # Examples:

-         #  * name: RHEL-7.0

-         #  * name: Satellite-6.0.1-RHEL-6

-         #  ** -<type>.<arch>

-         #  * version: YYYYMMDD[.n|.t].X

-         #  * release: 1

- 

-         cmd = self._get_cmd()

- 

-         if image_type == "live":

-             cmd.append("spin-livecd")

-         elif image_type == "appliance":

-             cmd.append("spin-appliance")

-         else:

-             raise ValueError("Invalid image type: %s" % image_type)

- 

-         if not archive:

-             cmd.append("--scratch")

- 

-         cmd.append("--noprogress")

- 

-         if wait:

-             cmd.append("--wait")

-         else:

-             cmd.append("--nowait")

- 

-         if specfile:

-             cmd.append("--specfile=%s" % specfile)

- 

-         if ksurl:

-             cmd.append("--ksurl=%s" % ksurl)

- 

-         if isinstance(repos, list):

-             for repo in repos:

-                 cmd.append("--repo=%s" % repo)

-         else:

-             cmd.append("--repo=%s" % repos)

- 

-         if image_format:

-             if image_type != "appliance":

-                 raise ValueError("Format can be specified only for appliance images'")

-             supported_formats = ["raw", "qcow", "qcow2", "vmx"]

-             if image_format not in supported_formats:

-                 raise ValueError(

-                     "Format is not supported: %s. Supported formats: %s"

-                     % (image_format, " ".join(sorted(supported_formats)))

-                 )

-             cmd.append("--format=%s" % image_format)

- 

-         if release is not None:

-             cmd.append("--release=%s" % release)

- 

-         # IMPORTANT: all --opts have to be provided *before* args

-         # Usage:

-         #  koji spin-livecd [options] <name> <version> <target> <arch> <kickstart-file>

- 

-         cmd.append(name)

-         cmd.append(version)

-         cmd.append(target)

- 

-         # i686 -> i386 etc.

-         arch = getBaseArch(arch)

-         cmd.append(arch)

- 

-         cmd.append(ks_file)

- 

-         return cmd

- 

- class LiveImageKojiWrapperTest(KojiWrapperBaseTestCase):

-     def test_get_create_image_cmd_minimal(self):

-         cmd = self.koji.get_create_image_cmd(

-             "my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks", ["/repo/1"]

-         )

-         self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-livecd"])

-         six.assertCountEqual(

-             self, cmd[3:7], ["--noprogress", "--scratch", "--wait", "--repo=/repo/1"]

-         )

-         self.assertEqual(

-             cmd[7:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]

-         )

- 

-     def test_get_create_image_cmd_full(self):

-         cmd = self.koji.get_create_image_cmd(

-             "my_name",

-             "1.0",

-             "f24-candidate",

-             "x86_64",

-             "/path/to/ks",

-             ["/repo/1", "/repo/2"],

-             release="1",

-             wait=False,

-             archive=True,

-             specfile="foo.spec",

-             ksurl="https://git.example.com/",

-         )

-         self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-livecd"])

-         self.assertEqual(

-             cmd[-5:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]

-         )

-         six.assertCountEqual(

-             self,

-             cmd[3:-5],

-             [

-                 "--noprogress",

-                 "--nowait",

-                 "--repo=/repo/1",

-                 "--repo=/repo/2",

-                 "--release=1",

-                 "--specfile=foo.spec",

-                 "--ksurl=https://git.example.com/",

-             ],

-         )

- 

-     def test_spin_livecd_with_format(self):

-         with self.assertRaises(ValueError):

-             self.koji.get_create_image_cmd(

-                 "my_name",

-                 "1.0",

-                 "f24-candidate",

-                 "x86_64",

-                 "/path/to/ks",

-                 [],

-                 image_format="qcow",

-             )

- 

-     def test_spin_appliance_with_format(self):

-         cmd = self.koji.get_create_image_cmd(

-             "my_name",

-             "1.0",

-             "f24-candidate",

-             "x86_64",

-             "/path/to/ks",

-             [],

-             image_type="appliance",

-             image_format="qcow",

-         )

-         self.assertEqual(cmd[0:3], ["koji", "--profile=custom-koji", "spin-appliance"])

-         six.assertCountEqual(

-             self, cmd[3:7], ["--noprogress", "--scratch", "--wait", "--format=qcow"]

-         )

-         self.assertEqual(

-             cmd[7:], ["my_name", "1.0", "f24-candidate", "x86_64", "/path/to/ks"]

-         )

- 

-     def test_spin_appliance_with_wrong_format(self):

-         with self.assertRaises(ValueError):

-             self.koji.get_create_image_cmd(

-                 "my_name",

-                 "1.0",

-                 "f24-candidate",

-                 "x86_64",

-                 "/path/to/ks",

-                 [],

-                 image_type="appliance",

-                 image_format="pretty",

-             )

- 

- 

- # -*- coding: utf-8 -*-

- 

- try:

-     from unittest import mock

- except ImportError:

-     import mock

- 

- import six

- 

- from pungi.phases.live_images import LiveImagesPhase, CreateLiveImageThread

- from tests.helpers import DummyCompose, PungiTestCase, boom

- 

- 

- class TestLiveImagesPhase(PungiTestCase):

-     @mock.patch("pungi.phases.live_images.ThreadPool")

-     def test_live_image_build(self, ThreadPool):

-         compose = DummyCompose(

-             self.topdir,

-             {