| |
@@ -0,0 +1,170 @@
|
| |
+ # -*- coding: utf-8 -*-
|
| |
+
|
| |
+
|
| |
+ import os
|
| |
+ import re
|
| |
+ import shutil
|
| |
+ import time
|
| |
+ import pipes
|
| |
+
|
| |
+ from pungi.util import get_arch_variant_data
|
| |
+ from pungi.phases.base import PhaseBase
|
| |
+ from pungi.paths import translate_path
|
| |
+ from pungi.wrappers.kojiwrapper import KojiWrapper
|
| |
+ from pungi.wrappers.iso import IsoWrapper
|
| |
+ from kobo.shortcuts import run, read_checksum_file
|
| |
+ from kobo.threads import ThreadPool, WorkerThread
|
| |
+ from productmd.images import Image
|
| |
+
|
| |
+ def http_from_nfs(repo):
|
| |
+ return re.sub(r"^/mnt/redhat/", "http://download.lab.bos.redhat.com/", repo)
|
| |
+
|
| |
+ class ImageBuildPhase(PhaseBase):
|
| |
+ """Generic ImageBuild class which is supposed to be inherited"""
|
| |
+ name = "override_me" # docker, qcow2 ...
|
| |
+ config_section = "image_build"
|
| |
+ allowed_suffixes = ["override_me"] # tar.xz, tar.gz ... necessary evil to match brew output filenames
|
| |
+
|
| |
+ def __init__(self, compose):
|
| |
+ PhaseBase.__init__(self, compose)
|
| |
+ self.pool = ThreadPool(logger=self.compose._logger)
|
| |
+
|
| |
+ def skip(self):
|
| |
+ if PhaseBase.skip(self):
|
| |
+ return True
|
| |
+ config = self.compose.conf.get(self.config_section)
|
| |
+ if not config:
|
| |
+ return True
|
| |
+ else:
|
| |
+ #[('^Server$', {'x86_64': [{'name': 'rhel-server-docker', 'format': 'docker', ...}]}),]
|
| |
+ for variant_pattern, data in config:
|
| |
+ for arch, images in data.iteritems():
|
| |
+ for image in images:
|
| |
+ if image['format'] == self.name: # docker, qcow2
|
| |
+ return False # don't skip phase if coressponding image was found
|
| |
+ return True
|
| |
+
|
| |
+ def run(self):
|
| |
+ for arch in self.compose.get_arches(): # src will be skipped
|
| |
+ for variant in self.compose.get_variants(arch=arch):
|
| |
+ image_build_data = get_arch_variant_data(self.compose.conf, self.config_section, arch, variant)
|
| |
+ for image_conf in image_build_data:
|
| |
+ image_conf["arches"] = arch # passed to get_image_build_cmd as dict
|
| |
+ image_conf["variant"] = variant # ^
|
| |
+ image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
|
| |
+ if image_conf.has_key("repos") and not isinstance(image_conf["repos"], str):
|
| |
+ image_conf["repos"] = ",".join(image_conf["repos"]) # supply repos as str separated by , instead of list
|
| |
+ cmd = {
|
| |
+ "image_conf": image_conf,
|
| |
+ "conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf["format"]),
|
| |
+ "image_dir": self.compose.paths.compose.image_dir(arch, variant, image_type=image_conf['format']),
|
| |
+ "relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, image_type=image_conf['format'], create_dir=False, relative=True),
|
| |
+ "allowed_suffixes": self.allowed_suffixes
|
| |
+ }
|
| |
+ self.pool.add(CreateImageBuildThread(self.pool))
|
| |
+ self.pool.queue_put((self.compose, cmd))
|
| |
+ self.pool.start()
|
| |
+
|
| |
+ def stop(self, *args, **kwargs):
|
| |
+ PhaseBase.stop(self, *args, **kwargs)
|
| |
+ if self.skip():
|
| |
+ return
|
| |
+
|
| |
+ class CreateImageBuildThread(WorkerThread):
|
| |
+ def fail(self, compose, cmd):
|
| |
+ compose.log_error("CreateImageBuild failed.")
|
| |
+
|
| |
+ def process(self, item, num):
|
| |
+ compose, cmd = item
|
| |
+ mounts = [compose.topdir]
|
| |
+ if "mount" in cmd:
|
| |
+ mounts.append(cmd["mount"])
|
| |
+ runroot = compose.conf.get("runroot", False)
|
| |
+ log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd["image_conf"]["format"]))
|
| |
+ msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"], cmd["image_conf"]["arches"], cmd["image_conf"]["variant"])
|
| |
+ self.pool.log_info("[BEGIN] %s" % msg)
|
| |
+
|
| |
+ koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
| |
+ # paths module doesn't hold compose object, so we have to generate path here
|
| |
+
|
| |
+ # writes conf file for koji image-build
|
| |
+ self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"]))
|
| |
+ koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False)
|
| |
+
|
| |
+ # avoid race conditions?
|
| |
+ # Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
| |
+ time.sleep(num * 3)
|
| |
+
|
| |
+ output = koji_wrapper.run_create_image_cmd(koji_cmd, log_file=log_file)
|
| |
+ self.pool.log_debug("build-image outputs: %s" % (output))
|
| |
+ if output["retcode"] != 0:
|
| |
+ self.fail(compose, cmd)
|
| |
+ raise RuntimeError("ImageBuild task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
| |
+ # copy image to images/
|
| |
+ image_paths = []
|
| |
+ for filename in koji_wrapper.get_image_path(output["task_id"]):
|
| |
+ for suffix in cmd['allowed_suffixes']:
|
| |
+ if filename.endswith(suffix):
|
| |
+ image_paths.append({'filename' :filename, 'suffix': suffix}) # used to determine format
|
| |
+ if len(image_paths) != 1:
|
| |
+ self.pool.log_error("Expected to find exactly one %s file in output of koji task %s. Got '%s'" % (", ".join(cmd['allowed_suffixes']), output["task_id"], len(image_paths)))
|
| |
+ self.fail(compose, cmd)
|
| |
+
|
| |
+ image_path = image_paths[0]['filename']
|
| |
+ image_suffix = image_paths[0]['suffix']
|
| |
+ # let's not change filename from brew tasks
|
| |
+ image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_path))
|
| |
+ self.pool.log_debug("Copying %s -> %s" % (image_path, image_dest))
|
| |
+ shutil.copy2(image_path, image_dest)
|
| |
+
|
| |
+ iso = IsoWrapper(logger=compose._logger) # required for checksums only
|
| |
+ checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))]
|
| |
+ checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest)))
|
| |
+ checksum_cmd = " && ".join(checksum_cmd)
|
| |
+
|
| |
+ if runroot:
|
| |
+ packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"]
|
| |
+ runroot_channel = compose.conf.get("runroot_channel", None)
|
| |
+ runroot_tag = compose.conf["runroot_tag"]
|
| |
+ koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts)
|
| |
+
|
| |
+ # avoid race conditions?
|
| |
+ # Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
| |
+ time.sleep(num * 3)
|
| |
+
|
| |
+ output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
|
| |
+ if output["retcode"] != 0:
|
| |
+ self.fail(compose, cmd)
|
| |
+ raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file))
|
| |
+
|
| |
+ else:
|
| |
+ # run locally
|
| |
+ try:
|
| |
+ run(checksum_cmd, show_cmd=True, logfile=log_file)
|
| |
+ except:
|
| |
+ self.fail(compose, cmd)
|
| |
+ raise
|
| |
+
|
| |
+ # Update image manifest
|
| |
+ img = Image(compose.im)
|
| |
+ img.type = cmd["image_conf"]["format"] # image/manifest format vs productmd/format
|
| |
+ img.format = image_suffix
|
| |
+ img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest))
|
| |
+ img.mtime = int(os.stat(image_dest).st_mtime)
|
| |
+ img.size = os.path.getsize(image_dest)
|
| |
+ img.arch = cmd["image_conf"]["arches"]
|
| |
+ img.disc_number = 1 # We don't expect multiple disks
|
| |
+ img.disc_count = 1
|
| |
+ for checksum_type in ("md5", "sha1", "sha256"):
|
| |
+ checksum_path = image_dest + ".%sSUM" % checksum_type.upper()
|
| |
+ checksum_value = None
|
| |
+ if os.path.isfile(checksum_path):
|
| |
+ checksum_value, image_name = read_checksum_file(checksum_path)[0]
|
| |
+ if image_name != os.path.basename(img.path):
|
| |
+ raise ValueError("Image name doesn't match checksum: %s" % checksum_path)
|
| |
+ img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value)
|
| |
+ img.bootable = False
|
| |
+ # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch
|
| |
+ compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img)
|
| |
+
|
| |
+ self.pool.log_info("[DONE ] %s" % msg)
|
| |
