| |
@@ -1,10 +1,10 @@
|
| |
# -*- coding: utf-8 -*-
|
| |
|
| |
-
|
| |
+ import copy
|
| |
import os
|
| |
import time
|
| |
|
| |
- from pungi.util import get_arch_variant_data, resolve_git_url
|
| |
+ from pungi.util import get_variant_data, resolve_git_url
|
| |
from pungi.phases.base import PhaseBase
|
| |
from pungi.linker import Linker
|
| |
from pungi.paths import translate_path
|
| |
@@ -30,41 +30,65 @@
|
| |
return False
|
| |
|
| |
def run(self):
|
| |
- for arch in self.compose.get_arches(): # src will be skipped
|
| |
- for variant in self.compose.get_variants(arch=arch):
|
| |
- image_build_data = get_arch_variant_data(self.compose.conf, self.name, arch, variant)
|
| |
- for image_conf in image_build_data:
|
| |
- # Replace possible ambiguous ref name with explicit hash.
|
| |
- if 'ksurl' in image_conf:
|
| |
- image_conf['ksurl'] = resolve_git_url(image_conf['ksurl'])
|
| |
- image_conf["arches"] = arch # passed to get_image_build_cmd as dict
|
| |
- image_conf["variant"] = variant # ^
|
| |
- image_conf["install_tree"] = translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)) # ^
|
| |
- format = image_conf["format"] # transform format into right 'format' for image-build
|
| |
- image_conf["format"] = ",".join([x[0] for x in image_conf["format"]]) # 'docker,qcow2'
|
| |
-
|
| |
- repos = image_conf.get('repos', [])
|
| |
- if isinstance(repos, str):
|
| |
- repos = [repos]
|
| |
- repos.append(translate_path(self.compose, self.compose.paths.compose.os_tree(arch, variant)))
|
| |
- image_conf['repos'] = ",".join(repos) # supply repos as str separated by , instead of list
|
| |
-
|
| |
- cmd = {
|
| |
- "format": format,
|
| |
- "image_conf": image_conf,
|
| |
- "conf_file": self.compose.paths.work.image_build_conf(image_conf["arches"], image_conf['variant'], image_name=image_conf['name'], image_type=image_conf['format'].replace(",", "-")),
|
| |
- "image_dir": self.compose.paths.compose.image_dir(arch, variant),
|
| |
- "relative_image_dir": self.compose.paths.compose.image_dir(arch, variant, create_dir=False, relative=True),
|
| |
- "link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
|
| |
- }
|
| |
- self.pool.add(CreateImageBuildThread(self.pool))
|
| |
- self.pool.queue_put((self.compose, cmd))
|
| |
+ for variant in self.compose.get_variants():
|
| |
+ arches = set([x for x in variant.arches if x != 'src'])
|
| |
+
|
| |
+ for image_conf in get_variant_data(self.compose.conf, self.name, variant):
|
| |
+ # We will modify the data, so we need to make a copy to
|
| |
+ # prevent problems in next iteration where the original
|
| |
+ # value is needed.
|
| |
+ image_conf = copy.deepcopy(image_conf)
|
| |
+
|
| |
+ # Replace possible ambiguous ref name with explicit hash.
|
| |
+ if 'ksurl' in image_conf:
|
| |
+ image_conf['ksurl'] = resolve_git_url(image_conf['ksurl'])
|
| |
+
|
| |
+ # image_conf is passed to get_image_build_cmd as dict
|
| |
+
|
| |
+ if 'arches' in image_conf:
|
| |
+ image_conf["arches"] = ','.join(sorted(set(image_conf.get('arches', [])) & arches))
|
| |
+ else:
|
| |
+ image_conf['arches'] = ','.join(sorted(arches))
|
| |
+
|
| |
+ if not image_conf['arches']:
|
| |
+ continue
|
| |
+
|
| |
+ image_conf["variant"] = variant
|
| |
+ image_conf["install_tree"] = translate_path(
|
| |
+ self.compose,
|
| |
+ self.compose.paths.compose.os_tree('$arch', variant)
|
| |
+ )
|
| |
+ # transform format into right 'format' for image-build
|
| |
+ # e.g. 'docker,qcow2'
|
| |
+ format = image_conf["format"]
|
| |
+ image_conf["format"] = ",".join([x[0] for x in image_conf["format"]])
|
| |
+
|
| |
+ repo = image_conf.get('repo', [])
|
| |
+ if isinstance(repo, str):
|
| |
+ repo = [repo]
|
| |
+ repo.append(translate_path(self.compose, self.compose.paths.compose.os_tree('$arch', variant)))
|
| |
+ # supply repo as str separated by , instead of list
|
| |
+ image_conf['repo'] = ",".join(repo)
|
| |
+
|
| |
+ cmd = {
|
| |
+ "format": format,
|
| |
+ "image_conf": image_conf,
|
| |
+ "conf_file": self.compose.paths.work.image_build_conf(
|
| |
+ image_conf['variant'],
|
| |
+ image_name=image_conf['name'],
|
| |
+ image_type=image_conf['format'].replace(",", "-")
|
| |
+ ),
|
| |
+ "image_dir": self.compose.paths.compose.image_dir(variant),
|
| |
+ "relative_image_dir": self.compose.paths.compose.image_dir(
|
| |
+ variant, create_dir=False, relative=True
|
| |
+ ),
|
| |
+ "link_type": self.compose.conf.get("link_type", "hardlink-or-copy")
|
| |
+ }
|
| |
+ self.pool.add(CreateImageBuildThread(self.pool))
|
| |
+ self.pool.queue_put((self.compose, cmd))
|
| |
+
|
| |
self.pool.start()
|
| |
|
| |
- def stop(self, *args, **kwargs):
|
| |
- PhaseBase.stop(self, *args, **kwargs)
|
| |
- if self.skip():
|
| |
- return
|
| |
|
| |
class CreateImageBuildThread(WorkerThread):
|
| |
def fail(self, compose, cmd):
|
| |
@@ -72,19 +96,29 @@
|
| |
|
| |
def process(self, item, num):
|
| |
compose, cmd = item
|
| |
+ arches = cmd['image_conf']['arches'].split(',')
|
| |
+
|
| |
mounts = [compose.paths.compose.topdir()]
|
| |
if "mount" in cmd:
|
| |
mounts.append(cmd["mount"])
|
| |
- log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-")))
|
| |
- msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"])
|
| |
+ log_file = compose.paths.log.log_file(
|
| |
+ cmd["image_conf"]["arches"],
|
| |
+ "imagebuild-%s-%s-%s" % ('-'.join(arches),
|
| |
+ cmd["image_conf"]["variant"],
|
| |
+ cmd['image_conf']['format'].replace(",", "-"))
|
| |
+ )
|
| |
+ msg = "Creating %s image (arches: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",", "-"),
|
| |
+ '-'.join(arches),
|
| |
+ cmd["image_conf"]["variant"])
|
| |
self.pool.log_info("[BEGIN] %s" % msg)
|
| |
|
| |
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
| |
- # paths module doesn't hold compose object, so we have to generate path here
|
| |
|
| |
# writes conf file for koji image-build
|
| |
- self.pool.log_info("Writing image-build config for %s.%s into %s" % (cmd["image_conf"]["variant"], cmd["image_conf"]["arches"], cmd["conf_file"]))
|
| |
- koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'], conf_file_dest=cmd["conf_file"], wait=True, scratch=False)
|
| |
+ self.pool.log_info("Writing image-build config for %s.%s into %s" % (
|
| |
+ cmd["image_conf"]["variant"], '-'.join(arches), cmd["conf_file"]))
|
| |
+ koji_cmd = koji_wrapper.get_image_build_cmd(cmd['image_conf'],
|
| |
+ conf_file_dest=cmd["conf_file"])
|
| |
|
| |
# avoid race conditions?
|
| |
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
| |
@@ -98,15 +132,22 @@
|
| |
# copy image to images/
|
| |
image_infos = []
|
| |
|
| |
- for filename in koji_wrapper.get_image_path(output["task_id"]):
|
| |
- # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
|
| |
- for format, suffix in cmd['format']:
|
| |
- if filename.endswith(suffix):
|
| |
- image_infos.append({'filename': filename, 'suffix': suffix, 'type': format}) # the type/format ... image-build has it wrong
|
| |
-
|
| |
- if len(image_infos) != len(cmd['format']):
|
| |
- self.pool.log_error("Error in koji task %s. Expected to find same amount of images as in suffixes attr in image-build (%s). Got '%s'." %
|
| |
- (output["task_id"], len(cmd['image_conf']['format']), len(image_infos)))
|
| |
+ paths = koji_wrapper.get_image_build_paths(output["task_id"])
|
| |
+
|
| |
+ for arch, paths in paths.iteritems():
|
| |
+ for path in paths:
|
| |
+ # format is list of tuples [('qcow2', '.qcow2'), ('raw-xz', 'raw.xz'),]
|
| |
+ for format, suffix in cmd['format']:
|
| |
+ if path.endswith(suffix):
|
| |
+ image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch})
|
| |
+ break
|
| |
+
|
| |
+ if len(image_infos) != len(cmd['format']) * len(arches):
|
| |
+ self.pool.log_error(
|
| |
+ "Error in koji task %s. Expected to find same amount of images "
|
| |
+ "as in suffixes attr in image-build (%s) for each arch (%s). Got '%s'." %
|
| |
+ (output["task_id"], len(cmd['format']),
|
| |
+ len(arches), len(image_infos)))
|
| |
self.fail(compose, cmd)
|
| |
|
| |
# The usecase here is that you can run koji image-build with multiple --format
|
| |
@@ -114,22 +155,26 @@
|
| |
# image_build record
|
| |
linker = Linker(logger=compose._logger)
|
| |
for image_info in image_infos:
|
| |
+ image_dir = cmd["image_dir"] % {"arch": image_info['arch']}
|
| |
+ relative_image_dir = cmd["relative_image_dir"] % {"arch": image_info['arch']}
|
| |
+
|
| |
# let's not change filename of koji outputs
|
| |
- image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename']))
|
| |
- linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"])
|
| |
+ image_dest = os.path.join(image_dir, os.path.basename(image_info['path']))
|
| |
+ linker.link(image_info['path'], image_dest, link_type=cmd["link_type"])
|
| |
|
| |
# Update image manifest
|
| |
img = Image(compose.im)
|
| |
img.type = image_info['type']
|
| |
img.format = image_info['suffix']
|
| |
- img.path = os.path.join(cmd["relative_image_dir"], os.path.basename(image_dest))
|
| |
+ img.path = os.path.join(relative_image_dir, os.path.basename(image_dest))
|
| |
img.mtime = int(os.stat(image_dest).st_mtime)
|
| |
img.size = os.path.getsize(image_dest)
|
| |
- img.arch = cmd["image_conf"]["arches"] # arches should be always single arch
|
| |
- img.disc_number = 1 # We don't expect multiple disks
|
| |
+ img.arch = image_info['arch']
|
| |
+ img.disc_number = 1 # We don't expect multiple disks
|
| |
img.disc_count = 1
|
| |
img.bootable = False
|
| |
- # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch
|
| |
- compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img)
|
| |
+ compose.im.add(variant=cmd["image_conf"]["variant"].uid,
|
| |
+ arch=image_info['arch'],
|
| |
+ image=img)
|
| |
|
| |
self.pool.log_info("[DONE ] %s" % msg)
|
| |
