|
|
f116d93 |
# -*- coding: utf-8 -*-
|
|
|
f116d93 |
|
|
|
f116d93 |
|
|
|
f116d93 |
# This program is free software; you can redistribute it and/or modify
|
|
|
f116d93 |
# it under the terms of the GNU General Public License as published by
|
|
|
f116d93 |
# the Free Software Foundation; version 2 of the License.
|
|
|
f116d93 |
#
|
|
|
f116d93 |
# This program is distributed in the hope that it will be useful,
|
|
|
f116d93 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
f116d93 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
f116d93 |
# GNU Library General Public License for more details.
|
|
|
f116d93 |
#
|
|
|
f116d93 |
# You should have received a copy of the GNU General Public License
|
|
|
d6dc269 |
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
f116d93 |
|
|
|
f116d93 |
|
|
|
f116d93 |
__all__ = (
|
|
|
f116d93 |
"Compose",
|
|
|
f116d93 |
)
|
|
|
f116d93 |
|
|
|
f116d93 |
|
|
|
f116d93 |
import errno
|
|
|
f116d93 |
import os
|
|
|
f116d93 |
import time
|
|
|
f116d93 |
import tempfile
|
|
|
f116d93 |
import shutil
|
|
|
18b6020 |
import json
|
|
|
f116d93 |
|
|
|
f116d93 |
import kobo.log
|
|
Adam Miller |
5fa5fcd |
from productmd.composeinfo import ComposeInfo
|
|
|
771ed2e |
from productmd.images import Images
|
|
|
f116d93 |
|
|
|
07e90f0 |
from pungi.wrappers.variants import VariantsXmlParser
|
|
|
07e90f0 |
from pungi.paths import Paths
|
|
|
07e90f0 |
from pungi.wrappers.scm import get_file_from_scm
|
|
|
1ebb9d1 |
from pungi.util import makedirs, get_arch_variant_data, get_format_substs
|
|
|
fd8cd62 |
from pungi.metadata import compose_to_composeinfo
|
|
|
f116d93 |
|
|
|
f116d93 |
|
|
|
f116d93 |
def get_compose_dir(topdir, conf, compose_type="production", compose_date=None, compose_respin=None, compose_label=None, already_exists_callbacks=None):
|
|
|
f116d93 |
already_exists_callbacks = already_exists_callbacks or []
|
|
|
f116d93 |
|
|
|
771ed2e |
# create an incomplete composeinfo to generate compose ID
|
|
Adam Miller |
5fa5fcd |
ci = ComposeInfo()
|
|
|
72302bd |
ci.release.name = conf["release_name"]
|
|
|
72302bd |
ci.release.short = conf["release_short"]
|
|
|
72302bd |
ci.release.version = conf["release_version"]
|
|
|
72302bd |
ci.release.is_layered = bool(conf.get("release_is_layered", False))
|
|
|
ba9df6d |
ci.release.type = conf.get("release_type", "ga").lower()
|
|
|
d3cad47 |
ci.release.internal = bool(conf.get("release_internal", False))
|
|
|
72302bd |
if ci.release.is_layered:
|
|
|
f116d93 |
ci.base_product.name = conf["base_product_name"]
|
|
|
f116d93 |
ci.base_product.short = conf["base_product_short"]
|
|
|
f116d93 |
ci.base_product.version = conf["base_product_version"]
|
|
|
385f0e9 |
ci.base_product.type = conf.get("base_product_type", "ga").lower()
|
|
|
f116d93 |
|
|
|
f116d93 |
ci.compose.label = compose_label
|
|
|
f116d93 |
ci.compose.type = compose_type
|
|
|
f116d93 |
ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime())
|
|
|
f116d93 |
ci.compose.respin = compose_respin or 0
|
|
|
f116d93 |
|
|
|
f116d93 |
while 1:
|
|
|
f116d93 |
ci.compose.id = ci.create_compose_id()
|
|
|
f116d93 |
|
|
|
f116d93 |
compose_dir = os.path.join(topdir, ci.compose.id)
|
|
|
f116d93 |
|
|
|
f116d93 |
exists = False
|
|
|
f116d93 |
# TODO: callbacks to determine if a composeid was already used
|
|
|
f116d93 |
# for callback in already_exists_callbacks:
|
|
|
f116d93 |
# if callback(data):
|
|
|
f116d93 |
# exists = True
|
|
|
f116d93 |
# break
|
|
|
f116d93 |
|
|
|
f116d93 |
# already_exists_callbacks fallback: does target compose_dir exist?
|
|
|
8d41a00 |
try:
|
|
|
8d41a00 |
os.makedirs(compose_dir)
|
|
|
8d41a00 |
except OSError as ex:
|
|
|
8d41a00 |
if ex.errno == errno.EEXIST:
|
|
|
8d41a00 |
exists = True
|
|
|
8d41a00 |
else:
|
|
|
8d41a00 |
raise
|
|
|
f116d93 |
|
|
|
f116d93 |
if exists:
|
|
|
f116d93 |
ci.compose.respin += 1
|
|
|
f116d93 |
continue
|
|
|
f116d93 |
break
|
|
|
f116d93 |
|
|
|
f116d93 |
open(os.path.join(compose_dir, "COMPOSE_ID"), "w").write(ci.compose.id)
|
|
|
f116d93 |
work_dir = os.path.join(compose_dir, "work", "global")
|
|
|
f116d93 |
makedirs(work_dir)
|
|
|
f116d93 |
ci.dump(os.path.join(work_dir, "composeinfo-base.json"))
|
|
|
f116d93 |
return compose_dir
|
|
|
f116d93 |
|
|
|
f116d93 |
|
|
|
f116d93 |
class Compose(kobo.log.LoggingBase):
|
|
|
bd8d814 |
def __init__(self, conf, topdir, debug=False, skip_phases=None, just_phases=None, old_composes=None, koji_event=None, supported=False, logger=None, notifier=None):
|
|
|
f116d93 |
kobo.log.LoggingBase.__init__(self, logger)
|
|
|
f116d93 |
# TODO: check if minimal conf values are set
|
|
|
f116d93 |
self.conf = conf
|
|
|
bd00920 |
# This is a dict mapping UID to Variant objects. It only contains top
|
|
|
bd00920 |
# level variants.
|
|
|
f116d93 |
self.variants = {}
|
|
|
bd00920 |
# This is a similar mapping, but contains even nested variants.
|
|
|
bd00920 |
self.all_variants = {}
|
|
|
f116d93 |
self.topdir = os.path.abspath(topdir)
|
|
|
f116d93 |
self.skip_phases = skip_phases or []
|
|
|
f116d93 |
self.just_phases = just_phases or []
|
|
|
f116d93 |
self.old_composes = old_composes or []
|
|
|
f116d93 |
self.koji_event = koji_event
|
|
|
bd8d814 |
self.notifier = notifier
|
|
|
f116d93 |
|
|
|
f116d93 |
# intentionally upper-case (visible in the code)
|
|
|
f116d93 |
self.DEBUG = debug
|
|
|
f116d93 |
|
|
|
f116d93 |
# path definitions
|
|
|
f116d93 |
self.paths = Paths(self)
|
|
|
f116d93 |
|
|
|
95cfbfb |
# Set up logging to file
|
|
|
95cfbfb |
if logger:
|
|
|
95cfbfb |
kobo.log.add_file_logger(logger, self.paths.log.log_file("global", "pungi.log"))
|
|
|
95cfbfb |
|
|
|
f116d93 |
# to provide compose_id, compose_date and compose_respin
|
|
Adam Miller |
5fa5fcd |
self.ci_base = ComposeInfo()
|
|
|
f116d93 |
self.ci_base.load(os.path.join(self.paths.work.topdir(arch="global"), "composeinfo-base.json"))
|
|
|
f116d93 |
|
|
|
f116d93 |
self.supported = supported
|
|
|
f116d93 |
if self.compose_label and self.compose_label.split("-")[0] == "RC":
|
|
|
f116d93 |
self.log_info("Automatically setting 'supported' flag for a Release Candidate (%s) compose." % self.compose_label)
|
|
|
f116d93 |
self.supported = True
|
|
|
f116d93 |
|
|
Adam Miller |
5fa5fcd |
self.im = Images()
|
|
|
f116d93 |
if self.DEBUG:
|
|
|
f116d93 |
try:
|
|
|
f116d93 |
self.im.load(self.paths.compose.metadata("images.json"))
|
|
|
f116d93 |
except RuntimeError:
|
|
|
f116d93 |
pass
|
|
|
d496eeb |
# images.json doesn't exists
|
|
|
831352b |
except IOError:
|
|
|
831352b |
pass
|
|
|
d496eeb |
# images.json is not a valid json file, for example, it's an empty file
|
|
|
d496eeb |
except ValueError:
|
|
|
d496eeb |
pass
|
|
|
f116d93 |
self.im.compose.id = self.compose_id
|
|
|
f116d93 |
self.im.compose.type = self.compose_type
|
|
|
f116d93 |
self.im.compose.date = self.compose_date
|
|
|
f116d93 |
self.im.compose.respin = self.compose_respin
|
|
|
f116d93 |
self.im.metadata_path = self.paths.compose.metadata()
|
|
|
f116d93 |
|
|
|
5f0675d |
# Stores list of deliverables that failed, but did not abort the
|
|
|
5f0675d |
# compose.
|
|
|
18b6020 |
# {deliverable: [(Variant.uid, arch, subvariant)]}
|
|
|
5f0675d |
self.failed_deliverables = {}
|
|
|
18b6020 |
self.attempted_deliverables = {}
|
|
|
18b6020 |
self.required_deliverables = {}
|
|
|
5f0675d |
|
|
|
f116d93 |
get_compose_dir = staticmethod(get_compose_dir)
|
|
|
f116d93 |
|
|
|
f116d93 |
def __getitem__(self, name):
|
|
|
f116d93 |
return self.variants[name]
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_id(self):
|
|
|
f116d93 |
return self.ci_base.compose.id
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_date(self):
|
|
|
f116d93 |
return self.ci_base.compose.date
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_respin(self):
|
|
|
f116d93 |
return self.ci_base.compose.respin
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_type(self):
|
|
|
f116d93 |
return self.ci_base.compose.type
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_type_suffix(self):
|
|
|
f116d93 |
return self.ci_base.compose.type_suffix
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def compose_label(self):
|
|
|
f116d93 |
return self.ci_base.compose.label
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
0e18ffc |
def compose_label_major_version(self):
|
|
|
0e18ffc |
return self.ci_base.compose.label_major_version
|
|
|
0e18ffc |
|
|
|
0e18ffc |
@property
|
|
|
f116d93 |
def has_comps(self):
|
|
|
f116d93 |
return bool(self.conf.get("comps_file", False))
|
|
|
f116d93 |
|
|
|
f116d93 |
@property
|
|
|
f116d93 |
def config_dir(self):
|
|
|
f116d93 |
return os.path.dirname(self.conf._open_file or "")
|
|
|
f116d93 |
|
|
|
f116d93 |
def read_variants(self):
|
|
|
f116d93 |
# TODO: move to phases/init ?
|
|
|
f116d93 |
variants_file = self.paths.work.variants_file(arch="global")
|
|
|
f116d93 |
msg = "Writing variants file: %s" % variants_file
|
|
|
f116d93 |
|
|
|
f116d93 |
if self.DEBUG and os.path.isfile(variants_file):
|
|
|
f116d93 |
self.log_warning("[SKIP ] %s" % msg)
|
|
|
f116d93 |
else:
|
|
|
f116d93 |
scm_dict = self.conf["variants_file"]
|
|
|
f116d93 |
if isinstance(scm_dict, dict):
|
|
|
f116d93 |
file_name = os.path.basename(scm_dict["file"])
|
|
|
f116d93 |
if scm_dict["scm"] == "file":
|
|
|
f116d93 |
scm_dict["file"] = os.path.join(self.config_dir, os.path.basename(scm_dict["file"]))
|
|
|
f116d93 |
else:
|
|
|
f116d93 |
file_name = os.path.basename(scm_dict)
|
|
|
f116d93 |
scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict))
|
|
|
f116d93 |
|
|
|
f116d93 |
self.log_debug(msg)
|
|
|
6fbf1e8 |
tmp_dir = self.mkdtemp(prefix="variants_file_")
|
|
|
f116d93 |
get_file_from_scm(scm_dict, tmp_dir, logger=self._logger)
|
|
|
f116d93 |
shutil.copy2(os.path.join(tmp_dir, file_name), variants_file)
|
|
|
f116d93 |
shutil.rmtree(tmp_dir)
|
|
|
f116d93 |
|
|
|
f116d93 |
tree_arches = self.conf.get("tree_arches", None)
|
|
|
d383e6c |
tree_variants = self.conf.get("tree_variants", None)
|
|
|
d383e6c |
with open(variants_file, "r") as file_obj:
|
|
|
d383e6c |
parser = VariantsXmlParser(file_obj, tree_arches, tree_variants, logger=self._logger)
|
|
|
d383e6c |
self.variants = parser.parse()
|
|
|
f116d93 |
|
|
|
bd00920 |
self.all_variants = {}
|
|
|
bd00920 |
for variant in self.get_variants():
|
|
|
bd00920 |
self.all_variants[variant.uid] = variant
|
|
|
bd00920 |
|
|
|
f116d93 |
# populate ci_base with variants - needed for layered-products (compose_id)
|
|
Adam Miller |
5fa5fcd |
####FIXME - compose_to_composeinfo is no longer needed and has been
|
|
Adam Miller |
5fa5fcd |
#### removed, but I'm not entirely sure what this is needed for
|
|
Adam Miller |
5fa5fcd |
#### or if it is at all
|
|
|
fd8cd62 |
self.ci_base = compose_to_composeinfo(self)
|
|
|
f116d93 |
|
|
|
13871b6 |
def get_variants(self, types=None, arch=None):
|
|
|
f116d93 |
result = []
|
|
|
d383e6c |
for i in self.variants.itervalues():
|
|
|
ad120f2 |
if (not types or i.type in types) and (not arch or arch in i.arches):
|
|
|
f116d93 |
result.append(i)
|
|
|
13871b6 |
result.extend(i.get_variants(types=types, arch=arch))
|
|
|
f116d93 |
return sorted(set(result))
|
|
|
f116d93 |
|
|
|
f116d93 |
def get_arches(self):
|
|
|
f116d93 |
result = set()
|
|
|
f116d93 |
for variant in self.get_variants():
|
|
|
f116d93 |
for arch in variant.arches:
|
|
|
d383e6c |
result.add(arch)
|
|
|
f116d93 |
return sorted(result)
|
|
|
f116d93 |
|
|
|
5f0675d |
@property
|
|
|
5f0675d |
def status_file(self):
|
|
|
5f0675d |
"""Path to file where the compose status will be stored."""
|
|
|
5f0675d |
if not hasattr(self, '_status_file'):
|
|
|
5f0675d |
self._status_file = os.path.join(self.topdir, 'STATUS')
|
|
|
5f0675d |
return self._status_file
|
|
|
5f0675d |
|
|
|
5f0675d |
def _log_failed_deliverables(self):
|
|
|
18b6020 |
for kind, data in self.failed_deliverables.iteritems():
|
|
|
18b6020 |
for variant, arch, subvariant in data:
|
|
|
18b6020 |
self.log_info('Failed %s on variant <%s>, arch <%s>, subvariant <%s>.'
|
|
|
18b6020 |
% (kind, variant, arch, subvariant))
|
|
|
18b6020 |
log = os.path.join(self.paths.log.topdir('global'), 'deliverables.json')
|
|
|
18b6020 |
with open(log, 'w') as f:
|
|
|
18b6020 |
json.dump({'required': self.required_deliverables,
|
|
|
18b6020 |
'failed': self.failed_deliverables,
|
|
|
18b6020 |
'attempted': self.attempted_deliverables},
|
|
|
18b6020 |
f, indent=4)
|
|
|
5f0675d |
|
|
|
f116d93 |
def write_status(self, stat_msg):
|
|
|
f116d93 |
if stat_msg not in ("STARTED", "FINISHED", "DOOMED"):
|
|
|
f116d93 |
self.log_warning("Writing nonstandard compose status: %s" % stat_msg)
|
|
|
f116d93 |
old_status = self.get_status()
|
|
|
f116d93 |
if stat_msg == old_status:
|
|
|
f116d93 |
return
|
|
|
f116d93 |
if old_status == "FINISHED":
|
|
|
f116d93 |
msg = "Could not modify a FINISHED compose: %s" % self.topdir
|
|
|
f116d93 |
self.log_error(msg)
|
|
|
f116d93 |
raise RuntimeError(msg)
|
|
|
5f0675d |
|
|
|
5f0675d |
if stat_msg == 'FINISHED' and self.failed_deliverables:
|
|
|
5f0675d |
stat_msg = 'FINISHED_INCOMPLETE'
|
|
|
18b6020 |
|
|
|
18b6020 |
self._log_failed_deliverables()
|
|
|
5f0675d |
|
|
|
5f0675d |
with open(self.status_file, "w") as f:
|
|
|
5f0675d |
f.write(stat_msg + "\n")
|
|
|
5f0675d |
|
|
|
5f0675d |
if self.notifier:
|
|
|
5f0675d |
self.notifier.send('status-change', status=stat_msg)
|
|
|
f116d93 |
|
|
|
f116d93 |
def get_status(self):
|
|
|
5f0675d |
if not os.path.isfile(self.status_file):
|
|
|
f116d93 |
return
|
|
|
5f0675d |
return open(self.status_file, "r").read().strip()
|
|
|
0e237db |
|
|
|
0e237db |
def get_image_name(self, arch, variant, disc_type='dvd',
|
|
|
0e237db |
disc_num=1, suffix='.iso', format=None):
|
|
|
0e237db |
"""Create a filename for image with given parameters.
|
|
|
0e237db |
|
|
|
0e237db |
:raises RuntimeError: when unknown ``disc_type`` is given
|
|
|
0e237db |
"""
|
|
|
a72182c |
default_format = "{compose_id}-{variant}-{arch}-{disc_type}{disc_num}{suffix}"
|
|
|
0e237db |
format = format or self.conf.get('image_name_format', default_format)
|
|
|
0e237db |
|
|
|
0e237db |
if arch == "src":
|
|
|
0e237db |
arch = "source"
|
|
|
0e237db |
|
|
|
0e237db |
if disc_num:
|
|
|
0e237db |
disc_num = int(disc_num)
|
|
|
0e237db |
else:
|
|
|
0e237db |
disc_num = ""
|
|
|
0e237db |
|
|
|
b46af7a |
kwargs = {
|
|
|
b46af7a |
'arch': arch,
|
|
|
b46af7a |
'disc_type': disc_type,
|
|
|
b46af7a |
'disc_num': disc_num,
|
|
|
b46af7a |
'suffix': suffix
|
|
|
b46af7a |
}
|
|
|
0e237db |
if variant.type == "layered-product":
|
|
|
0e237db |
variant_uid = variant.parent.uid
|
|
|
b46af7a |
kwargs['compose_id'] = self.ci_base[variant.uid].compose_id
|
|
|
0e237db |
else:
|
|
|
0e237db |
variant_uid = variant.uid
|
|
|
b46af7a |
args = get_format_substs(self, variant=variant_uid, **kwargs)
|
|
|
195b13d |
try:
|
|
|
a72182c |
return (format % args).format(**args)
|
|
|
195b13d |
except KeyError as err:
|
|
|
195b13d |
raise RuntimeError('Failed to create image name: unknown format element: %s' % err.message)
|
|
|
a6b673d |
|
|
|
a6b673d |
def can_fail(self, variant, arch, deliverable):
|
|
|
a6b673d |
"""Figure out if deliverable can fail on variant.arch.
|
|
|
a6b673d |
|
|
|
a6b673d |
Variant can be None.
|
|
|
a6b673d |
"""
|
|
|
a6b673d |
failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
|
|
|
18b6020 |
return deliverable in failable
|
|
|
18b6020 |
|
|
|
18b6020 |
def attempt_deliverable(self, variant, arch, kind, subvariant=None):
|
|
|
18b6020 |
"""Log information about attempted deliverable."""
|
|
|
18b6020 |
variant_uid = variant.uid if variant else ''
|
|
|
18b6020 |
self.attempted_deliverables.setdefault(kind, []).append(
|
|
|
18b6020 |
(variant_uid, arch, subvariant))
|
|
|
18b6020 |
|
|
|
18b6020 |
def require_deliverable(self, variant, arch, kind, subvariant=None):
|
|
|
18b6020 |
"""Log information about attempted deliverable."""
|
|
|
18b6020 |
variant_uid = variant.uid if variant else ''
|
|
|
18b6020 |
self.required_deliverables.setdefault(kind, []).append(
|
|
|
18b6020 |
(variant_uid, arch, subvariant))
|
|
|
18b6020 |
|
|
|
18b6020 |
def fail_deliverable(self, variant, arch, kind, subvariant=None):
|
|
|
18b6020 |
"""Log information about failed deliverable."""
|
|
|
18b6020 |
variant_uid = variant.uid if variant else ''
|
|
|
18b6020 |
self.failed_deliverables.setdefault(kind, []).append(
|
|
|
18b6020 |
(variant_uid, arch, subvariant))
|
|
|
ecbf08c |
|
|
|
ecbf08c |
@property
|
|
|
ecbf08c |
def image_release(self):
|
|
|
43fda1e |
"""Generate a value to pass to Koji as image release.
|
|
|
43fda1e |
|
|
|
43fda1e |
If this compose has a label, the version from it will be used,
|
|
|
43fda1e |
otherwise we will create a string with date, compose type and respin.
|
|
|
43fda1e |
"""
|
|
|
43fda1e |
if self.compose_label:
|
|
|
43fda1e |
milestone, release = self.compose_label.split('-')
|
|
|
43fda1e |
return release
|
|
|
43fda1e |
|
|
|
ecbf08c |
return '%s%s.%s' % (self.compose_date, self.ci_base.compose.type_suffix,
|
|
|
ecbf08c |
self.compose_respin)
|
|
|
223a015 |
|
|
|
223a015 |
@property
|
|
|
223a015 |
def image_version(self):
|
|
|
223a015 |
"""Generate a value to pass to Koji as image version.
|
|
|
223a015 |
|
|
|
223a015 |
The value is based on release version. If compose has a label, the
|
|
|
223a015 |
milestone from it is appended to the version (unless it is RC).
|
|
|
223a015 |
"""
|
|
|
223a015 |
version = self.ci_base.release.version
|
|
|
223a015 |
if self.compose_label and not self.compose_label.startswith('RC-'):
|
|
|
223a015 |
milestone, release = self.compose_label.split('-')
|
|
|
223a015 |
return '%s_%s' % (version, milestone)
|
|
|
223a015 |
|
|
|
223a015 |
return version
|
|
|
6fbf1e8 |
|
|
|
6fbf1e8 |
def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"):
|
|
|
6fbf1e8 |
"""
|
|
|
6fbf1e8 |
Create and return a unique temporary directory under dir of
|
|
|
6fbf1e8 |
<compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/
|
|
|
6fbf1e8 |
"""
|
|
|
6fbf1e8 |
path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
|
|
|
6fbf1e8 |
return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)
|