f116d93
# -*- coding: utf-8 -*-
f116d93
f116d93
f116d93
# This program is free software; you can redistribute it and/or modify
f116d93
# it under the terms of the GNU General Public License as published by
f116d93
# the Free Software Foundation; version 2 of the License.
f116d93
#
f116d93
# This program is distributed in the hope that it will be useful,
f116d93
# but WITHOUT ANY WARRANTY; without even the implied warranty of
f116d93
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
f116d93
# GNU Library General Public License for more details.
f116d93
#
f116d93
# You should have received a copy of the GNU General Public License
d6dc269
# along with this program; if not, see <https://gnu.org/licenses/>.
f116d93
f116d93
f116d93
__all__ = (
f116d93
    "Compose",
f116d93
)
f116d93
f116d93
f116d93
import errno
f116d93
import os
f116d93
import time
f116d93
import tempfile
f116d93
import shutil
18b6020
import json
f116d93
f116d93
import kobo.log
Adam Miller 5fa5fcd
from productmd.composeinfo import ComposeInfo
771ed2e
from productmd.images import Images
f116d93
07e90f0
from pungi.wrappers.variants import VariantsXmlParser
07e90f0
from pungi.paths import Paths
07e90f0
from pungi.wrappers.scm import get_file_from_scm
1ebb9d1
from pungi.util import makedirs, get_arch_variant_data, get_format_substs
fd8cd62
from pungi.metadata import compose_to_composeinfo
f116d93
f116d93
f116d93
def get_compose_dir(topdir, conf, compose_type="production", compose_date=None, compose_respin=None, compose_label=None, already_exists_callbacks=None):
f116d93
    already_exists_callbacks = already_exists_callbacks or []
f116d93
771ed2e
    # create an incomplete composeinfo to generate compose ID
Adam Miller 5fa5fcd
    ci = ComposeInfo()
72302bd
    ci.release.name = conf["release_name"]
72302bd
    ci.release.short = conf["release_short"]
72302bd
    ci.release.version = conf["release_version"]
72302bd
    ci.release.is_layered = bool(conf.get("release_is_layered", False))
ba9df6d
    ci.release.type = conf.get("release_type", "ga").lower()
d3cad47
    ci.release.internal = bool(conf.get("release_internal", False))
72302bd
    if ci.release.is_layered:
f116d93
        ci.base_product.name = conf["base_product_name"]
f116d93
        ci.base_product.short = conf["base_product_short"]
f116d93
        ci.base_product.version = conf["base_product_version"]
385f0e9
        ci.base_product.type = conf.get("base_product_type", "ga").lower()
f116d93
f116d93
    ci.compose.label = compose_label
f116d93
    ci.compose.type = compose_type
f116d93
    ci.compose.date = compose_date or time.strftime("%Y%m%d", time.localtime())
f116d93
    ci.compose.respin = compose_respin or 0
f116d93
f116d93
    while 1:
f116d93
        ci.compose.id = ci.create_compose_id()
f116d93
f116d93
        compose_dir = os.path.join(topdir, ci.compose.id)
f116d93
f116d93
        exists = False
f116d93
        # TODO: callbacks to determine if a composeid was already used
f116d93
        # for callback in already_exists_callbacks:
f116d93
        #     if callback(data):
f116d93
        #         exists = True
f116d93
        #         break
f116d93
f116d93
        # already_exists_callbacks fallback: does target compose_dir exist?
8d41a00
        try:
8d41a00
            os.makedirs(compose_dir)
8d41a00
        except OSError as ex:
8d41a00
            if ex.errno == errno.EEXIST:
8d41a00
                exists = True
8d41a00
            else:
8d41a00
                raise
f116d93
f116d93
        if exists:
f116d93
            ci.compose.respin += 1
f116d93
            continue
f116d93
        break
f116d93
f116d93
    open(os.path.join(compose_dir, "COMPOSE_ID"), "w").write(ci.compose.id)
f116d93
    work_dir = os.path.join(compose_dir, "work", "global")
f116d93
    makedirs(work_dir)
f116d93
    ci.dump(os.path.join(work_dir, "composeinfo-base.json"))
f116d93
    return compose_dir
f116d93
f116d93
f116d93
class Compose(kobo.log.LoggingBase):
bd8d814
    def __init__(self, conf, topdir, debug=False, skip_phases=None, just_phases=None, old_composes=None, koji_event=None, supported=False, logger=None, notifier=None):
f116d93
        kobo.log.LoggingBase.__init__(self, logger)
f116d93
        # TODO: check if minimal conf values are set
f116d93
        self.conf = conf
bd00920
        # This is a dict mapping UID to Variant objects. It only contains top
bd00920
        # level variants.
f116d93
        self.variants = {}
bd00920
        # This is a similar mapping, but contains even nested variants.
bd00920
        self.all_variants = {}
f116d93
        self.topdir = os.path.abspath(topdir)
f116d93
        self.skip_phases = skip_phases or []
f116d93
        self.just_phases = just_phases or []
f116d93
        self.old_composes = old_composes or []
f116d93
        self.koji_event = koji_event
bd8d814
        self.notifier = notifier
f116d93
f116d93
        # intentionally upper-case (visible in the code)
f116d93
        self.DEBUG = debug
f116d93
f116d93
        # path definitions
f116d93
        self.paths = Paths(self)
f116d93
95cfbfb
        # Set up logging to file
95cfbfb
        if logger:
95cfbfb
            kobo.log.add_file_logger(logger, self.paths.log.log_file("global", "pungi.log"))
95cfbfb
f116d93
        # to provide compose_id, compose_date and compose_respin
Adam Miller 5fa5fcd
        self.ci_base = ComposeInfo()
f116d93
        self.ci_base.load(os.path.join(self.paths.work.topdir(arch="global"), "composeinfo-base.json"))
f116d93
f116d93
        self.supported = supported
f116d93
        if self.compose_label and self.compose_label.split("-")[0] == "RC":
f116d93
            self.log_info("Automatically setting 'supported' flag for a Release Candidate (%s) compose." % self.compose_label)
f116d93
            self.supported = True
f116d93
Adam Miller 5fa5fcd
        self.im = Images()
f116d93
        if self.DEBUG:
f116d93
            try:
f116d93
                self.im.load(self.paths.compose.metadata("images.json"))
f116d93
            except RuntimeError:
f116d93
                pass
d496eeb
            # images.json doesn't exists
831352b
            except IOError:
831352b
                pass
d496eeb
            # images.json is not a valid json file, for example, it's an empty file
d496eeb
            except ValueError:
d496eeb
                pass
f116d93
        self.im.compose.id = self.compose_id
f116d93
        self.im.compose.type = self.compose_type
f116d93
        self.im.compose.date = self.compose_date
f116d93
        self.im.compose.respin = self.compose_respin
f116d93
        self.im.metadata_path = self.paths.compose.metadata()
f116d93
5f0675d
        # Stores list of deliverables that failed, but did not abort the
5f0675d
        # compose.
18b6020
        # {deliverable: [(Variant.uid, arch, subvariant)]}
5f0675d
        self.failed_deliverables = {}
18b6020
        self.attempted_deliverables = {}
18b6020
        self.required_deliverables = {}
5f0675d
f116d93
    get_compose_dir = staticmethod(get_compose_dir)
f116d93
f116d93
    def __getitem__(self, name):
f116d93
        return self.variants[name]
f116d93
f116d93
    @property
f116d93
    def compose_id(self):
f116d93
        return self.ci_base.compose.id
f116d93
f116d93
    @property
f116d93
    def compose_date(self):
f116d93
        return self.ci_base.compose.date
f116d93
f116d93
    @property
f116d93
    def compose_respin(self):
f116d93
        return self.ci_base.compose.respin
f116d93
f116d93
    @property
f116d93
    def compose_type(self):
f116d93
        return self.ci_base.compose.type
f116d93
f116d93
    @property
f116d93
    def compose_type_suffix(self):
f116d93
        return self.ci_base.compose.type_suffix
f116d93
f116d93
    @property
f116d93
    def compose_label(self):
f116d93
        return self.ci_base.compose.label
f116d93
f116d93
    @property
0e18ffc
    def compose_label_major_version(self):
0e18ffc
        return self.ci_base.compose.label_major_version
0e18ffc
0e18ffc
    @property
f116d93
    def has_comps(self):
f116d93
        return bool(self.conf.get("comps_file", False))
f116d93
f116d93
    @property
f116d93
    def config_dir(self):
f116d93
        return os.path.dirname(self.conf._open_file or "")
f116d93
f116d93
    def read_variants(self):
f116d93
        # TODO: move to phases/init ?
f116d93
        variants_file = self.paths.work.variants_file(arch="global")
f116d93
        msg = "Writing variants file: %s" % variants_file
f116d93
f116d93
        if self.DEBUG and os.path.isfile(variants_file):
f116d93
            self.log_warning("[SKIP ] %s" % msg)
f116d93
        else:
f116d93
            scm_dict = self.conf["variants_file"]
f116d93
            if isinstance(scm_dict, dict):
f116d93
                file_name = os.path.basename(scm_dict["file"])
f116d93
                if scm_dict["scm"] == "file":
f116d93
                    scm_dict["file"] = os.path.join(self.config_dir, os.path.basename(scm_dict["file"]))
f116d93
            else:
f116d93
                file_name = os.path.basename(scm_dict)
f116d93
                scm_dict = os.path.join(self.config_dir, os.path.basename(scm_dict))
f116d93
f116d93
            self.log_debug(msg)
6fbf1e8
            tmp_dir = self.mkdtemp(prefix="variants_file_")
f116d93
            get_file_from_scm(scm_dict, tmp_dir, logger=self._logger)
f116d93
            shutil.copy2(os.path.join(tmp_dir, file_name), variants_file)
f116d93
            shutil.rmtree(tmp_dir)
f116d93
f116d93
        tree_arches = self.conf.get("tree_arches", None)
d383e6c
        tree_variants = self.conf.get("tree_variants", None)
d383e6c
        with open(variants_file, "r") as file_obj:
d383e6c
            parser = VariantsXmlParser(file_obj, tree_arches, tree_variants, logger=self._logger)
d383e6c
            self.variants = parser.parse()
f116d93
bd00920
        self.all_variants = {}
bd00920
        for variant in self.get_variants():
bd00920
            self.all_variants[variant.uid] = variant
bd00920
f116d93
        # populate ci_base with variants - needed for layered-products (compose_id)
Adam Miller 5fa5fcd
        ####FIXME - compose_to_composeinfo is no longer needed and has been
Adam Miller 5fa5fcd
        ####        removed, but I'm not entirely sure what this is needed for
Adam Miller 5fa5fcd
        ####        or if it is at all
fd8cd62
        self.ci_base = compose_to_composeinfo(self)
f116d93
13871b6
    def get_variants(self, types=None, arch=None):
f116d93
        result = []
d383e6c
        for i in self.variants.itervalues():
ad120f2
            if (not types or i.type in types) and (not arch or arch in i.arches):
f116d93
                result.append(i)
13871b6
            result.extend(i.get_variants(types=types, arch=arch))
f116d93
        return sorted(set(result))
f116d93
f116d93
    def get_arches(self):
f116d93
        result = set()
f116d93
        for variant in self.get_variants():
f116d93
            for arch in variant.arches:
d383e6c
                result.add(arch)
f116d93
        return sorted(result)
f116d93
5f0675d
    @property
5f0675d
    def status_file(self):
5f0675d
        """Path to file where the compose status will be stored."""
5f0675d
        if not hasattr(self, '_status_file'):
5f0675d
            self._status_file = os.path.join(self.topdir, 'STATUS')
5f0675d
        return self._status_file
5f0675d
5f0675d
    def _log_failed_deliverables(self):
18b6020
        for kind, data in self.failed_deliverables.iteritems():
18b6020
            for variant, arch, subvariant in data:
18b6020
                self.log_info('Failed %s on variant <%s>, arch <%s>, subvariant <%s>.'
18b6020
                              % (kind, variant, arch, subvariant))
18b6020
        log = os.path.join(self.paths.log.topdir('global'), 'deliverables.json')
18b6020
        with open(log, 'w') as f:
18b6020
            json.dump({'required': self.required_deliverables,
18b6020
                       'failed': self.failed_deliverables,
18b6020
                       'attempted': self.attempted_deliverables},
18b6020
                      f, indent=4)
5f0675d
f116d93
    def write_status(self, stat_msg):
f116d93
        if stat_msg not in ("STARTED", "FINISHED", "DOOMED"):
f116d93
            self.log_warning("Writing nonstandard compose status: %s" % stat_msg)
f116d93
        old_status = self.get_status()
f116d93
        if stat_msg == old_status:
f116d93
            return
f116d93
        if old_status == "FINISHED":
f116d93
            msg = "Could not modify a FINISHED compose: %s" % self.topdir
f116d93
            self.log_error(msg)
f116d93
            raise RuntimeError(msg)
5f0675d
5f0675d
        if stat_msg == 'FINISHED' and self.failed_deliverables:
5f0675d
            stat_msg = 'FINISHED_INCOMPLETE'
18b6020
18b6020
        self._log_failed_deliverables()
5f0675d
5f0675d
        with open(self.status_file, "w") as f:
5f0675d
            f.write(stat_msg + "\n")
5f0675d
5f0675d
        if self.notifier:
5f0675d
            self.notifier.send('status-change', status=stat_msg)
f116d93
f116d93
    def get_status(self):
5f0675d
        if not os.path.isfile(self.status_file):
f116d93
            return
5f0675d
        return open(self.status_file, "r").read().strip()
0e237db
0e237db
    def get_image_name(self, arch, variant, disc_type='dvd',
0e237db
                       disc_num=1, suffix='.iso', format=None):
0e237db
        """Create a filename for image with given parameters.
0e237db
0e237db
        :raises RuntimeError: when unknown ``disc_type`` is given
0e237db
        """
a72182c
        default_format = "{compose_id}-{variant}-{arch}-{disc_type}{disc_num}{suffix}"
0e237db
        format = format or self.conf.get('image_name_format', default_format)
0e237db
0e237db
        if arch == "src":
0e237db
            arch = "source"
0e237db
0e237db
        if disc_num:
0e237db
            disc_num = int(disc_num)
0e237db
        else:
0e237db
            disc_num = ""
0e237db
b46af7a
        kwargs = {
b46af7a
            'arch': arch,
b46af7a
            'disc_type': disc_type,
b46af7a
            'disc_num': disc_num,
b46af7a
            'suffix': suffix
b46af7a
        }
0e237db
        if variant.type == "layered-product":
0e237db
            variant_uid = variant.parent.uid
b46af7a
            kwargs['compose_id'] = self.ci_base[variant.uid].compose_id
0e237db
        else:
0e237db
            variant_uid = variant.uid
b46af7a
        args = get_format_substs(self, variant=variant_uid, **kwargs)
195b13d
        try:
a72182c
            return (format % args).format(**args)
195b13d
        except KeyError as err:
195b13d
            raise RuntimeError('Failed to create image name: unknown format element: %s' % err.message)
a6b673d
a6b673d
    def can_fail(self, variant, arch, deliverable):
a6b673d
        """Figure out if deliverable can fail on variant.arch.
a6b673d
a6b673d
        Variant can be None.
a6b673d
        """
a6b673d
        failable = get_arch_variant_data(self.conf, 'failable_deliverables', arch, variant)
18b6020
        return deliverable in failable
18b6020
18b6020
    def attempt_deliverable(self, variant, arch, kind, subvariant=None):
18b6020
        """Log information about attempted deliverable."""
18b6020
        variant_uid = variant.uid if variant else ''
18b6020
        self.attempted_deliverables.setdefault(kind, []).append(
18b6020
            (variant_uid, arch, subvariant))
18b6020
18b6020
    def require_deliverable(self, variant, arch, kind, subvariant=None):
18b6020
        """Log information about attempted deliverable."""
18b6020
        variant_uid = variant.uid if variant else ''
18b6020
        self.required_deliverables.setdefault(kind, []).append(
18b6020
            (variant_uid, arch, subvariant))
18b6020
18b6020
    def fail_deliverable(self, variant, arch, kind, subvariant=None):
18b6020
        """Log information about failed deliverable."""
18b6020
        variant_uid = variant.uid if variant else ''
18b6020
        self.failed_deliverables.setdefault(kind, []).append(
18b6020
            (variant_uid, arch, subvariant))
ecbf08c
ecbf08c
    @property
ecbf08c
    def image_release(self):
43fda1e
        """Generate a value to pass to Koji as image release.
43fda1e
43fda1e
        If this compose has a label, the version from it will be used,
43fda1e
        otherwise we will create a string with date, compose type and respin.
43fda1e
        """
43fda1e
        if self.compose_label:
43fda1e
            milestone, release = self.compose_label.split('-')
43fda1e
            return release
43fda1e
ecbf08c
        return '%s%s.%s' % (self.compose_date, self.ci_base.compose.type_suffix,
ecbf08c
                            self.compose_respin)
223a015
223a015
    @property
223a015
    def image_version(self):
223a015
        """Generate a value to pass to Koji as image version.
223a015
223a015
        The value is based on release version. If compose has a label, the
223a015
        milestone from it is appended to the version (unless it is RC).
223a015
        """
223a015
        version = self.ci_base.release.version
223a015
        if self.compose_label and not self.compose_label.startswith('RC-'):
223a015
            milestone, release = self.compose_label.split('-')
223a015
            return '%s_%s' % (version, milestone)
223a015
223a015
        return version
6fbf1e8
6fbf1e8
    def mkdtemp(self, arch=None, variant=None, suffix="", prefix="tmp"):
6fbf1e8
        """
6fbf1e8
        Create and return a unique temporary directory under dir of
6fbf1e8
        <compose_topdir>/work/{global,<arch>}/tmp[-<variant>]/
6fbf1e8
        """
6fbf1e8
        path = os.path.join(self.paths.work.tmp_dir(arch=arch, variant=variant))
6fbf1e8
        return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=path)