f116d93
# -*- coding: utf-8 -*-
f116d93
f116d93
f116d93
# This program is free software; you can redistribute it and/or modify
f116d93
# it under the terms of the GNU General Public License as published by
f116d93
# the Free Software Foundation; version 2 of the License.
f116d93
#
f116d93
# This program is distributed in the hope that it will be useful,
f116d93
# but WITHOUT ANY WARRANTY; without even the implied warranty of
f116d93
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
f116d93
# GNU Library General Public License for more details.
f116d93
#
f116d93
# You should have received a copy of the GNU General Public License
f116d93
# along with this program; if not, write to the Free Software
f116d93
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
f116d93
f116d93
f116d93
import os
f116d93
import cPickle as pickle
f116d93
f116d93
from kobo.shortcuts import run
f116d93
07e90f0
import pungi.phases.pkgset.pkgsets
07e90f0
from pungi.arch import get_valid_arches
07e90f0
from pungi.util import makedirs
07e90f0
from pungi.wrappers.pungi import PungiWrapper
f116d93
07e90f0
from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets
07e90f0
from pungi.phases.gather import get_prepopulate_packages
07e90f0
from pungi.linker import LinkerThread, LinkerPool
f116d93
f116d93
07e90f0
import pungi.phases.pkgset.source
f116d93
f116d93
07e90f0
class PkgsetSourceRepos(pungi.phases.pkgset.source.PkgsetSourceBase):
f116d93
    enabled = True
f116d93
    config_options = (
f116d93
        {
f116d93
            "name": "pkgset_source",
f116d93
            "expected_types": [str],
f116d93
            "expected_values": "repos",
f116d93
        },
f116d93
        {
f116d93
            "name": "pkgset_repos",
f116d93
            "expected_types": [dict],
f116d93
        },
f116d93
    )
f116d93
f116d93
    def __call__(self):
f116d93
        package_sets, path_prefix = get_pkgset_from_repos(self.compose)
f116d93
        return (package_sets, path_prefix)
f116d93
f116d93
f116d93
def get_pkgset_from_repos(compose):
f116d93
    # populate pkgset from yum repos
f116d93
    # TODO: noarch hack - secondary arches, use x86_64 noarch where possible
f116d93
    flist = []
f116d93
f116d93
    link_type = compose.conf.get("link_type", "hardlink-or-copy")
f116d93
    pool = LinkerPool(link_type, logger=compose._logger)
f116d93
    for i in range(10):
f116d93
        pool.add(LinkerThread(pool))
f116d93
f116d93
    seen_packages = set()
f116d93
    for arch in compose.get_arches():
f116d93
        # write a pungi config for remote repos and a local comps repo
f116d93
        repos = {}
f116d93
        for num, repo in enumerate(compose.conf["pkgset_repos"][arch]):
f116d93
            repo_path = repo
f116d93
            if "://" not in repo_path:
f116d93
                repo_path = os.path.join(compose.config_dir, repo)
f116d93
            repos["repo-%s" % num] = repo_path
f116d93
f116d93
        comps_repo = None
f116d93
        if compose.has_comps:
f116d93
            repos["comps"] = compose.paths.work.comps_repo(arch=arch)
f116d93
            comps_repo = "comps"
f116d93
        write_pungi_config(compose, arch, None, repos=repos, comps_repo=comps_repo)
f116d93
f116d93
        pungi = PungiWrapper()
f116d93
        pungi_conf = compose.paths.work.pungi_conf(arch=arch)
f116d93
        pungi_log = compose.paths.log.log_file(arch, "fooo")
f116d93
        pungi_dir = compose.paths.work.pungi_download_dir(arch)
f116d93
        cmd = pungi.get_pungi_cmd(pungi_conf, destdir=pungi_dir, name="FOO", selfhosting=True, fulltree=True, multilib_methods=["all"], nodownload=False, full_archlist=True, arch=arch, cache_dir=compose.paths.work.pungi_cache_dir(arch=arch))
f116d93
        cmd.append("--force")
f116d93
f116d93
        # TODO: runroot
f116d93
        run(cmd, logfile=pungi_log, show_cmd=True, stdout=False)
f116d93
f116d93
        path_prefix = os.path.join(compose.paths.work.topdir(arch="global"), "download") + "/"
f116d93
        makedirs(path_prefix)
f116d93
        for root, dirs, files in os.walk(pungi_dir):
f116d93
            for fn in files:
f116d93
                if not fn.endswith(".rpm"):
f116d93
                    continue
f116d93
                if fn in seen_packages:
f116d93
                    continue
f116d93
                seen_packages.add(fn)
f116d93
                src = os.path.join(root, fn)
f116d93
                dst = os.path.join(path_prefix, os.path.basename(src))
f116d93
                flist.append(dst)
f116d93
                pool.queue_put((src, dst))
f116d93
f116d93
    msg = "Linking downloaded pkgset packages"
f116d93
    compose.log_info("[BEGIN] %s" % msg)
f116d93
    pool.start()
f116d93
    pool.stop()
f116d93
    compose.log_info("[DONE ] %s" % msg)
f116d93
f116d93
    flist = sorted(set(flist))
f116d93
    pkgset_global = populate_global_pkgset(compose, flist, path_prefix)
f116d93
#    get_extra_packages(compose, pkgset_global)
f116d93
    package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
f116d93
f116d93
    create_global_repo(compose, path_prefix)
f116d93
    for arch in compose.get_arches():
f116d93
        # TODO: threads? runroot?
f116d93
        create_arch_repos(compose, arch, path_prefix)
f116d93
f116d93
    package_sets["global"] = pkgset_global
f116d93
    return package_sets, path_prefix
f116d93
f116d93
f116d93
def populate_global_pkgset(compose, file_list, path_prefix):
f116d93
    ALL_ARCHES = set(["src"])
f116d93
    for arch in compose.get_arches():
f116d93
        is_multilib = arch in compose.conf["multilib_arches"]
f116d93
        arches = get_valid_arches(arch, is_multilib)
f116d93
        ALL_ARCHES.update(arches)
f116d93
f116d93
    msg = "Populating the global package set from a file list"
f116d93
    global_pkgset_path = os.path.join(compose.paths.work.topdir(arch="global"), "packages.pickle")
f116d93
    if compose.DEBUG and os.path.isfile(global_pkgset_path):
f116d93
        compose.log_warning("[SKIP ] %s" % msg)
f116d93
        pkgset = pickle.load(open(global_pkgset_path, "r"))
f116d93
    else:
f116d93
        compose.log_info(msg)
07e90f0
        pkgset = pungi.phases.pkgset.pkgsets.FilelistPackageSet(compose.conf["sigkeys"], logger=compose._logger, arches=ALL_ARCHES)
f116d93
        pkgset.populate(file_list)
f116d93
        f = open(global_pkgset_path, "w")
f116d93
        data = pickle.dumps(pkgset)
f116d93
        f.write(data)
f116d93
        f.close()
f116d93
f116d93
    # write global package list
f116d93
    pkgset.save_file_list(compose.paths.work.package_list(arch="global"), remove_path_prefix=path_prefix)
f116d93
    return pkgset
f116d93
f116d93
f116d93
def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, package_set=None):
f116d93
    """write pungi config (kickstart) for arch/variant"""
ec74243
    pungi_wrapper = PungiWrapper()
f116d93
    pungi_cfg = compose.paths.work.pungi_conf(variant=variant, arch=arch)
f116d93
    msg = "Writing pungi config (arch: %s, variant: %s): %s" % (arch, variant, pungi_cfg)
f116d93
f116d93
    if compose.DEBUG and os.path.isfile(pungi_cfg):
f116d93
        compose.log_warning("[SKIP ] %s" % msg)
f116d93
        return
f116d93
f116d93
    compose.log_info(msg)
f116d93
f116d93
    # TODO move to a function
f116d93
    gather_source = "GatherSource%s" % compose.conf["gather_source"]
07e90f0
    from pungi.phases.gather.source import GatherSourceContainer
07e90f0
    import pungi.phases.gather.sources
07e90f0
    GatherSourceContainer.register_module(pungi.phases.gather.sources)
f116d93
    container = GatherSourceContainer()
f116d93
    SourceClass = container[gather_source]
f116d93
    src = SourceClass(compose)
f116d93
f116d93
    packages = []
f116d93
    pkgs, grps = src(arch, variant)
f116d93
    for pkg_name, pkg_arch in pkgs:
f116d93
        if pkg_arch is None:
f116d93
            packages.append(pkg_name)
f116d93
        else:
f116d93
            packages.append("%s.%s" % (pkg_name, pkg_arch))
f116d93
f116d93
    # include *all* packages providing system-release
f116d93
    if "system-release" not in packages:
f116d93
        packages.append("system-release")
f116d93
f116d93
    prepopulate = get_prepopulate_packages(compose, arch, None)
ec74243
    pungi_wrapper.write_kickstart(ks_path=pungi_cfg, repos=repos, groups=grps, packages=packages, exclude_packages=[], comps_repo=None, prepopulate=prepopulate)