From 97a14a545ffb7754c2bc3a4d0fde2a28d8b798c7 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 27 2017 10:30:49 +0000 Subject: Fully resolve build dependencies Also account for the bootstrap module as a potential dependency for build requirements. --- diff --git a/README.md b/README.md index ff56937..6e833ba 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,18 @@ directly to a file instead by passing the ``--output` (or `-o`) option: The following metadata is currently used as input to the draft module generation process: -* Module definitions are pulled from the modular Fedora Bikeshed repository, - with the metadata being downloaded for local use via the - `fedmod fetch-metadata` command - * Package dependency definitions are pulled from the regular Fedora 27 release and updates repositories, with the metadata being downloaded for local use via the `fedmod fetch-metadata` command +* Installable module definitions are pulled from the modular Fedora Bikeshed + repository, with the metadata being downloaded for local use via the + `fedmod fetch-metadata` command + +* The definition of Fedora's build-only `bootstrap` module is retrieved + directly from the relevant + [dist-git repository](https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml) + * Descriptive metadata is taken from the system running `fedmod`. Due to this, `fedmod` currently only supports Fedora 26+. (This will be fixed to use the same repository metadata as is used for package dependency resolution) diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 3323f95..9f2d340 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -11,9 +11,6 @@ from . import _repodata log = logging.getLogger(__name__) -def get_module_for_rpm(rpm_name): - return _repodata._RPM_REVERSE_LOOKUP.get(rpm_name) - def setup_pool(arch, repos=()): pool = solv.Pool() #pool.set_debuglevel(2) @@ -114,7 +111,11 @@ def _solve(solver, pkgnames): jobs = [] # Initial jobs, no conflicting packages for n in pkgnames: - sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) + if "." in n: + search_criteria = solv.Selection.SELECTION_CANON + else: + search_criteria = solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH + sel = pool.select(n, search_criteria) if sel.isempty(): log.warn("Could not find package for {}".format(n)) continue @@ -132,26 +133,26 @@ def _solve(solver, pkgnames): continue if s.arch in ("src", "nosrc"): continue - result.add(s) + # Ensure the solvables don't outlive the solver that created them + result.add(s.name) return result -def ensure_buildable(pkgset, pool=None): +def ensure_buildable(pool, pkgnames): """Given a set of solvables, returns a set of source packages & build deps""" # The given package set may not be installable on its own # That's OK, since other modules will provide those packages # The goal of *this* method is to report the SRPMs that need to be # built, and their build dependencies - sources = set(get_sourcepkg(s, only_name=True) for s in pkgset) - builddeps = ensure_installable(sources, pool=pool) + sources = set(get_srpm_for_rpm(pool, n) for n in pkgnames) + builddeps = ensure_installable(pool, sources) return sources, builddeps def make_pool(arch): - _repodata._populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction return setup_pool(arch, _repodata.setup_repos()) _DEFAULT_HINTS = ("glibc-minimal-langpack",) -def ensure_installable(pkgnames, hints=_DEFAULT_HINTS, recommendations=False, pool=None): +def ensure_installable(pool, pkgnames, hints=_DEFAULT_HINTS, recommendations=False): """Iterate over the resolved dependency set for the given packages *hints*: Packages that have higher priority when more than one package @@ -196,23 +197,43 @@ def get_srpm_for_rpm(pool, pkg): if sel.isempty(): return f"unknown-component-{pkg}" found = sel.solvables() - if len(found) > 1: - raise RuntimeError("More matching solvables were found, {}".format(found)) + num_results = len(found) + s = None + if num_results == 1: + s = found[0] + elif num_results == 2: + # Handle x86 32-bit vs 64-bit multilib packages + first, second = found + if first.arch == "x86_64" and second.arch == "i686": + s = first + elif first.arch == "i686" and second.arch == "x86_64": + s = second + if s is None: + raise RuntimeError("More matching solvables were found, {}".format(found)) s = found[0] return get_sourcepkg(s, only_name=True) -def pkgs_by_sourcepkg(pool, sourcepkg): - - sel = pool.select(sourcepkg, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) - if not sel.isempty(): - found = sel.solvables() - assert len(found) == 1, "More matching solvables were found, {}".format(found) - if found[0].arch in ("src", "nosrc"): - sourcepkg = str(found[0]) - else: - sourcepkg = get_sourcepkg(found[0], only_name=True) +def get_rpms_for_srpms(pool, pkgnames): + sources = set() + for n in pkgnames: + sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH | solv.Selection.SELECTION_WITH_SOURCE) + if not sel.isempty(): + found = sel.solvables() + for rpm in found: + if rpm.arch in ("src", "nosrc"): + sources.add(str(found[0])) + break + else: + # Multilib means we may see multiple binary RPMs with the same name + sources.add(get_sourcepkg(rpm, only_name=True)) + + # This search is O(N) where N = the number of packages in Fedora + # so it would be nice to find a more algorithmically efficient approach + # OTOH, we only run this when *fetching* metadata, so it isn't too bad + result = set() for p in (s for s in pool.solvables if s.arch not in ("src", "nosrc")): - if get_sourcepkg(p, only_name=True) == sourcepkg: - print(p) + if get_sourcepkg(p, only_name=True) in sources: + result.add(p.name) + return result diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 8eaa7ed..1689aac 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -8,6 +8,7 @@ import gzip import logging import modulemd import solv +import json from attr import attributes, attrib from requests_toolbelt.downloadutils.tee import tee_to_file from fnmatch import fnmatch @@ -25,6 +26,7 @@ ARCH = 'x86_64' _F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" _F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" _F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" +_F27_BOOTSTRAP_MODULEMD = "https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml" @attributes class RepoPaths: @@ -67,6 +69,8 @@ _ALL_REPOS = ( _x86_64_UPDATES_INFO, _SOURCE_UPDATES_INFO, ) +_BOOTSTRAP_MODULEMD = os.path.join(CACHEDIR, "f27-bootstrap.yaml") +_BOOTSTRAP_REVERSE_LOOKUP_CACHE = os.path.join(CACHEDIR, "f27-bootstrap-cache.json") METADATA_SECTIONS = ("filelists", "primary", "modules") @@ -78,7 +82,7 @@ def _read_repomd_location(repomd_xml, section): return None def _download_one_file(remote_url, filename): - if os.path.exists(filename) and not filename.endswith("repomd.xml"): + if os.path.exists(filename) and not filename.endswith((".xml", ".yaml")): print(f" Skipping download; {filename} already exists") return with requests.get(remote_url, stream=True) as response: @@ -123,16 +127,38 @@ def _download_metadata_files(repo_paths): # TODO: Actually prune old metadata files pass +def _download_bootstrap_modulemd(): + from ._depchase import make_pool, get_rpms_for_srpms + print("Downloading build bootstrap module details") + _download_one_file(_F27_BOOTSTRAP_MODULEMD, _BOOTSTRAP_MODULEMD) + # TODO: Cache the modulemd file hash, and only regenerate the cache + # if that has changed + mmd = modulemd.ModuleMetadata() + mmd.load(_BOOTSTRAP_MODULEMD) + pool = make_pool("x86_64") + bootstrap_rpms = {} + rpms = get_rpms_for_srpms(pool, mmd.components.rpms) + for rpmname in rpms: + bootstrap_rpms[rpmname] = "bootstrap" + for srpmname in mmd.components.rpms: + bootstrap_rpms[srpmname] = "bootstrap" + with open(_BOOTSTRAP_REVERSE_LOOKUP_CACHE, "w") as cachefile: + json.dump(bootstrap_rpms, cachefile) + print(f" Added {_BOOTSTRAP_REVERSE_LOOKUP_CACHE} to cache") def download_repo_metadata(): """Downloads the latest repo metadata""" for repo_definition in _ALL_REPOS: _download_metadata_files(repo_definition) + _download_bootstrap_modulemd() _SRPM_REVERSE_LOOKUP = {} # SRPM name : module name _RPM_REVERSE_LOOKUP = {} # RPM name : module name +_BOOTSTRAP_REVERSE_LOOKUP = {} def _populate_module_reverse_lookup(): - # TODO: Cache the reverse mapping in _depchase.Repo instances, as with the solver data + # TODO: Cache the reverse mapping as a JSON file, as with _BOOTSTRAP_REVERSE_LOOKUP_CACHE + if _RPM_REVERSE_LOOKUP: + return metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") repomd_xml = etree.parse(repomd_fname) @@ -155,6 +181,16 @@ def _populate_module_reverse_lookup(): # published by at most one module rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] _RPM_REVERSE_LOOKUP[rpmprefix] = module.name + # Read the extra RPM bootstrap metadata + with open(_BOOTSTRAP_REVERSE_LOOKUP_CACHE, "r") as cachefile: + _BOOTSTRAP_REVERSE_LOOKUP.update(json.load(cachefile)) + + +def get_module_for_rpm(rpm_name, *, allow_bootstrap=False): + result = _RPM_REVERSE_LOOKUP.get(rpm_name) + if allow_bootstrap and result is None: + _BOOTSTRAP_REVERSE_LOOKUP.get(rpm_name) + return result class Repo(object): def __init__(self, name, metadata_path): diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index 9ddda30..f63636a 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -4,21 +4,21 @@ import sys import modulemd import logging import dnf -from . import _depchase +from . import _depchase, _repodata def _name_only(rpm_name): name, version, release = rpm_name.rsplit("-", 2) return name -def _categorise_deps(pool, all_rpm_deps): +def _categorise_deps(all_rpm_deps, *, allow_bootstrap=False): module_deps = set() remaining_rpm_deps = set() - for pkg in all_rpm_deps: - modname = _depchase.get_module_for_rpm(pkg.name) + for pkgname in all_rpm_deps: + modname = _repodata.get_module_for_rpm(pkgname, allow_bootstrap=allow_bootstrap) if modname is not None: module_deps.add(modname) else: - remaining_rpm_deps.add(pkg) + remaining_rpm_deps.add(pkgname) return module_deps, remaining_rpm_deps class ModuleGenerator(object): @@ -31,23 +31,24 @@ class ModuleGenerator(object): def _calculate_dependencies(self): pkgs = self.pkgs + _repodata._populate_module_reverse_lookup() pool = _depchase.make_pool("x86_64") self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} - run_deps = _depchase.ensure_installable(pkgs, pool=pool) - module_run_deps, rpm_run_deps = _categorise_deps(pool, run_deps) + run_deps = _depchase.ensure_installable(pool, pkgs) + module_run_deps, rpm_run_deps = _categorise_deps(run_deps) self.module_run_deps = module_run_deps - run_srpms, build_deps = _depchase.ensure_buildable(rpm_run_deps) + run_srpms, build_deps = _depchase.ensure_buildable(pool, rpm_run_deps) module_build_deps = set() resolved_build_deps = set() build_srpms = set() for i in range(10): # Arbitrary bound of 10 levels of SRPM bootstrapping - new_module_build_deps, remaining_build_deps = _categorise_deps(pool, build_deps) + new_module_build_deps, remaining_build_deps = _categorise_deps(build_deps, allow_bootstrap=True) module_build_deps |= new_module_build_deps resolved_build_deps |= (build_deps - remaining_build_deps) build_deps -= resolved_build_deps if build_deps: - new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(build_deps) + new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(pool, build_deps) build_srpms |= new_build_srpms resolved_build_deps |= (build_deps - remaining_build_deps) build_deps -= resolved_build_deps @@ -62,7 +63,7 @@ class ModuleGenerator(object): self.run_srpms = run_srpm_names - build_srpm_names self.build_srpms = build_srpm_names - run_srpm_names self.build_and_run_srpms = run_srpm_names & build_srpm_names - self.unresolved_build_rpms = {_name_only(s.name) for s in build_deps} + self.unresolved_build_rpms = {n for n in build_deps} def _get_pkg_info(self): diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index d8c7aa8..4da5864 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -60,7 +60,7 @@ class TestSinglePackageInput(object): class TestMultiplePackageInput(object): def setup(self): - self.input_rpms = input_rpms = ('grep', 'mariadb') + self.input_rpms = input_rpms = ('grep', 'haproxy') self.md, self.output_fname = _generate_modulemd(input_rpms) def teardown(self): @@ -77,7 +77,7 @@ class TestMultiplePackageInput(object): assert modmd.summary == "" assert modmd.description == "" - # Expected licenses for grep + mariadb + # Expected licenses for grep + haproxy assert len(modmd.module_licenses) == 1 assert sorted(modmd.module_licenses) == sorted(['MIT']) assert len(modmd.content_licenses) == 0 # This doesn't seem right... @@ -85,11 +85,10 @@ class TestMultiplePackageInput(object): # Only given modules are listed in the public API assert sorted(modmd.api.rpms) == sorted(self.input_rpms) - # Expected components for grep + mariadb + # Expected components for grep + haproxy expected_components = set(self.input_rpms) assert set(modmd.components.rpms) == expected_components - # Expected module dependencies for grep + mariadb - assert set(modmd.buildrequires) == set() - expected_modules = set(('platform', 'mariadb', 'perl')) - assert set(modmd.requires) == expected_modules + # Expected module dependencies for grep + haproxy + assert set(modmd.buildrequires) == set(('platform',)) + assert set(modmd.requires) == set(('platform',))