From b467339604129abe0dc42c9fc815d6d7d763284c Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 23 2017 07:26:05 +0000 Subject: First iteration of module level dependencies - no caching (reads the YAML files every time) - not separated per repo - doesn't account for RPM output filters in other modules - arbitrary winner when multiple modules publish the same RPM --- diff --git a/fedmod/_depchase.py b/fedmod/_depchase.py index 5cf0698..5f4de1f 100644 --- a/fedmod/_depchase.py +++ b/fedmod/_depchase.py @@ -4,15 +4,18 @@ import itertools import logging import os import sys +import gzip import tempfile import click import smartcols import solv +import modulemd import requests from requests_toolbelt.downloadutils.tee import tee_to_file from fnmatch import fnmatch from urllib.parse import urljoin from bs4 import BeautifulSoup, SoupStrainer +from lxml import etree XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") @@ -22,17 +25,17 @@ log = logging.getLogger(__name__) FALLBACK_STREAM = 'master' STREAM = 'f27' ARCH = 'x86_64' -REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" +REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" REPO_METADATA_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/repodata/") REPO_METADATA_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/repodata/") LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27") LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH) LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source") -METADATA_FILES = ("*-filelists.xml.gz", "*-primary.xml.gz", "repomd.xml") +METADATA_FILES = ("*-filelists.xml.gz", "*-primary.xml.gz", "*-modules.yaml.gz", "repomd.xml") def _download_one_file(remote_url, filename): - if os.path.exists(filename): + if os.path.exists(filename) and not filename.endswith("repomd.xml"): print(f"Skipping download; {filename} already exists") return with requests.get(remote_url, stream=True) as response: @@ -47,7 +50,7 @@ def _download_one_file(remote_url, filename): print(f"Added {filename} to cache") def _download_metadata_files(metadata_url, local_path): - os.makedirs(local_path, exist_ok=True) + os.makedirs(os.path.join(local_path, "repodata"), exist_ok=True) response = requests.get(metadata_url) response.raise_for_status() link_filter = SoupStrainer("a", href=True) @@ -80,6 +83,30 @@ def download_repo_metadata(): _download_metadata_files(REPO_METADATA_ARCH, LOCAL_REPO_INFO_ARCH) _download_metadata_files(REPO_METADATA_SOURCE, LOCAL_REPO_INFO_SOURCE) +_MODULE_REVERSE_LOOKUP = {} +def _populate_module_reverse_lookup(): + # TODO: Cache the reverse mapping in Repo instances, as with the solver data + metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + repomd_xml = etree.parse(repomd_fname) + repo_modulemd = repomd_xml.find("rpm:data[@type='modules']/rpm:location", {"rpm": "http://linux.duke.edu/metadata/repo"}) + if repo_modulemd is None: + raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?") + repo_modulemd_fname = os.path.join(metadata_dir, repo_modulemd.attrib["href"]) + with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + modules_yaml = modules_yaml_gz.read() + modules = modulemd.loads_all(modules_yaml) + for module in modules: + # This isn't entirely valid, as it doesn't account for RPM + # filtering, but it's good enough as a starting point + for pkgname in module.components.rpms: + _MODULE_REVERSE_LOOKUP[pkgname] = module.name + for rpmname in module.artifacts.rpms: + rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] + _MODULE_REVERSE_LOOKUP[rpmprefix] = module.name + +def get_module_for_rpm(rpm_name): + return _MODULE_REVERSE_LOOKUP.get(rpm_name) class Repo(object): def __init__(self, name, metadata_path): @@ -481,6 +508,7 @@ def solve(solver, pkgnames, selfhost=False): def make_pool(arch): + _populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction return setup_pool(arch, setup_repos()) _DEFAULT_HINTS = ("glibc-minimal-langpack",) diff --git a/fedmod/module_generator.py b/fedmod/module_generator.py index 8bd8c58..520e7d5 100644 --- a/fedmod/module_generator.py +++ b/fedmod/module_generator.py @@ -5,6 +5,18 @@ import logging import dnf from . import _depchase +def _categorise_deps(all_rpm_deps): + module_deps = set() + remaining_rpm_deps = set() + for pkgname in all_rpm_deps: + modname = _depchase.get_module_for_rpm(pkgname) + if modname is not None: + module_deps.add(modname) + else: + remaining_rpm_deps.add(pkgname) + return module_deps, remaining_rpm_deps + + class ModuleGenerator(object): def __init__(self, pkgs): @@ -64,25 +76,36 @@ class ModuleGenerator(object): self.mmd.add_content_license(str(self.pkg.license)) + # Declare the public API for pkg in self.pkgs: self.mmd.api.add_rpm(pkg) # TODO: Restore resolution of pkg to the actual provider self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) - for pkg in (self.build_deps - self.mmd.api.rpms - self.run_deps): - self.mmd.filter.add_rpm(pkg) + # Resolve dependencies using other modules wherever possible + module_build_deps, rpm_build_deps = _categorise_deps(self.build_deps) + module_run_deps, rpm_run_deps = _categorise_deps(self.run_deps) - for pkg in self.build_deps.intersection(self.run_deps): + # Declare module level dependencies + for modname in module_build_deps: + self.mmd.buildrequires[modname] = "f27" + for modname in module_run_deps: + self.mmd.requires[modname] = "f27" + + # Add any other RPMs not available from existing modules as components + for pkg in rpm_build_deps.intersection(rpm_run_deps): self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) - for pkg in (self.build_deps - self.run_deps): + for pkg in (rpm_build_deps - rpm_run_deps): self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) - for pkg in (self.run_deps - self.build_deps): + for pkg in (rpm_run_deps - rpm_build_deps): self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) - # TODO: Restore module level dependency declarations for modules that - # depend on more than just the base platform + # Filter out any build-only packages + for pkg in (rpm_build_deps - rpm_run_deps - self.mmd.api.rpms): + self.mmd.filter.add_rpm(pkg) + def _get_build_order(self, pkg): if pkg in self.mmd.api.rpms: diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index e4d2f57..50fdbb9 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -46,10 +46,14 @@ class TestSinglePackageInput(object): assert sorted(modmd.content_licenses) == sorted(['GPLv3+']) # Only given modules are listed in the public API - assert len(modmd.api.rpms) == 1 assert sorted(modmd.api.rpms) == sorted(self.input_rpms) - # Expected dependencies for grep + # Expected components for grep + assert set(modmd.components.rpms) == set(self.input_rpms) + + # Expected module dependencies for grep + assert set(modmd.buildrequires) == set() + assert set(modmd.requires) == set(('platform',)) @@ -79,5 +83,11 @@ class TestMultiplePackageInput(object): assert len(modmd.content_licenses) == 0 # This doesn't seem right... # Only given modules are listed in the public API - assert len(modmd.api.rpms) == len(self.input_rpms) assert sorted(modmd.api.rpms) == sorted(self.input_rpms) + + # Expected components for grep + mariadb + assert set(modmd.components.rpms) == set(self.input_rpms) + + # Expected module dependencies for grep + mariadb + assert set(modmd.buildrequires) == set() + assert set(modmd.requires) == set(('platform', 'mariadb', 'perl'))