PR#11: First iteration of module level dependencies - modularity/fedmod

modularity / fedmod

#11 First iteration of module level dependencies

Merged 6 years ago by ncoghlan. Opened 6 years ago by ncoghlan.

modularity/ ncoghlan/fedmod issue-9-module-dependencies into master

First iteration of module level dependencies

Nick Coghlan • 6 years ago

b467339

fedmod/_depchase.py

file modified

+32 -4

		`@@ -4,15 +4,18 @@`
		`import logging`
		`import os`
		`import sys`
		`+ import gzip`
		`import tempfile`
		`import click`
		`import smartcols`
		`import solv`
		`+ import modulemd`
		`import requests`
		`from requests_toolbelt.downloadutils.tee import tee_to_file`
		`from fnmatch import fnmatch`
		`from urllib.parse import urljoin`
		`from bs4 import BeautifulSoup, SoupStrainer`
		`+ from lxml import etree`

		`XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")`
		`CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod")`
		`@@ -22,17 +25,17 @@`
		`FALLBACK_STREAM = 'master'`
		`STREAM = 'f27'`
		`ARCH = 'x86_64'`
		`- REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/"`
		`+ REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/"`
		`REPO_METADATA_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/repodata/")`
		`REPO_METADATA_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/repodata/")`
		`LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27")`
		`LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH)`
		`LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source")`

		`- METADATA_FILES = ("-filelists.xml.gz", "-primary.xml.gz", "repomd.xml")`
		`+ METADATA_FILES = ("-filelists.xml.gz", "-primary.xml.gz", "*-modules.yaml.gz", "repomd.xml")`

		`def _download_one_file(remote_url, filename):`
		`- if os.path.exists(filename):`
		`+ if os.path.exists(filename) and not filename.endswith("repomd.xml"):`
		`print(f"Skipping download; {filename} already exists")`
		`return`
		`with requests.get(remote_url, stream=True) as response:`
		`@@ -47,7 +50,7 @@`
		`print(f"Added {filename} to cache")`

		`def _download_metadata_files(metadata_url, local_path):`
		`- os.makedirs(local_path, exist_ok=True)`
		`+ os.makedirs(os.path.join(local_path, "repodata"), exist_ok=True)`
		`response = requests.get(metadata_url)`
		`response.raise_for_status()`
		`link_filter = SoupStrainer("a", href=True)`
		`@@ -80,6 +83,30 @@`
		`_download_metadata_files(REPO_METADATA_ARCH, LOCAL_REPO_INFO_ARCH)`
		`_download_metadata_files(REPO_METADATA_SOURCE, LOCAL_REPO_INFO_SOURCE)`

		`+ _MODULE_REVERSE_LOOKUP = {}`
		`+ def _populate_module_reverse_lookup():`
		`+ # TODO: Cache the reverse mapping in Repo instances, as with the solver data`
		`+ metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH)`
		`+ repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml")`
		`+ repomd_xml = etree.parse(repomd_fname)`
		`+ repo_modulemd = repomd_xml.find("rpm:data[@type='modules']/rpm:location", {"rpm": "http://linux.duke.edu/metadata/repo"})`
		`+ if repo_modulemd is None:`
		`+ raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?")`
		`+ repo_modulemd_fname = os.path.join(metadata_dir, repo_modulemd.attrib["href"])`
		`+ with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz:`
		`+ modules_yaml = modules_yaml_gz.read()`
		`+ modules = modulemd.loads_all(modules_yaml)`
		`+ for module in modules:`
		`+ # This isn't entirely valid, as it doesn't account for RPM`
		`+ # filtering, but it's good enough as a starting point`
		`+ for pkgname in module.components.rpms:`
		`+ _MODULE_REVERSE_LOOKUP[pkgname] = module.name`
		`+ for rpmname in module.artifacts.rpms:`
		`+ rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0]`
		`+ _MODULE_REVERSE_LOOKUP[rpmprefix] = module.name`
		`+`
		`+ def get_module_for_rpm(rpm_name):`
		`+ return _MODULE_REVERSE_LOOKUP.get(rpm_name)`

		`class Repo(object):`
		`def __init__(self, name, metadata_path):`
		`@@ -481,6 +508,7 @@`


		`def make_pool(arch):`
		`+ _populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction`
		`return setup_pool(arch, setup_repos())`

		`_DEFAULT_HINTS = ("glibc-minimal-langpack",)`

fedmod/module_generator.py

file modified

+30 -7

		`@@ -5,6 +5,18 @@`
		`import dnf`
		`from . import _depchase`

		`+ def _categorise_deps(all_rpm_deps):`
		`+ module_deps = set()`
		`+ remaining_rpm_deps = set()`
		`+ for pkgname in all_rpm_deps:`
		`+ modname = _depchase.get_module_for_rpm(pkgname)`
		`+ if modname is not None:`
		`+ module_deps.add(modname)`
		`+ else:`
		`+ remaining_rpm_deps.add(pkgname)`
		`+ return module_deps, remaining_rpm_deps`
		`+`
		`+`
		`class ModuleGenerator(object):`

		`def __init__(self, pkgs):`
		`@@ -64,25 +76,36 @@`

		`self.mmd.add_content_license(str(self.pkg.license))`

		`+ # Declare the public API`
		`for pkg in self.pkgs:`
		`self.mmd.api.add_rpm(pkg)`
		`# TODO: Restore resolution of pkg to the actual provider`
		`self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg))`

		`- for pkg in (self.build_deps - self.mmd.api.rpms - self.run_deps):`
		`- self.mmd.filter.add_rpm(pkg)`
		`+ # Resolve dependencies using other modules wherever possible`
		`+ module_build_deps, rpm_build_deps = _categorise_deps(self.build_deps)`
		`+ module_run_deps, rpm_run_deps = _categorise_deps(self.run_deps)`

		`- for pkg in self.build_deps.intersection(self.run_deps):`
		`+ # Declare module level dependencies`
		`+ for modname in module_build_deps:`
		`+ self.mmd.buildrequires[modname] = "f27"`
		`+ for modname in module_run_deps:`
		`+ self.mmd.requires[modname] = "f27"`
		`+`
		`+ # Add any other RPMs not available from existing modules as components`
		`+ for pkg in rpm_build_deps.intersection(rpm_run_deps):`
		`self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg))`

		`- for pkg in (self.build_deps - self.run_deps):`
		`+ for pkg in (rpm_build_deps - rpm_run_deps):`
		`self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg))`

		`- for pkg in (self.run_deps - self.build_deps):`
		`+ for pkg in (rpm_run_deps - rpm_build_deps):`
		`self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg))`

		`- # TODO: Restore module level dependency declarations for modules that`
		`- # depend on more than just the base platform`
		`+ # Filter out any build-only packages`
		`+ for pkg in (rpm_build_deps - rpm_run_deps - self.mmd.api.rpms):`
		`+ self.mmd.filter.add_rpm(pkg)`
		`+`

		`def _get_build_order(self, pkg):`
		`if pkg in self.mmd.api.rpms:`

tests/test_module_generator.py

file modified

+13 -3

		`@@ -46,10 +46,14 @@`
		`assert sorted(modmd.content_licenses) == sorted(['GPLv3+'])`

		`# Only given modules are listed in the public API`
		`- assert len(modmd.api.rpms) == 1`
		`assert sorted(modmd.api.rpms) == sorted(self.input_rpms)`

		`- # Expected dependencies for grep`
		`+ # Expected components for grep`
		`+ assert set(modmd.components.rpms) == set(self.input_rpms)`
		`+`
		`+ # Expected module dependencies for grep`
		`+ assert set(modmd.buildrequires) == set()`
		`+ assert set(modmd.requires) == set(('platform',))`



		`@@ -79,5 +83,11 @@`
		`assert len(modmd.content_licenses) == 0 # This doesn't seem right...`

		`# Only given modules are listed in the public API`
		`- assert len(modmd.api.rpms) == len(self.input_rpms)`
		`assert sorted(modmd.api.rpms) == sorted(self.input_rpms)`
		`+`
		`+ # Expected components for grep + mariadb`
		`+ assert set(modmd.components.rpms) == set(self.input_rpms)`
		`+`
		`+ # Expected module dependencies for grep + mariadb`
		`+ assert set(modmd.buildrequires) == set()`
		`+ assert set(modmd.requires) == set(('platform', 'mariadb', 'perl'))`