From 482bf2b4ffc3eada58608db243e377e5f489cbb4 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 20:25:01 +0000 Subject: [PATCH 1/6] Move gzip import into the right file --- diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py index 3b678aa..7f9bd3b 100644 --- a/src/_fedmod/_fetchrepodata.py +++ b/src/_fedmod/_fetchrepodata.py @@ -1,4 +1,5 @@ """_fetchrepodata: Map yum/dnf repo metadata to local lookup caches""" +import gzip import json import logging import os diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 9f3c718..a468a5a 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -4,7 +4,6 @@ import tempfile import os.path import requests import click -import gzip import logging import modulemd import solv From c6b242d23b5b2edb9c18344a12703a5f578c6bce Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 20:25:01 +0000 Subject: [PATCH 2/6] Remove references to bootstrap module The bootstrap module is no longer needed with new F28 modularity plan, so simplify by removing specific handling. --- diff --git a/src/README.md b/src/README.md index 5e3ff51..7533dd7 100644 --- a/src/README.md +++ b/src/README.md @@ -181,10 +181,6 @@ process: repository, with the metadata being downloaded for local use via the `fedmod fetch-metadata` command -* The definition of Fedora's build-only `bootstrap` module is retrieved - directly from the relevant - [dist-git repository](https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml) - * Descriptive metadata is taken from the system running `fedmod`. Due to this, `fedmod` currently only supports Fedora 26+. (This will be fixed to use the same repository metadata as is used for package dependency resolution) diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py index 7f9bd3b..28f415e 100644 --- a/src/_fedmod/_fetchrepodata.py +++ b/src/_fedmod/_fetchrepodata.py @@ -25,7 +25,6 @@ ARCH = 'x86_64' _F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" _F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" _F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" -_F27_BOOTSTRAP_MODULEMD = "https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml" class MissingMetadata(Exception): """Reports failure to find the local metadata cache""" @@ -71,10 +70,8 @@ _ALL_REPOS = ( _x86_64_UPDATES_INFO, _SOURCE_UPDATES_INFO, ) -_BOOTSTRAP_MODULEMD = os.path.join(CACHEDIR, "f27-bootstrap.yaml") _LOOKUP_CACHES = { - "_BOOTSTRAP_COMPONENTS_CACHE": os.path.join(CACHEDIR, "f27-bootstrap-cache.json"), "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-module-contents-cache.json"), "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-srpm-to-module-cache.json"), "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-rpm-to-module-cache.json"), @@ -151,23 +148,6 @@ def _read_cache(cache_name): with open(cache_fname, "r") as cache_file: return json.load(cache_file) -def _download_bootstrap_modulemd(): - from ._depchase import make_pool, get_rpms_for_srpms - print("Downloading build bootstrap module details") - _download_one_file(_F27_BOOTSTRAP_MODULEMD, _BOOTSTRAP_MODULEMD) - # TODO: Cache the modulemd file hash, and only regenerate the cache - # if that has changed - mmd = modulemd.ModuleMetadata() - mmd.load(_BOOTSTRAP_MODULEMD) - pool = make_pool("x86_64") - bootstrap_rpms = set() - rpms = get_rpms_for_srpms(pool, mmd.components.rpms) - for rpmname in rpms: - bootstrap_rpms.add(rpmname) - for srpmname in mmd.components.rpms: - bootstrap_rpms.add(srpmname) - _write_cache("_BOOTSTRAP_COMPONENTS_CACHE", list(bootstrap_rpms)) - def _write_lookup_caches(): metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") @@ -198,7 +178,6 @@ def download_repo_metadata(): """Downloads the latest repo metadata""" for repo_definition in _ALL_REPOS: _download_metadata_files(repo_definition) - _download_bootstrap_modulemd() _write_lookup_caches() @attributes @@ -207,7 +186,6 @@ class LocalMetadataCache: cache_dir = attrib(str) srpm_to_modules = attrib(dict) rpm_to_modules = attrib(dict) - bootstrap_components = attrib(set) module_to_packages = attrib(dict) source_repo_cache = attrib(str) arch_repo_cache = attrib(str) @@ -236,7 +214,6 @@ def load_cached_repodata(dataset_name): cache_dir = CACHEDIR, srpm_to_modules = _read_cache("_SRPM_REVERSE_LOOKUP_CACHE"), rpm_to_modules = _read_cache("_RPM_REVERSE_LOOKUP_CACHE"), - bootstrap_components = _read_cache("_BOOTSTRAP_COMPONENTS_CACHE"), module_to_packages = _read_cache("_MODULE_FORWARD_LOOKUP_CACHE"), source_repo_cache = _SOURCE_PACKAGE_INFO.local_cache_path, arch_repo_cache = _x86_64_PACKAGE_INFO.local_cache_path, diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index a468a5a..718043b 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -34,14 +34,12 @@ def get_modules_for_rpm(rpm_name): result = _get_dataset().rpm_to_modules.get(rpm_name) return result -def get_module_for_rpm(rpm_name, *, allow_bootstrap=False): +def get_module_for_rpm(rpm_name): result = _get_dataset().rpm_to_modules.get(rpm_name) if result is not None: if len(result) > 1: log.warn(f"Multiple modules found for {rpm_name!r}: {','.join(result)}") result = result[0] - elif allow_bootstrap and rpm_name in _get_dataset().bootstrap_components: - result = "bootstrap" return result def get_rpm_reverse_lookup(): diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index ee03dcc..ed5b4bb 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -7,11 +7,11 @@ def _name_only(rpm_name): name, version, release = rpm_name.rsplit("-", 2) return name -def _categorise_deps(all_rpm_deps, *, allow_bootstrap=False): +def _categorise_deps(all_rpm_deps): module_deps = set() remaining_rpm_deps = set() for pkgname in all_rpm_deps: - modname = _repodata.get_module_for_rpm(pkgname, allow_bootstrap=allow_bootstrap) + modname = _repodata.get_module_for_rpm(pkgname) if modname is not None: module_deps.add(modname) else: @@ -40,7 +40,7 @@ class ModuleGenerator(object): for i in range(build_deps_iterations+1): # Give up on making the module self-hosting after the requested # number of iterations - new_module_build_deps, remaining_build_deps = _categorise_deps(build_deps, allow_bootstrap=True) + new_module_build_deps, remaining_build_deps = _categorise_deps(build_deps) module_build_deps |= new_module_build_deps resolved_build_deps |= (build_deps - remaining_build_deps) build_deps -= resolved_build_deps diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index 5c2f66c..aba7a8a 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -110,7 +110,6 @@ class TestRecursiveBuildDeps(object): # Expected module dependencies expected_build_deps = { 'platform', - 'bootstrap', 'mariadb', 'networking-base', 'host', @@ -172,7 +171,6 @@ class TestNonRecursiveBuildDeps(object): 'fonts', 'python2', 'python2-ecosystem', - 'bootstrap' } _unexpected_build_requires = { 'pki', From ca1ff8c23bf4291b81af350ddcb44d13f3a9450c Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 20:25:01 +0000 Subject: [PATCH 3/6] Remove special handling of fedora-release We no longer have a different -release package for Modularity, so don't exclude fedora-release - it's just a normal package that will be found in the base package set. --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 6665cb1..9d61424 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -155,9 +155,6 @@ def _solve(solver, pkgnames, full_info=False): else: result = set() for s in solver.transaction().newpackages(): - if s.name.startswith("fedora-release"): - # Relying on the F27 metadata injects irrelevant fedora-release deps - continue if s.arch in ("src", "nosrc"): continue # Ensure the solvables don't outlive the solver that created them by From 431c571dd67e39d0223a065830f65b07d46e499f Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 20:25:01 +0000 Subject: [PATCH 4/6] Move from F27 to F28 Change the default stream names to f28, and pull package information from the branched f28 repositories. --- diff --git a/README.md b/README.md index fadf515..db00f02 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Currently, this consists of: * `fedmod rpm2module`: generates a draft modulemd file based on the given RPM name (multiple RPM names can be given, but the resulting draft module will lack any descriptive metadata in that case) -* `fedmod fetch-metadata`: download the F27 package and module metadata needed +* `fedmod fetch-metadata`: download the F28 package and module metadata needed to generate draft module definitions (the metadata sets to use are not yet configurable) diff --git a/src/README.md b/src/README.md index 7533dd7..85b4255 100644 --- a/src/README.md +++ b/src/README.md @@ -13,7 +13,7 @@ Currently, this consists of: * `fedmod rpm2module`: generates a draft modulemd file based on the given RPM name (multiple RPM names can be given, but the resulting draft module will lack any descriptive metadata in that case) -* `fedmod fetch-metadata`: download the F27 package and module metadata needed +* `fedmod fetch-metadata`: download the F28 package and module metadata needed to generate draft module definitions (the metadata sets to use are not yet configurable) @@ -173,7 +173,7 @@ module) The following metadata is currently used as input to the draft module generation process: -* Package dependency definitions are pulled from the regular Fedora 27 +* Package dependency definitions are pulled from the regular Fedora 28 release and updates repositories, with the metadata being downloaded for local use via the `fedmod fetch-metadata` command @@ -198,6 +198,6 @@ Other limitations in generated `modulemd` files: * `components` are only given a name and rationale, relying on the default settings for everything else -* the stream for module level dependencies is currently hardcoded to `f27`. +* the stream for module level dependencies is currently hardcoded to `f28`. This isn't right, but we can't set anything better until the mechanism for depending on multiple streams without naming them specifically is defined. diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py index 28f415e..0e7e6d2 100644 --- a/src/_fedmod/_fetchrepodata.py +++ b/src/_fedmod/_fetchrepodata.py @@ -20,11 +20,11 @@ CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") log = logging.getLogger(__name__) FALLBACK_STREAM = 'master' -STREAM = 'f27' +STREAM = 'f28' ARCH = 'x86_64' -_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" -_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" +_F28_MODULAR_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/28/Modular/" +_F28_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/28/Everything/" +_F28_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/28/Everything/" class MissingMetadata(Exception): """Reports failure to find the local metadata cache""" @@ -56,12 +56,12 @@ def _define_repo(remote_prefix, local_cache_name, arch=None): return RepoPaths(remote_repo_url, remote_metadata_url, local_cache_path, local_metadata_path) -_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) -_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") -_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) -_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") -_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) -_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") +_x86_64_MODULE_INFO = _define_repo(_F28_MODULAR_REPO, "f28-modular", ARCH) +_SOURCE_MODULE_INFO = _define_repo(_F28_MODULAR_REPO, "f28-modular") +_x86_64_PACKAGE_INFO = _define_repo(_F28_MAIN_REPO, "f28-packages", ARCH) +_SOURCE_PACKAGE_INFO = _define_repo(_F28_MAIN_REPO, "f28-packages") +_x86_64_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates", ARCH) +_SOURCE_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates") _ALL_REPOS = ( _x86_64_MODULE_INFO, _SOURCE_MODULE_INFO, @@ -72,9 +72,9 @@ _ALL_REPOS = ( ) _LOOKUP_CACHES = { - "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-module-contents-cache.json"), - "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-srpm-to-module-cache.json"), - "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-rpm-to-module-cache.json"), + "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f28-module-contents-cache.json"), + "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f28-srpm-to-module-cache.json"), + "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f28-rpm-to-module-cache.json"), } METADATA_SECTIONS = ("filelists", "primary", "modules") @@ -194,8 +194,8 @@ class LocalMetadataCache: def load_cached_repodata(dataset_name): - if dataset_name != "f27-bikeshed-x86_64": - raise RuntimeError("Data sets other than 'f27-bikeshed-x86_64' are not yet supported") + if dataset_name != "f28-x86_64": + raise RuntimeError("Data sets other than 'f28-x86_64' are not yet supported") # Check whether or not fetch-metadata has been run at all metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 718043b..43ea1a7 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -13,7 +13,7 @@ from ._fetchrepodata import load_cached_repodata log = logging.getLogger(__name__) _ACTIVE_DATASET = None -_DEFAULT_DATASET_NAME = "f27-bikeshed-x86_64" +_DEFAULT_DATASET_NAME = "f28-x86_64" def _load_dataset(dataset_name): global _ACTIVE_DATASET diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index ed5b4bb..d109fe0 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -103,9 +103,9 @@ class ModuleGenerator(object): # Declare module level dependencies for modname in self.module_build_deps: - self.mmd.buildrequires[modname] = "f27" + self.mmd.buildrequires[modname] = "f28" for modname in self.module_run_deps: - self.mmd.requires[modname] = "f27" + self.mmd.requires[modname] = "f28" # Add any other RPMs not available from existing modules as components for pkg in self.build_and_run_srpms: diff --git a/tests/test_module_repoquery.py b/tests/test_module_repoquery.py index b8b355a..dde2ed3 100644 --- a/tests/test_module_repoquery.py +++ b/tests/test_module_repoquery.py @@ -13,8 +13,8 @@ class TestListingModules(object): self.mr.list_modules() out, err = capfd.readouterr() - assert "platform" in out - assert "host" in out + assert "reviewboard" in out + assert "django" in out class TestListingPackages(object): @@ -26,17 +26,17 @@ class TestListingPackages(object): self.mr.list_modularized_pkgs() out, err = capfd.readouterr() - assert "kernel" in out - assert "gcc" in out - assert "(platform)" not in out + assert "ReviewBoard" in out + assert "python2-django" in out + assert "(reviewboard)" not in out def test_list_packages_with_modules(self, capfd): self.mr.list_modularized_pkgs(list_modules=True) out, err = capfd.readouterr() - assert "kernel" in out - assert "gcc" in out - assert "(platform)" in out + assert "ReviewBoard" in out + assert "python2-django" in out + assert "(reviewboard)" in out class TestResolvingDependencies(object): @@ -45,18 +45,19 @@ class TestResolvingDependencies(object): self.mr = ModuleRepoquery() def test_list_pkg_deps(self, capfd): - self.mr.list_pkg_deps(["nginx"], []) + self.mr.list_pkg_deps(["timeline"], []) out, err = capfd.readouterr() - assert "gzip" in out - assert "nginx" in out + assert "python2-markdown" in out + assert "python2-setuptools" in out def test_list_pkg_deps_with_module_deps(self, capfd): - self.mr.list_pkg_deps(["nginx"], ["platform", "host"]) + self.mr.list_pkg_deps(["timeline"], ["reviewboard"]) out, err = capfd.readouterr() - assert "gzip" not in out - assert "nginx" in out + assert "python2-markdown" not in out + assert "python2-setuptools" in out + class TestListingPackagesInModule(object): @@ -64,11 +65,10 @@ class TestListingPackagesInModule(object): self.mr = ModuleRepoquery() def test_list_rpms_in_module(self, capfd): - self.mr.list_rpms_in_module("host") + self.mr.list_rpms_in_module("reviewboard") out, err = capfd.readouterr() - assert "kernel" in out - assert "httpd" not in out + assert "ReviewBoard" in out class TestListingModulesWithPackage(object): @@ -76,11 +76,10 @@ class TestListingModulesWithPackage(object): self.mr = ModuleRepoquery() def test_list_modules_for_rpm(self, capfd): - self.mr.list_modules_for_rpm("kernel") + self.mr.list_modules_for_rpm("ReviewBoard") out, err = capfd.readouterr() - assert "host" in out - assert "httpd" not in out + assert "reviewboard" in out class TestGettingSRPMOfRPM(object): From 4087dc06b47d42a221b6a2402668736501e92766 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 20:25:01 +0000 Subject: [PATCH 5/6] Comment out references to updates repo Prior to the release of F28, there's no content in the F28 updates repo, and it's not clear if the structure of the current F28 updates repo is final, or will change further. --- diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py index 0e7e6d2..2fbce41 100644 --- a/src/_fedmod/_fetchrepodata.py +++ b/src/_fedmod/_fetchrepodata.py @@ -24,7 +24,8 @@ STREAM = 'f28' ARCH = 'x86_64' _F28_MODULAR_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/28/Modular/" _F28_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/28/Everything/" -_F28_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/28/Everything/" +# F28 updates repo is empty until F28 is released, and the structure is still evolving +#_F28_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/28/Everything/" class MissingMetadata(Exception): """Reports failure to find the local metadata cache""" @@ -60,15 +61,15 @@ _x86_64_MODULE_INFO = _define_repo(_F28_MODULAR_REPO, "f28-modular", ARCH) _SOURCE_MODULE_INFO = _define_repo(_F28_MODULAR_REPO, "f28-modular") _x86_64_PACKAGE_INFO = _define_repo(_F28_MAIN_REPO, "f28-packages", ARCH) _SOURCE_PACKAGE_INFO = _define_repo(_F28_MAIN_REPO, "f28-packages") -_x86_64_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates", ARCH) -_SOURCE_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates") +#_x86_64_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates", ARCH) +#_SOURCE_UPDATES_INFO = _define_repo(_F28_UPDATES_REPO, "f28-updates") _ALL_REPOS = ( _x86_64_MODULE_INFO, _SOURCE_MODULE_INFO, _x86_64_PACKAGE_INFO, _SOURCE_PACKAGE_INFO, - _x86_64_UPDATES_INFO, - _SOURCE_UPDATES_INFO, +# _x86_64_UPDATES_INFO, +# _SOURCE_UPDATES_INFO, ) _LOOKUP_CACHES = { @@ -189,8 +190,8 @@ class LocalMetadataCache: module_to_packages = attrib(dict) source_repo_cache = attrib(str) arch_repo_cache = attrib(str) - source_updates_cache = attrib(str) - arch_updates_cache = attrib(str) +# source_updates_cache = attrib(str) +# arch_updates_cache = attrib(str) def load_cached_repodata(dataset_name): @@ -217,6 +218,6 @@ def load_cached_repodata(dataset_name): module_to_packages = _read_cache("_MODULE_FORWARD_LOOKUP_CACHE"), source_repo_cache = _SOURCE_PACKAGE_INFO.local_cache_path, arch_repo_cache = _x86_64_PACKAGE_INFO.local_cache_path, - source_updates_cache = _SOURCE_UPDATES_INFO.local_cache_path, - arch_updates_cache = _x86_64_UPDATES_INFO.local_cache_path, +# source_updates_cache = _SOURCE_UPDATES_INFO.local_cache_path, +# arch_updates_cache = _x86_64_UPDATES_INFO.local_cache_path, ) diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 43ea1a7..950733b 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -275,7 +275,8 @@ def setup_repos(): srcrepo = Repo("distro-source", dataset.source_repo_cache) repo = Repo("distro", dataset.arch_repo_cache) repo.srcrepo = srcrepo - updates_srcrepo = Repo("distro-updates-source", dataset.source_updates_cache) - updates_repo = Repo("distro-updates", dataset.arch_updates_cache) - updates_repo.srcrepo = updates_srcrepo - return [repo, srcrepo, updates_repo, updates_srcrepo] + # updates_srcrepo = Repo("distro-updates-source", dataset.source_updates_cache) + # updates_repo = Repo("distro-updates", dataset.arch_updates_cache) + # updates_repo.srcrepo = updates_srcrepo + # return [repo, srcrepo, updates_repo] + return [repo, srcrepo] From 8e4fb14d40ec81863867e9fc7e0a97afb88e6c98 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Mar 16 2018 21:11:02 +0000 Subject: [PATCH 6/6] Strip down module generation With hybrid modularity, we don't need to do complicated build-dependency chasing - build dependencies can simply come from the base package set. For runtime dependencies, still look for matches in modules, but if not found, just assume that these dependencies will come from the base package set. --- diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index 8ecc1b5..f8e1bae 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -51,13 +51,11 @@ def fetch_metadata(): # modulemd generation @_cli_commands.command() @click.option("--output", "-o", metavar="FILE", help="Write to FILE instead of stdout.") -@click.option("--build-deps", metavar="N", default=0, - help="Attempt to ensure N levels of build dependencies (Default: 0).") @click.argument("pkgs", metavar='PKGS', nargs=-1, required=True) -def rpm2module(pkgs, output, build_deps): +def rpm2module(pkgs, output): """Generate a draft modulemd from an RPM (or list of RPMs)""" mg = ModuleGenerator(pkgs) - mg.run(output, build_deps) + mg.run(output) # Checking availability of dependencies through module streams @_cli_commands.command('resolve-deps') diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index d109fe0..054ceb0 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -26,42 +26,18 @@ class ModuleGenerator(object): self.mmd = modulemd.ModuleMetadata() self._pool = _depchase.make_pool("x86_64") - def _calculate_dependencies(self, build_deps_iterations): + def _calculate_dependencies(self): pkgs = self.pkgs pool = self._pool self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} run_deps = _depchase.ensure_installable(pool, pkgs) module_run_deps, rpm_run_deps = _categorise_deps(run_deps) + # The platform module provides any other runtime dependencies - we expect we'll + # always depend on it, but check for the heck of it. + if len(rpm_run_deps) > 0: + module_run_deps.add('platform') self.module_run_deps = module_run_deps - run_srpms, build_deps = _depchase.ensure_buildable(pool, rpm_run_deps) - module_build_deps = set() - resolved_build_deps = set() - build_srpms = set() - for i in range(build_deps_iterations+1): - # Give up on making the module self-hosting after the requested - # number of iterations - new_module_build_deps, remaining_build_deps = _categorise_deps(build_deps) - module_build_deps |= new_module_build_deps - resolved_build_deps |= (build_deps - remaining_build_deps) - build_deps -= resolved_build_deps - if build_deps and i < build_deps_iterations: - new_build_srpms, remaining_build_deps = _depchase.ensure_buildable(pool, build_deps) - build_srpms |= new_build_srpms - resolved_build_deps |= (build_deps - remaining_build_deps) - build_deps -= resolved_build_deps - if not build_deps: - break - else: - if build_deps_iterations: - logging.warn(f"Failed to close out build dependencies after {build_deps_iterations} iterations") - self.module_run_deps = module_run_deps - self.module_build_deps = module_build_deps - run_srpm_names = {_name_only(n) for n in run_srpms} - build_srpm_names = {_name_only(n) for n in build_srpms} - self.run_srpms = run_srpm_names - build_srpm_names - self.build_srpms = build_srpm_names - run_srpm_names - self.build_and_run_srpms = run_srpm_names & build_srpm_names - self.unresolved_build_rpms = {n for n in build_deps} + self.run_srpms = set() def _set_core_srpm(self): """Set core SRPM based on first listed package""" @@ -102,25 +78,13 @@ class ModuleGenerator(object): self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) # Declare module level dependencies - for modname in self.module_build_deps: - self.mmd.buildrequires[modname] = "f28" for modname in self.module_run_deps: + self.mmd.buildrequires[modname] = "f28" self.mmd.requires[modname] = "f28" - # Add any other RPMs not available from existing modules as components - for pkg in self.build_and_run_srpms: - self.mmd.components.add_rpm(pkg, "Build and runtime dependency.", buildorder=self._get_build_order(pkg)) - for pkg in self.run_srpms: self.mmd.components.add_rpm(pkg, "Runtime dependency.", buildorder=self._get_build_order(pkg)) - for pkg in self.build_srpms: - self.mmd.components.add_rpm(pkg, "Build dependency.", buildorder=self._get_build_order(pkg)) - # Filter out any build-only packages - # TODO: This won't filter out all the RPMs, only the one matching the SRPM name - # See https://pagure.io/modulemd/issue/54 for discussion - self.mmd.filter.add_rpm(pkg) - # TODO: Always set content licenses appropriately # TODO: Emit something for non-empty self.unresolved_build_rpms @@ -133,9 +97,9 @@ class ModuleGenerator(object): else: return 0 - def run(self, output_fname, build_deps_iterations): + def run(self, output_fname): if len(self.pkgs) == 1: self._set_core_srpm() - self._calculate_dependencies(build_deps_iterations) + self._calculate_dependencies() self._update_module_md() self._save_module_md(output_fname) diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index aba7a8a..f5b9d3a 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -7,10 +7,8 @@ import modulemd from _fedmod.cli import _cli_commands from _fedmod.module_generator import ModuleGenerator -def _generate_modulemd(rpms, build_deps_iterations=0): +def _generate_modulemd(rpms): cmd = ['rpm2module'] - if build_deps_iterations: - cmd.extend(['--build-deps', str(build_deps_iterations)]) cmd.extend(rpms) runner = CliRunner() result = runner.invoke(_cli_commands, cmd) @@ -41,7 +39,7 @@ class TestSinglePackageInput(object): assert set(modmd.components.rpms) == set(input_rpms) # Expected module dependencies for grep - assert set(modmd.buildrequires) == set() + assert set(modmd.buildrequires) == {'platform',} assert set(modmd.requires) == {'platform',} @@ -71,133 +69,3 @@ class TestMultiplePackageInput(object): assert set(modmd.requires) == {'platform',} -class TestRecursiveBuildDeps(object): - - def test_generated_modulemd_file(self): - input_rpms = ('mariadb',) - modmd = _generate_modulemd(input_rpms, 100) - - # Descriptive metadata - assert modmd.summary == "Generated module for mariadb" - assert modmd.description == "Module auto-generated by fedmod" - - # Expected licenses - assert modmd.module_licenses == {'MIT'} - assert modmd.content_licenses == set() - - # Only given modules are listed in the public API - assert sorted(modmd.api.rpms) == sorted(input_rpms) - - # MariaDB's complicated test suite poses some real challenges for - # build dependency resolution, even when the generator is kind of - # cheating and relying on the actual Fedora MariaDB module at runtime - - # Expected components - expected_components = set(input_rpms) - expected_components |= { - 'systemtap', - 'libselinux', - 'numpy', - 'Judy', - 'setools', - 'pyparsing', - 'boost', - 'libsemanage', - 'jemalloc', - } - assert set(modmd.components.rpms) == expected_components - - # Expected module dependencies - expected_build_deps = { - 'platform', - 'mariadb', - 'networking-base', - 'host', - 'perl', - 'installer', - } - assert set(modmd.buildrequires) == expected_build_deps - assert set(modmd.requires) == {'platform', 'perl', 'mariadb'} - - -class TestNonRecursiveBuildDeps(object): - - def test_generated_modulemd_file(self): - input_rpms = ('graphite-web',) - modmd = _generate_modulemd(input_rpms) - - # Descriptive metadata - assert modmd.summary == "Generated module for graphite-web" - assert modmd.description == "Module auto-generated by fedmod" - - # Expected licenses - assert modmd.module_licenses == {'MIT'} - assert modmd.content_licenses == set() - - # Only given modules are listed in the public API - assert sorted(modmd.api.rpms) == sorted(input_rpms) - - # Expected components - expected_components = set(input_rpms) - expected_components |= { - 'python-twisted', - 'python-simplejson', - 'python-service-identity', - 'python-crypto', - 'python-xpyb', - 'python-memcached', - 'python-zope-interface', - 'python-whitenoise', - 'python-whisper', - 'pyparsing', - 'dejavu-fonts', - 'python-zope-event', - 'python-django', - 'python-django-tagging', - 'python-attrs', - 'python-carbon', - 'python-fadvise', - 'pyserial', - } - assert set(modmd.components.rpms) == expected_components - - # Expected module dependencies - # graphite-web is currently generating more runtime module dependencies - # than expected, so it makes for an interesting test case to look for - # cases where fedmod is picking up module implementation details - - expected_build_requires = { - 'platform', - 'fonts', - 'python2', - 'python2-ecosystem', - } - _unexpected_build_requires = { - 'pki', - 'samba', - 'python3', - 'python3-ecosystem', - 'installer', - 'X11-base', - 'freeipa', - 'perl', - 'fonts', - 'freeipa', - 'java', - 'host', - 'networking-base', - } - assert set(modmd.buildrequires) == expected_build_requires | _unexpected_build_requires - expected_requires = { - 'platform', - 'httpd', - 'fonts', - } - _unexpected_requires = { - 'pki', - 'samba', - 'installer', - 'X11-base', - 'freeipa', - } - assert set(modmd.requires) == expected_requires | _unexpected_requires