From b64bf836d122220dd2ecdd2f056477ea3a015511 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Dec 04 2017 06:05:58 +0000 Subject: Separate metadata fetching to its own file This chance also introduces a "LocalMetadataCache" object, and switches the repo querying to all work on the notion of an "active data set" that avoids any assumptions about the filesystem cache layout. While it retains the single architecture assumption, as while as the "only base + updates" repository structure, it removes the hardcoding from the query layer (it's still hardcoded in the download layer for now) --- diff --git a/src/_fedmod/_depchase.py b/src/_fedmod/_depchase.py index 4adeea7..065855e 100644 --- a/src/_fedmod/_depchase.py +++ b/src/_fedmod/_depchase.py @@ -1,3 +1,5 @@ +""""_depchase: Resolve dependency & build relationships between RPMs and SRPMs""" + import configparser import itertools import logging diff --git a/src/_fedmod/_fetchrepodata.py b/src/_fedmod/_fetchrepodata.py new file mode 100644 index 0000000..3b678aa --- /dev/null +++ b/src/_fedmod/_fetchrepodata.py @@ -0,0 +1,244 @@ +"""_fetchrepodata: Map yum/dnf repo metadata to local lookup caches""" +import json +import logging +import os +from collections import defaultdict + +import click +import modulemd +import requests + +from attr import attributes, attrib +from lxml import etree +from requests_toolbelt.downloadutils.tee import tee_to_file +from urllib.parse import urljoin + +XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") +CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") + +log = logging.getLogger(__name__) + +FALLBACK_STREAM = 'master' +STREAM = 'f27' +ARCH = 'x86_64' +_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" +_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" +_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" +_F27_BOOTSTRAP_MODULEMD = "https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml" + +class MissingMetadata(Exception): + """Reports failure to find the local metadata cache""" + +@attributes +class RepoPaths: + remote_repo_url = attrib(str) + remote_metadata_url = attrib(str) + local_cache_path = attrib(str) + local_metadata_path = attrib(str) + +def _define_repo(remote_prefix, local_cache_name, arch=None): + if arch is None: + local_arch_path = "source" + if "updates" in remote_prefix: + remote_arch_path = "SRPMS" + else: + remote_arch_path = "source/tree/" + else: + local_arch_path = arch + if "updates" in remote_prefix: + remote_arch_path = arch + else: + remote_arch_path = os.path.join(arch, "os/") + remote_repo_url = os.path.join(remote_prefix, remote_arch_path) + remote_metadata_url = os.path.join(remote_repo_url, "repodata/") + local_cache_path = os.path.join(CACHEDIR, "repos", local_cache_name, local_arch_path) + local_metadata_path = os.path.join(local_cache_path, "repodata/") + return RepoPaths(remote_repo_url, remote_metadata_url, + local_cache_path, local_metadata_path) + +_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) +_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") +_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) +_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") +_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) +_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") +_ALL_REPOS = ( + _x86_64_MODULE_INFO, + _SOURCE_MODULE_INFO, + _x86_64_PACKAGE_INFO, + _SOURCE_PACKAGE_INFO, + _x86_64_UPDATES_INFO, + _SOURCE_UPDATES_INFO, +) +_BOOTSTRAP_MODULEMD = os.path.join(CACHEDIR, "f27-bootstrap.yaml") + +_LOOKUP_CACHES = { + "_BOOTSTRAP_COMPONENTS_CACHE": os.path.join(CACHEDIR, "f27-bootstrap-cache.json"), + "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-module-contents-cache.json"), + "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-srpm-to-module-cache.json"), + "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-rpm-to-module-cache.json"), +} + +METADATA_SECTIONS = ("filelists", "primary", "modules") + +_REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} +def _read_repomd_location(repomd_xml, section): + location = repomd_xml.find(f"rpm:data[@type='{section}']/rpm:location", _REPOMD_XML_NAMESPACE) + if location is not None: + return location.attrib["href"] + return None + +def _download_one_file(remote_url, filename): + if os.path.exists(filename) and not filename.endswith((".xml", ".yaml")): + print(f" Skipping download; {filename} already exists") + return + response = requests.get(remote_url, stream=True) + try: + print(f" Downloading {remote_url}") + chunksize = 65536 + expected_chunks = int(response.headers["content-length"]) / chunksize + downloader = tee_to_file(response, filename=filename, chunksize=chunksize) + show_progress = click.progressbar(downloader, length=expected_chunks) + with show_progress: + for chunk in show_progress: + pass + finally: + response.close() + print(f" Added {filename} to cache") + +def _download_metadata_files(repo_paths): + local_path = repo_paths.local_cache_path + local_metadata_path = repo_paths.local_metadata_path + os.makedirs(local_metadata_path, exist_ok=True) + repomd_url = urljoin(repo_paths.remote_metadata_url, "repomd.xml") + print(f"Remote metadata: {repomd_url}") + response = requests.get(repomd_url) + response.raise_for_status() + repomd_filename = os.path.join(local_metadata_path, "repomd.xml") + with open(repomd_filename, "wb") as f: + f.write(response.content) + print(f" Cached metadata in {repomd_filename}") + repomd_xml = etree.parse(repomd_filename) + files_to_fetch = set() + for section in METADATA_SECTIONS: + relative_href = _read_repomd_location(repomd_xml, section) + if relative_href is not None: + files_to_fetch.add(relative_href) + predownload = set(os.listdir(local_path)) + for relative_href in files_to_fetch: + absolute_href = urljoin(repo_paths.remote_repo_url, relative_href) + filename = os.path.join(local_path, relative_href) + # This could be parallelised with concurrent.futures, but + # probably not worth it (it makes the progress bars trickier) + _download_one_file(absolute_href, filename) + postdownload = set(os.listdir(local_path)) + # Prune any old metadata files automatically + if len(postdownload) >= (len(predownload) + len(METADATA_SECTIONS)): + # TODO: Actually prune old metadata files + pass + +def _write_cache(cache_name, data): + """Write the given data to the nominated cache file""" + cache_fname = _LOOKUP_CACHES[cache_name] + with open(cache_fname, "w") as cache_file: + json.dump(data, cache_file) + print(f" Added {cache_fname} to cache") + +def _read_cache(cache_name): + """Read the parsed data from the nominated cache file""" + cache_fname = _LOOKUP_CACHES[cache_name] + with open(cache_fname, "r") as cache_file: + return json.load(cache_file) + +def _download_bootstrap_modulemd(): + from ._depchase import make_pool, get_rpms_for_srpms + print("Downloading build bootstrap module details") + _download_one_file(_F27_BOOTSTRAP_MODULEMD, _BOOTSTRAP_MODULEMD) + # TODO: Cache the modulemd file hash, and only regenerate the cache + # if that has changed + mmd = modulemd.ModuleMetadata() + mmd.load(_BOOTSTRAP_MODULEMD) + pool = make_pool("x86_64") + bootstrap_rpms = set() + rpms = get_rpms_for_srpms(pool, mmd.components.rpms) + for rpmname in rpms: + bootstrap_rpms.add(rpmname) + for srpmname in mmd.components.rpms: + bootstrap_rpms.add(srpmname) + _write_cache("_BOOTSTRAP_COMPONENTS_CACHE", list(bootstrap_rpms)) + +def _write_lookup_caches(): + metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + repomd_xml = etree.parse(repomd_fname) + repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") + repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) + with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + modules_yaml = modules_yaml_gz.read() + modules = modulemd.loads_all(modules_yaml) + module_forward_lookup = {} + srpm_reverse_lookup = defaultdict(list) + rpm_reverse_lookup = defaultdict(list) + for module in modules: + module_forward_lookup[module.name] = list(set(module.artifacts.rpms)) + for srpmname in module.components.rpms: + srpm_reverse_lookup[srpmname].append(module.name) + for rpmname in module.artifacts.rpms: + rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] + rpm_reverse_lookup[rpmprefix].append(module.name) + # Cache the lookup tables as local JSON files + print("Caching lookup tables") + _write_cache("_MODULE_FORWARD_LOOKUP_CACHE", module_forward_lookup) + _write_cache("_SRPM_REVERSE_LOOKUP_CACHE", srpm_reverse_lookup) + _write_cache("_RPM_REVERSE_LOOKUP_CACHE", rpm_reverse_lookup) + + +def download_repo_metadata(): + """Downloads the latest repo metadata""" + for repo_definition in _ALL_REPOS: + _download_metadata_files(repo_definition) + _download_bootstrap_modulemd() + _write_lookup_caches() + +@attributes +class LocalMetadataCache: + dataset_name = attrib(str) + cache_dir = attrib(str) + srpm_to_modules = attrib(dict) + rpm_to_modules = attrib(dict) + bootstrap_components = attrib(set) + module_to_packages = attrib(dict) + source_repo_cache = attrib(str) + arch_repo_cache = attrib(str) + source_updates_cache = attrib(str) + arch_updates_cache = attrib(str) + + +def load_cached_repodata(dataset_name): + if dataset_name != "f27-bikeshed-x86_64": + raise RuntimeError("Data sets other than 'f27-bikeshed-x86_64' are not yet supported") + # Check whether or not fetch-metadata has been run at all + metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + if not os.path.exists(repomd_fname): + msg = f"{repomd_fname!r} does not exist. Run `fedmod fetch-metadata`." + raise MissingMetadata(msg) + # Check whether or not fetch-metadata actually finished + for cache_entry in _LOOKUP_CACHES.values(): + if not os.path.exists(cache_entry): + msg = (f"{cache_entry!r} does not exist. " + "Try running `fedmod fetch-metadata` again.") + raise MissingMetadata(msg) + # Load the metadata + return LocalMetadataCache( + dataset_name = dataset_name, + cache_dir = CACHEDIR, + srpm_to_modules = _read_cache("_SRPM_REVERSE_LOOKUP_CACHE"), + rpm_to_modules = _read_cache("_RPM_REVERSE_LOOKUP_CACHE"), + bootstrap_components = _read_cache("_BOOTSTRAP_COMPONENTS_CACHE"), + module_to_packages = _read_cache("_MODULE_FORWARD_LOOKUP_CACHE"), + source_repo_cache = _SOURCE_PACKAGE_INFO.local_cache_path, + arch_repo_cache = _x86_64_PACKAGE_INFO.local_cache_path, + source_updates_cache = _SOURCE_UPDATES_INFO.local_cache_path, + arch_updates_cache = _x86_64_UPDATES_INFO.local_cache_path, + ) diff --git a/src/_fedmod/_repodata.py b/src/_fedmod/_repodata.py index 219eea5..9f3c718 100644 --- a/src/_fedmod/_repodata.py +++ b/src/_fedmod/_repodata.py @@ -1,4 +1,4 @@ -"""Helpers for metadata management""" +"""_repodata: Resolve module relationship queries against the local cache""" import sys import tempfile import os.path @@ -9,249 +9,44 @@ import logging import modulemd import solv import json -from collections import defaultdict -from fnmatch import fnmatch -from urllib.parse import urljoin -from attr import attributes, attrib -from requests_toolbelt.downloadutils.tee import tee_to_file -from lxml import etree - -XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") -CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") +from ._fetchrepodata import load_cached_repodata log = logging.getLogger(__name__) -FALLBACK_STREAM = 'master' -STREAM = 'f27' -ARCH = 'x86_64' -_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" -_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" -_F27_BOOTSTRAP_MODULEMD = "https://src.fedoraproject.org/modules/bootstrap/raw/master/f/bootstrap.yaml" - -class MissingMetadata(Exception): - """Reports failure to find the local metadata cache""" - -@attributes -class RepoPaths: - remote_repo_url = attrib(str) - remote_metadata_url = attrib(str) - local_cache_path = attrib(str) - local_metadata_path = attrib(str) - -def _define_repo(remote_prefix, local_cache_name, arch=None): - if arch is None: - local_arch_path = "source" - if "updates" in remote_prefix: - remote_arch_path = "SRPMS" - else: - remote_arch_path = "source/tree/" - else: - local_arch_path = arch - if "updates" in remote_prefix: - remote_arch_path = arch - else: - remote_arch_path = os.path.join(arch, "os/") - remote_repo_url = os.path.join(remote_prefix, remote_arch_path) - remote_metadata_url = os.path.join(remote_repo_url, "repodata/") - local_cache_path = os.path.join(CACHEDIR, "repos", local_cache_name, local_arch_path) - local_metadata_path = os.path.join(local_cache_path, "repodata/") - return RepoPaths(remote_repo_url, remote_metadata_url, - local_cache_path, local_metadata_path) - -_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) -_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") -_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) -_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") -_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) -_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") -_ALL_REPOS = ( - _x86_64_MODULE_INFO, - _SOURCE_MODULE_INFO, - _x86_64_PACKAGE_INFO, - _SOURCE_PACKAGE_INFO, - _x86_64_UPDATES_INFO, - _SOURCE_UPDATES_INFO, -) -_BOOTSTRAP_MODULEMD = os.path.join(CACHEDIR, "f27-bootstrap.yaml") - -_LOOKUP_CACHES = { - "_BOOTSTRAP_COMPONENTS_CACHE": os.path.join(CACHEDIR, "f27-bootstrap-cache.json"), - "_MODULE_FORWARD_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-module-contents-cache.json"), - "_SRPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-srpm-to-module-cache.json"), - "_RPM_REVERSE_LOOKUP_CACHE": os.path.join(CACHEDIR, "f27-rpm-to-module-cache.json"), -} - -METADATA_SECTIONS = ("filelists", "primary", "modules") - -_REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} -def _read_repomd_location(repomd_xml, section): - location = repomd_xml.find(f"rpm:data[@type='{section}']/rpm:location", _REPOMD_XML_NAMESPACE) - if location is not None: - return location.attrib["href"] - return None +_ACTIVE_DATASET = None +_DEFAULT_DATASET_NAME = "f27-bikeshed-x86_64" -def _download_one_file(remote_url, filename): - if os.path.exists(filename) and not filename.endswith((".xml", ".yaml")): - print(f" Skipping download; {filename} already exists") - return - response = requests.get(remote_url, stream=True) - try: - print(f" Downloading {remote_url}") - chunksize = 65536 - expected_chunks = int(response.headers["content-length"]) / chunksize - downloader = tee_to_file(response, filename=filename, chunksize=chunksize) - show_progress = click.progressbar(downloader, length=expected_chunks) - with show_progress: - for chunk in show_progress: - pass - finally: - response.close() - print(f" Added {filename} to cache") +def _load_dataset(dataset_name): + global _ACTIVE_DATASET + _ACTIVE_DATASET = load_cached_repodata(dataset_name) -def _download_metadata_files(repo_paths): - local_path = repo_paths.local_cache_path - local_metadata_path = repo_paths.local_metadata_path - os.makedirs(local_metadata_path, exist_ok=True) - repomd_url = urljoin(repo_paths.remote_metadata_url, "repomd.xml") - print(f"Remote metadata: {repomd_url}") - response = requests.get(repomd_url) - response.raise_for_status() - repomd_filename = os.path.join(local_metadata_path, "repomd.xml") - with open(repomd_filename, "wb") as f: - f.write(response.content) - print(f" Cached metadata in {repomd_filename}") - repomd_xml = etree.parse(repomd_filename) - files_to_fetch = set() - for section in METADATA_SECTIONS: - relative_href = _read_repomd_location(repomd_xml, section) - if relative_href is not None: - files_to_fetch.add(relative_href) - predownload = set(os.listdir(local_path)) - for relative_href in files_to_fetch: - absolute_href = urljoin(repo_paths.remote_repo_url, relative_href) - filename = os.path.join(local_path, relative_href) - # This could be parallelised with concurrent.futures, but - # probably not worth it (it makes the progress bars trickier) - _download_one_file(absolute_href, filename) - postdownload = set(os.listdir(local_path)) - # Prune any old metadata files automatically - if len(postdownload) >= (len(predownload) + len(METADATA_SECTIONS)): - # TODO: Actually prune old metadata files - pass - -def _write_cache(cache_name, data): - """Write the given data to the nominated cache file""" - cache_fname = _LOOKUP_CACHES[cache_name] - with open(cache_fname, "w") as cache_file: - json.dump(data, cache_file) - print(f" Added {cache_fname} to cache") - -def _read_cache(cache_name): - """Read the parsed data from the nominated cache file""" - cache_fname = _LOOKUP_CACHES[cache_name] - with open(cache_fname, "r") as cache_file: - return json.load(cache_file) - -def _download_bootstrap_modulemd(): - from ._depchase import make_pool, get_rpms_for_srpms - print("Downloading build bootstrap module details") - _download_one_file(_F27_BOOTSTRAP_MODULEMD, _BOOTSTRAP_MODULEMD) - # TODO: Cache the modulemd file hash, and only regenerate the cache - # if that has changed - mmd = modulemd.ModuleMetadata() - mmd.load(_BOOTSTRAP_MODULEMD) - pool = make_pool("x86_64") - bootstrap_rpms = set() - rpms = get_rpms_for_srpms(pool, mmd.components.rpms) - for rpmname in rpms: - bootstrap_rpms.add(rpmname) - for srpmname in mmd.components.rpms: - bootstrap_rpms.add(srpmname) - _write_cache("_BOOTSTRAP_COMPONENTS_CACHE", list(bootstrap_rpms)) - -def _write_lookup_caches(): - metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) - repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") - repomd_xml = etree.parse(repomd_fname) - repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") - repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) - with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: - modules_yaml = modules_yaml_gz.read() - modules = modulemd.loads_all(modules_yaml) - module_forward_lookup = {} - srpm_reverse_lookup = defaultdict(list) - rpm_reverse_lookup = defaultdict(list) - for module in modules: - module_forward_lookup[module.name] = list(set(module.artifacts.rpms)) - for srpmname in module.components.rpms: - srpm_reverse_lookup[srpmname].append(module.name) - for rpmname in module.artifacts.rpms: - rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] - rpm_reverse_lookup[rpmprefix].append(module.name) - # Cache the lookup tables as local JSON files - print("Caching lookup tables") - _write_cache("_MODULE_FORWARD_LOOKUP_CACHE", module_forward_lookup) - _write_cache("_SRPM_REVERSE_LOOKUP_CACHE", srpm_reverse_lookup) - _write_cache("_RPM_REVERSE_LOOKUP_CACHE", rpm_reverse_lookup) - - -def download_repo_metadata(): - """Downloads the latest repo metadata""" - for repo_definition in _ALL_REPOS: - _download_metadata_files(repo_definition) - _download_bootstrap_modulemd() - _write_lookup_caches() - - -_SRPM_REVERSE_LOOKUP = {} # SRPM name : [module names] -_RPM_REVERSE_LOOKUP = {} # RPM name : [module names] -_BOOTSTRAP_COMPONENTS = set() -_MODULE_FORWARD_LOOKUP = {} -def _populate_module_reverse_lookup(): - if _RPM_REVERSE_LOOKUP: - return - # Check whether or not fetch-metadata has been run at all - metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) - repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") - if not os.path.exists(repomd_fname): - msg = f"{repomd_fname!r} does not exist. Run `fedmod fetch-metadata`." - raise MissingMetadata(msg) - # Check whether or not fetch-metadata actually finished - for cache_entry in _LOOKUP_CACHES.values(): - if not os.path.exists(cache_entry): - msg = (f"{cache_entry!r} does not exist. " - "Try running `fedmod fetch-metadata` again.") - raise MissingMetadata(msg) - # Load the metadata - # TODO: Switch to lazy loading of the actual data - _SRPM_REVERSE_LOOKUP.update(_read_cache("_SRPM_REVERSE_LOOKUP_CACHE")) - _RPM_REVERSE_LOOKUP.update(_read_cache("_RPM_REVERSE_LOOKUP_CACHE")) - _BOOTSTRAP_COMPONENTS.update(_read_cache("_BOOTSTRAP_COMPONENTS_CACHE")) - _MODULE_FORWARD_LOOKUP.update(_read_cache("_MODULE_FORWARD_LOOKUP_CACHE")) +def _get_dataset(): + if _ACTIVE_DATASET is None: + _load_dataset(_DEFAULT_DATASET_NAME) + return _ACTIVE_DATASET def list_modules(): - return _MODULE_FORWARD_LOOKUP.keys() + return _get_dataset().module_to_packages.keys() def get_rpms_in_module(module_name): - return _MODULE_FORWARD_LOOKUP.get(module_name, []) + return _get_dataset().module_to_packages.get(module_name, []) def get_modules_for_rpm(rpm_name): - result = _RPM_REVERSE_LOOKUP.get(rpm_name) + result = _get_dataset().rpm_to_modules.get(rpm_name) return result def get_module_for_rpm(rpm_name, *, allow_bootstrap=False): - result = _RPM_REVERSE_LOOKUP.get(rpm_name) + result = _get_dataset().rpm_to_modules.get(rpm_name) if result is not None: if len(result) > 1: log.warn(f"Multiple modules found for {rpm_name!r}: {','.join(result)}") result = result[0] - elif allow_bootstrap and rpm_name in _BOOTSTRAP_COMPONENTS: + elif allow_bootstrap and rpm_name in _get_dataset().bootstrap_components: result = "bootstrap" return result def get_rpm_reverse_lookup(): - return _RPM_REVERSE_LOOKUP + return _get_dataset().rpm_to_modules class Repo(object): def __init__(self, name, metadata_path): @@ -283,7 +78,7 @@ class Repo(object): path = "{}-{}.solvx".format(path, ext) else: path = "{}.solv".format(path) - return os.path.join(CACHEDIR, path.replace("/", "_")) + return os.path.join(_get_dataset().cache_dir, path.replace("/", "_")) def usecachedrepo(self, ext, mark=False): try: @@ -326,9 +121,8 @@ class Repo(object): def writecachedrepo(self, ext, repodata=None): tmpname = None try: - if not os.path.isdir(CACHEDIR): - os.mkdir(CACHEDIR, 0o755) - fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) + fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", + dir=_get_dataset().cache_dir) os.fchmod(fd, 0o444) f = os.fdopen(fd, "wb+") f = solv.xfopen_fd(None, f.fileno()) @@ -480,11 +274,11 @@ def load_stub(repodata): return False def setup_repos(): - - srcrepo = Repo("f27-source", _SOURCE_PACKAGE_INFO.local_cache_path) - repo = Repo("f27", _x86_64_PACKAGE_INFO.local_cache_path) + dataset = _get_dataset() + srcrepo = Repo("distro-source", dataset.source_repo_cache) + repo = Repo("distro", dataset.arch_repo_cache) repo.srcrepo = srcrepo - updates_srcrepo = Repo("f27-updates-source", _SOURCE_UPDATES_INFO.local_cache_path) - updates_repo = Repo("f27-updates", _x86_64_UPDATES_INFO.local_cache_path) + updates_srcrepo = Repo("distro-updates-source", dataset.source_updates_cache) + updates_repo = Repo("distro-updates", dataset.arch_updates_cache) updates_repo.srcrepo = updates_srcrepo return [repo, srcrepo, updates_repo, updates_srcrepo] diff --git a/src/_fedmod/cli.py b/src/_fedmod/cli.py index 93af2df..1eeedd3 100644 --- a/src/_fedmod/cli.py +++ b/src/_fedmod/cli.py @@ -4,7 +4,7 @@ import logging from .module_generator import ModuleGenerator from .module_repoquery import ModuleRepoquery -from . import _depchase, _repodata +from . import _depchase, _repodata, _fetchrepodata # fedmod uses click for argument parsing, but currently does its own # standardised exception handling. This also requires handling click's standard @@ -19,7 +19,7 @@ def run(): except click.ClickException as e: e.show() rc = e.exit_code - except _repodata.MissingMetadata as e: + except _fetchrepodata.MissingMetadata as e: print(e, file=sys.stderr) rc = 2 except Exception as e: @@ -43,7 +43,7 @@ def _cli_commands(verbose): @_cli_commands.command('fetch-metadata') def fetch_metadata(): """Fetch and cache required module and RPM metadata""" - _repodata.download_repo_metadata() + _fetchrepodata.download_repo_metadata() # modulemd generation diff --git a/src/_fedmod/module_generator.py b/src/_fedmod/module_generator.py index 32e7b2d..5b33dfd 100644 --- a/src/_fedmod/module_generator.py +++ b/src/_fedmod/module_generator.py @@ -30,7 +30,6 @@ class ModuleGenerator(object): def _calculate_dependencies(self, build_deps_iterations): pkgs = self.pkgs - _repodata._populate_module_reverse_lookup() pool = _depchase.make_pool("x86_64") self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} run_deps = _depchase.ensure_installable(pool, pkgs) diff --git a/src/_fedmod/module_repoquery.py b/src/_fedmod/module_repoquery.py index 93d3f1f..713b6d0 100644 --- a/src/_fedmod/module_repoquery.py +++ b/src/_fedmod/module_repoquery.py @@ -9,25 +9,22 @@ from . import _depchase, _repodata def _name_only(rpm_name): name, version, release = rpm_name.rsplit("-", 2) return name - + class ModuleRepoquery(object): - + def list_modules(self): - _repodata._populate_module_reverse_lookup() module_names = _repodata.list_modules() if module_names: for name in sorted(module_names): print(name) def list_modules_for_rpm(self, pkg): - _repodata._populate_module_reverse_lookup() module_names = _repodata.get_modules_for_rpm(pkg) if module_names: for name in sorted(module_names): print(name) def list_rpms_in_module(self, module, full_nevra=False): - _repodata._populate_module_reverse_lookup() rpm_names = _repodata.get_rpms_in_module(module) if rpm_names: for name in sorted(rpm_names): @@ -35,15 +32,14 @@ class ModuleRepoquery(object): print(name) else: print(_name_only(name)) - + def list_pkg_deps(self, pkgs, module_deps): - _repodata._populate_module_reverse_lookup() pkgs_in_modules = set() if module_deps: for module in module_deps: rpm_names = _repodata.get_rpms_in_module(module) pkgs_in_modules |= set(map(lambda x: _name_only(x), rpm_names)) - + pool = _depchase.make_pool("x86_64") run_deps = _depchase.ensure_installable(pool, pkgs) rpm_names = run_deps - pkgs_in_modules @@ -52,7 +48,6 @@ class ModuleRepoquery(object): print(name) def list_modularized_pkgs(self, duplicate_only=False, list_modules=False): - _repodata._populate_module_reverse_lookup() rpm_names = _repodata.get_rpm_reverse_lookup() if rpm_names: for name in sorted(rpm_names.keys()): diff --git a/tests/test_cli_ux.py b/tests/test_cli_ux.py index edca483..ce1710d 100644 --- a/tests/test_cli_ux.py +++ b/tests/test_cli_ux.py @@ -6,7 +6,7 @@ import re import sys import subprocess -from _fedmod._repodata import CACHEDIR +from _fedmod._fetchrepodata import CACHEDIR def _run_fedmod(args): # Run via the -m switch to ensure we get the expected version of fedmod