From 56226485d0c8211cf277016f8b9fa5655db2d310 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 26 2017 06:43:24 +0000 Subject: [PATCH 1/3] Split repo metadata management into its own module --- diff --git a/README.md b/README.md index 2ade88d..7981893 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Currently, this consists of: * `python -m fedmod rpm2module`: generates a draft modulemd file based on the given RPM name (multiple RPM names can be given, but the resulting draft module will lack any descriptive metadata in that case) -* `python -m fedmod metadata`: download the F27 Modular Server repository +* `python -m fedmod fetch-metadata`: download the F27 Modular Server repository metadata needed to generate draft module definitions ## Project status diff --git a/fedmod/_depchase.py b/fedmod/_depchase.py index c990cfa..25a52ff 100644 --- a/fedmod/_depchase.py +++ b/fedmod/_depchase.py @@ -1,356 +1,24 @@ -#!/usr/bin/python3 import configparser import itertools import logging import os import sys -import gzip import tempfile -import click import smartcols import solv -import modulemd -import requests -from requests_toolbelt.downloadutils.tee import tee_to_file -from fnmatch import fnmatch -from urllib.parse import urljoin -from bs4 import BeautifulSoup, SoupStrainer -from lxml import etree -XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") -CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") +from . import _repodata log = logging.getLogger(__name__) -FALLBACK_STREAM = 'master' -STREAM = 'f27' -ARCH = 'x86_64' -REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -REPO_METADATA_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/repodata/") -REPO_METADATA_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/repodata/") -LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27") -LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH) -LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source") - -METADATA_FILES = ("*-filelists.xml.gz", "*-primary.xml.gz", "*-modules.yaml.gz", "repomd.xml") - -def _download_one_file(remote_url, filename): - if os.path.exists(filename) and not filename.endswith("repomd.xml"): - print(f"Skipping download; {filename} already exists") - return - with requests.get(remote_url, stream=True) as response: - print(f"Downloading {remote_url}") - chunksize = 65536 - expected_chunks = int(response.headers["content-length"]) / chunksize - downloader = tee_to_file(response, filename=filename, chunksize=chunksize) - show_progress = click.progressbar(downloader, length=expected_chunks) - with show_progress: - for chunk in show_progress: - pass - print(f"Added {filename} to cache") - -def _download_metadata_files(metadata_url, local_path): - os.makedirs(os.path.join(local_path, "repodata"), exist_ok=True) - response = requests.get(metadata_url) - response.raise_for_status() - link_filter = SoupStrainer("a", href=True) - metadata_links = BeautifulSoup(response.text, parse_only=link_filter, features="lxml") - patterns_to_check = set(METADATA_FILES) - files_to_fetch = set() - for link in metadata_links.find_all("a"): - href = link["href"] - for pattern in patterns_to_check: - if fnmatch(href, pattern): - patterns_to_check.remove(pattern) - files_to_fetch.add(href) - break # Go to next file - predownload = set(os.listdir(local_path)) - for relative_href in files_to_fetch: - absolute_href = urljoin(metadata_url, relative_href) - filename = os.path.join(local_path, "repodata", relative_href) - # This could be parallelised with concurrent.futures, but - # probably not worth it (it makes the progress bars trickier) - _download_one_file(absolute_href, filename) - postdownload = set(os.listdir(local_path)) - # Prune any old metadata files automatically - if len(postdownload) >= (len(predownload) + len(METADATA_FILES)): - # TODO: Actually prune old metadata files - pass - - -def download_repo_metadata(): - """Downloads the latest repo metadata""" - _download_metadata_files(REPO_METADATA_ARCH, LOCAL_REPO_INFO_ARCH) - _download_metadata_files(REPO_METADATA_SOURCE, LOCAL_REPO_INFO_SOURCE) - -_SRPM_REVERSE_LOOKUP = {} # SRPM name : module name -_RPM_REVERSE_LOOKUP = {} # RPM name : module name -def _populate_module_reverse_lookup(): - # TODO: Cache the reverse mapping in Repo instances, as with the solver data - metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH) - repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") - repomd_xml = etree.parse(repomd_fname) - repo_modulemd = repomd_xml.find("rpm:data[@type='modules']/rpm:location", {"rpm": "http://linux.duke.edu/metadata/repo"}) - if repo_modulemd is None: - raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?") - repo_modulemd_fname = os.path.join(metadata_dir, repo_modulemd.attrib["href"]) - with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: - modules_yaml = modules_yaml_gz.read() - modules = modulemd.loads_all(modules_yaml) - for module in modules: - for srpmname in module.components.rpms: - # This isn't entirely valid, as it doesn't account for multiple - # modules that include the same source RPM with different output - # filters (e.g. python3-ecosystem vs python2-ecosystem) - _SRPM_REVERSE_LOOKUP[srpmname] = module.name - for rpmname in module.artifacts.rpms: - # This is only valid for module sets that are guaranteed to be - # fully coinstallable, and hence only allow any given RPM to be - # published by at most one module - rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] - _RPM_REVERSE_LOOKUP[rpmprefix] = module.name - def get_module_for_rpm(rpm_name): - return _RPM_REVERSE_LOOKUP.get(rpm_name) - -class Repo(object): - def __init__(self, name, metadata_path): - self.name = name - self.metadata_path = metadata_path - self.handle = None - self.cookie = None - self.extcookie = None - self.srcrepo = None - - @staticmethod - def calc_cookie_fp(fp): - chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) - chksum.add("1.1") - chksum.add_fp(fp) - return chksum.raw() - - @staticmethod - def calc_cookie_ext(f, cookie): - chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) - chksum.add("1.1") - chksum.add(cookie) - chksum.add_fstat(f.fileno()) - return chksum.raw() - - def cachepath(self, ext=None): - path = "{}-{}".format(self.name.replace(".", "_"), self.metadata_path) - if ext: - path = "{}-{}.solvx".format(path, ext) - else: - path = "{}.solv".format(path) - return os.path.join(CACHEDIR, path.replace("/", "_")) - - def usecachedrepo(self, ext, mark=False): - try: - repopath = self.cachepath(ext) - f = open(repopath, "rb") - f.seek(-32, os.SEEK_END) - fcookie = f.read(32) - if len(fcookie) != 32: - return False - cookie = self.extcookie if ext else self.cookie - if cookie and fcookie != cookie: - return False - if not ext: - f.seek(-32 * 2, os.SEEK_END) - fextcookie = f.read(32) - if len(fextcookie) != 32: - return False - f.seek(0) - f = solv.xfopen_fd(None, f.fileno()) - flags = 0 - if ext: - flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES - if ext != "DL": - flags |= solv.Repo.REPO_LOCALPOOL - if not self.handle.add_solv(f, flags): - return False - if not ext: - self.cookie = fcookie - self.extcookie = fextcookie - if mark: - # no futimes in python? - try: - os.utime(repopath, None) - except Exception: - pass - except IOError: - return False - return True - - def writecachedrepo(self, ext, repodata=None): - tmpname = None - try: - if not os.path.isdir(CACHEDIR): - os.mkdir(CACHEDIR, 0o755) - fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) - os.fchmod(fd, 0o444) - f = os.fdopen(fd, "wb+") - f = solv.xfopen_fd(None, f.fileno()) - if not repodata: - self.handle.write(f) - elif ext: - repodata.write(f) - else: - # rewrite_repos case, do not write stubs - self.handle.write_first_repodata(f) - f.flush() - if not ext: - if not self.extcookie: - self.extcookie = self.calc_cookie_ext(f, self.cookie) - f.write(self.extcookie) - if not ext: - f.write(self.cookie) - else: - f.write(self.extcookie) - f.close - if self.handle.iscontiguous(): - # switch to saved repo to activate paging and save memory - nf = solv.xfopen(tmpname) - if not ext: - # main repo - self.handle.empty() - flags = solv.Repo.SOLV_ADD_NO_STUBS - if repodata: - # rewrite repos case, recreate stubs - flags = 0 - if not self.handle.add_solv(nf, flags): - sys.exit("internal error, cannot reload solv file") - else: - # extension repodata - # need to extend to repo boundaries, as this is how - # repodata.write() has written the data - repodata.extend_to_repo() - flags = solv.Repo.REPO_EXTEND_SOLVABLES - if ext != "DL": - flags |= solv.Repo.REPO_LOCALPOOL - repodata.add_solv(nf, flags) - os.rename(tmpname, self.cachepath(ext)) - except (OSError, IOError): - if tmpname: - os.unlink(tmpname) - - def load(self, pool): - assert not self.handle - self.handle = pool.add_repo(self.name) - self.handle.appdata = self - f = self.read_repo_metadata("repodata/repomd.xml", False, None) - if not f: - self.handle.free(True) - self.handle = None - return False - self.cookie = self.calc_cookie_fp(f) - if self.usecachedrepo(None, True): - return True - self.handle.add_repomdxml(f) - fname, fchksum = self.find("primary") - if not fname: - return False - f = self.read_repo_metadata(fname, True, fchksum) - if not f: - return False - self.handle.add_rpmmd(f, None) - self.add_exts() - self.writecachedrepo(None) - # Must be called after writing the repo - self.handle.create_stubs() - return True - - def read_repo_metadata(self, fname, uncompress, chksum): - f = open("{}/{}".format(self.metadata_path, fname)) - return solv.xfopen_fd(fname if uncompress else None, f.fileno()) - - def find(self, what): - di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING) - di.prepend_keyname(solv.REPOSITORY_REPOMD) - for d in di: - dp = d.parentpos() - filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION) - chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM) - if filename: - if not chksum: - print("No {} file checksum!".format(filename)) - return filename, chksum - return None, None - - def add_ext_keys(self, ext, repodata, handle): - if ext == "FL": - repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST) - repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY) - else: - raise NotImplementedError - - def add_ext(self, repodata, what, ext): - filename, chksum = self.find(what) - if not filename: - return - handle = repodata.new_handle() - repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what) - repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename) - repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum) - self.add_ext_keys(ext, repodata, handle) - repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle) - - def add_exts(self): - repodata = self.handle.add_repodata() - self.add_ext(repodata, "filelists", "FL") - repodata.internalize() - - def load_ext(self, repodata): - repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE) - if repomdtype == "filelists": - ext = "FL" - else: - assert False - if self.usecachedrepo(ext): - return True - filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION) - filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM) - f = self.read_repo_metadata(filename, True, filechksum) - if not f: - return False - if ext == "FL": - self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL) - self.writecachedrepo(ext, repodata) - return True - - def updateaddedprovides(self, addedprovides): - if self.handle.isempty(): - return - # make sure there's just one real repodata with extensions - repodata = self.handle.first_repodata() - if not repodata: - return - oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES) - if not set(addedprovides) <= set(oldaddedprovides): - for id in addedprovides: - repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id) - repodata.internalize() - self.writecachedrepo(None, repodata) - -def load_stub(repodata): - repo = repodata.repo.appdata - if repo: - return repo.load_ext(repodata) - return False - -def setup_repos(): - - srcrepo = Repo("f27-source", LOCAL_REPO_INFO_SOURCE) - repo = Repo("f27", LOCAL_REPO_INFO_ARCH) - repo.srcrepo = srcrepo - return [repo, srcrepo] + return _repodata._RPM_REVERSE_LOOKUP.get(rpm_name) def setup_pool(arch, repos=()): pool = solv.Pool() #pool.set_debuglevel(2) pool.setarch(arch) - pool.set_loadcallback(load_stub) + pool.set_loadcallback(_repodata.load_stub) for repo in repos: repo.metadata_path = repo.metadata_path.format(arch=arch) @@ -469,8 +137,8 @@ def ensure_buildable(pkgset, pool=None): return sources, builddeps def make_pool(arch): - _populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction - return setup_pool(arch, setup_repos()) + _repodata._populate_module_reverse_lookup() # TODO: Integrate this into the Pool abstraction + return setup_pool(arch, _repodata.setup_repos()) _DEFAULT_HINTS = ("glibc-minimal-langpack",) diff --git a/fedmod/_repodata.py b/fedmod/_repodata.py new file mode 100644 index 0000000..9898788 --- /dev/null +++ b/fedmod/_repodata.py @@ -0,0 +1,339 @@ +"""Helpers for metadata management""" +import os.path +import requests +import click +import gzip +import logging +import modulemd +import solv +from requests_toolbelt.downloadutils.tee import tee_to_file +from fnmatch import fnmatch +from urllib.parse import urljoin +from bs4 import BeautifulSoup, SoupStrainer +from lxml import etree + +XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") +CACHEDIR = os.path.join(XDG_CACHE_HOME, "fedmod") + +log = logging.getLogger(__name__) + +FALLBACK_STREAM = 'master' +STREAM = 'f27' +ARCH = 'x86_64' +REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" +REPO_METADATA_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/repodata/") +REPO_METADATA_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/repodata/") +LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27") +LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH) +LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source") + +METADATA_FILES = ("*-filelists.xml.gz", "*-primary.xml.gz", "*-modules.yaml.gz", "repomd.xml") + +def _download_one_file(remote_url, filename): + if os.path.exists(filename) and not filename.endswith("repomd.xml"): + print(f"Skipping download; {filename} already exists") + return + with requests.get(remote_url, stream=True) as response: + print(f"Downloading {remote_url}") + chunksize = 65536 + expected_chunks = int(response.headers["content-length"]) / chunksize + downloader = tee_to_file(response, filename=filename, chunksize=chunksize) + show_progress = click.progressbar(downloader, length=expected_chunks) + with show_progress: + for chunk in show_progress: + pass + print(f"Added {filename} to cache") + +def _download_metadata_files(metadata_url, local_path): + os.makedirs(os.path.join(local_path, "repodata"), exist_ok=True) + response = requests.get(metadata_url) + response.raise_for_status() + link_filter = SoupStrainer("a", href=True) + metadata_links = BeautifulSoup(response.text, parse_only=link_filter, features="lxml") + patterns_to_check = set(METADATA_FILES) + files_to_fetch = set() + for link in metadata_links.find_all("a"): + href = link["href"] + for pattern in patterns_to_check: + if fnmatch(href, pattern): + patterns_to_check.remove(pattern) + files_to_fetch.add(href) + break # Go to next file + predownload = set(os.listdir(local_path)) + for relative_href in files_to_fetch: + absolute_href = urljoin(metadata_url, relative_href) + filename = os.path.join(local_path, "repodata", relative_href) + # This could be parallelised with concurrent.futures, but + # probably not worth it (it makes the progress bars trickier) + _download_one_file(absolute_href, filename) + postdownload = set(os.listdir(local_path)) + # Prune any old metadata files automatically + if len(postdownload) >= (len(predownload) + len(METADATA_FILES)): + # TODO: Actually prune old metadata files + pass + + +def download_repo_metadata(): + """Downloads the latest repo metadata""" + _download_metadata_files(REPO_METADATA_ARCH, LOCAL_REPO_INFO_ARCH) + _download_metadata_files(REPO_METADATA_SOURCE, LOCAL_REPO_INFO_SOURCE) + +_SRPM_REVERSE_LOOKUP = {} # SRPM name : module name +_RPM_REVERSE_LOOKUP = {} # RPM name : module name +def _populate_module_reverse_lookup(): + # TODO: Cache the reverse mapping in _depchase.Repo instances, as with the solver data + metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH) + repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") + repomd_xml = etree.parse(repomd_fname) + repo_modulemd = repomd_xml.find("rpm:data[@type='modules']/rpm:location", {"rpm": "http://linux.duke.edu/metadata/repo"}) + if repo_modulemd is None: + raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?") + repo_modulemd_fname = os.path.join(metadata_dir, repo_modulemd.attrib["href"]) + with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: + modules_yaml = modules_yaml_gz.read() + modules = modulemd.loads_all(modules_yaml) + for module in modules: + for srpmname in module.components.rpms: + # This isn't entirely valid, as it doesn't account for multiple + # modules that include the same source RPM with different output + # filters (e.g. python3-ecosystem vs python2-ecosystem) + _SRPM_REVERSE_LOOKUP[srpmname] = module.name + for rpmname in module.artifacts.rpms: + # This is only valid for module sets that are guaranteed to be + # fully coinstallable, and hence only allow any given RPM to be + # published by at most one module + rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] + _RPM_REVERSE_LOOKUP[rpmprefix] = module.name + +class Repo(object): + def __init__(self, name, metadata_path): + self.name = name + self.metadata_path = metadata_path + self.handle = None + self.cookie = None + self.extcookie = None + self.srcrepo = None + + @staticmethod + def calc_cookie_fp(fp): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add_fp(fp) + return chksum.raw() + + @staticmethod + def calc_cookie_ext(f, cookie): + chksum = solv.Chksum(solv.REPOKEY_TYPE_SHA256) + chksum.add("1.1") + chksum.add(cookie) + chksum.add_fstat(f.fileno()) + return chksum.raw() + + def cachepath(self, ext=None): + path = "{}-{}".format(self.name.replace(".", "_"), self.metadata_path) + if ext: + path = "{}-{}.solvx".format(path, ext) + else: + path = "{}.solv".format(path) + return os.path.join(CACHEDIR, path.replace("/", "_")) + + def usecachedrepo(self, ext, mark=False): + try: + repopath = self.cachepath(ext) + f = open(repopath, "rb") + f.seek(-32, os.SEEK_END) + fcookie = f.read(32) + if len(fcookie) != 32: + return False + cookie = self.extcookie if ext else self.cookie + if cookie and fcookie != cookie: + return False + if not ext: + f.seek(-32 * 2, os.SEEK_END) + fextcookie = f.read(32) + if len(fextcookie) != 32: + return False + f.seek(0) + f = solv.xfopen_fd(None, f.fileno()) + flags = 0 + if ext: + flags = solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + if not self.handle.add_solv(f, flags): + return False + if not ext: + self.cookie = fcookie + self.extcookie = fextcookie + if mark: + # no futimes in python? + try: + os.utime(repopath, None) + except Exception: + pass + except IOError: + return False + return True + + def writecachedrepo(self, ext, repodata=None): + tmpname = None + try: + if not os.path.isdir(CACHEDIR): + os.mkdir(CACHEDIR, 0o755) + fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", dir=CACHEDIR) + os.fchmod(fd, 0o444) + f = os.fdopen(fd, "wb+") + f = solv.xfopen_fd(None, f.fileno()) + if not repodata: + self.handle.write(f) + elif ext: + repodata.write(f) + else: + # rewrite_repos case, do not write stubs + self.handle.write_first_repodata(f) + f.flush() + if not ext: + if not self.extcookie: + self.extcookie = self.calc_cookie_ext(f, self.cookie) + f.write(self.extcookie) + if not ext: + f.write(self.cookie) + else: + f.write(self.extcookie) + f.close + if self.handle.iscontiguous(): + # switch to saved repo to activate paging and save memory + nf = solv.xfopen(tmpname) + if not ext: + # main repo + self.handle.empty() + flags = solv.Repo.SOLV_ADD_NO_STUBS + if repodata: + # rewrite repos case, recreate stubs + flags = 0 + if not self.handle.add_solv(nf, flags): + sys.exit("internal error, cannot reload solv file") + else: + # extension repodata + # need to extend to repo boundaries, as this is how + # repodata.write() has written the data + repodata.extend_to_repo() + flags = solv.Repo.REPO_EXTEND_SOLVABLES + if ext != "DL": + flags |= solv.Repo.REPO_LOCALPOOL + repodata.add_solv(nf, flags) + os.rename(tmpname, self.cachepath(ext)) + except (OSError, IOError): + if tmpname: + os.unlink(tmpname) + + def load(self, pool): + assert not self.handle + self.handle = pool.add_repo(self.name) + self.handle.appdata = self + f = self.read_repo_metadata("repodata/repomd.xml", False, None) + if not f: + self.handle.free(True) + self.handle = None + return False + self.cookie = self.calc_cookie_fp(f) + if self.usecachedrepo(None, True): + return True + self.handle.add_repomdxml(f) + fname, fchksum = self.find("primary") + if not fname: + return False + f = self.read_repo_metadata(fname, True, fchksum) + if not f: + return False + self.handle.add_rpmmd(f, None) + self.add_exts() + self.writecachedrepo(None) + # Must be called after writing the repo + self.handle.create_stubs() + return True + + def read_repo_metadata(self, fname, uncompress, chksum): + f = open("{}/{}".format(self.metadata_path, fname)) + return solv.xfopen_fd(fname if uncompress else None, f.fileno()) + + def find(self, what): + di = self.handle.Dataiterator_meta(solv.REPOSITORY_REPOMD_TYPE, what, solv.Dataiterator.SEARCH_STRING) + di.prepend_keyname(solv.REPOSITORY_REPOMD) + for d in di: + dp = d.parentpos() + filename = dp.lookup_str(solv.REPOSITORY_REPOMD_LOCATION) + chksum = dp.lookup_checksum(solv.REPOSITORY_REPOMD_CHECKSUM) + if filename: + if not chksum: + print("No {} file checksum!".format(filename)) + return filename, chksum + return None, None + + def add_ext_keys(self, ext, repodata, handle): + if ext == "FL": + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.SOLVABLE_FILELIST) + repodata.add_idarray(handle, solv.REPOSITORY_KEYS, solv.REPOKEY_TYPE_DIRSTRARRAY) + else: + raise NotImplementedError + + def add_ext(self, repodata, what, ext): + filename, chksum = self.find(what) + if not filename: + return + handle = repodata.new_handle() + repodata.set_poolstr(handle, solv.REPOSITORY_REPOMD_TYPE, what) + repodata.set_str(handle, solv.REPOSITORY_REPOMD_LOCATION, filename) + repodata.set_checksum(handle, solv.REPOSITORY_REPOMD_CHECKSUM, chksum) + self.add_ext_keys(ext, repodata, handle) + repodata.add_flexarray(solv.SOLVID_META, solv.REPOSITORY_EXTERNAL, handle) + + def add_exts(self): + repodata = self.handle.add_repodata() + self.add_ext(repodata, "filelists", "FL") + repodata.internalize() + + def load_ext(self, repodata): + repomdtype = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_TYPE) + if repomdtype == "filelists": + ext = "FL" + else: + assert False + if self.usecachedrepo(ext): + return True + filename = repodata.lookup_str(solv.SOLVID_META, solv.REPOSITORY_REPOMD_LOCATION) + filechksum = repodata.lookup_checksum(solv.SOLVID_META, solv.REPOSITORY_REPOMD_CHECKSUM) + f = self.read_repo_metadata(filename, True, filechksum) + if not f: + return False + if ext == "FL": + self.handle.add_rpmmd(f, "FL", solv.Repo.REPO_USE_LOADING | solv.Repo.REPO_EXTEND_SOLVABLES | solv.Repo.REPO_LOCALPOOL) + self.writecachedrepo(ext, repodata) + return True + + def updateaddedprovides(self, addedprovides): + if self.handle.isempty(): + return + # make sure there's just one real repodata with extensions + repodata = self.handle.first_repodata() + if not repodata: + return + oldaddedprovides = repodata.lookup_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES) + if not set(addedprovides) <= set(oldaddedprovides): + for id in addedprovides: + repodata.add_idarray(solv.SOLVID_META, solv.REPOSITORY_ADDEDFILEPROVIDES, id) + repodata.internalize() + self.writecachedrepo(None, repodata) + +def load_stub(repodata): + repo = repodata.repo.appdata + if repo: + return repo.load_ext(repodata) + return False + +def setup_repos(): + + srcrepo = Repo("f27-source", LOCAL_REPO_INFO_SOURCE) + repo = Repo("f27", LOCAL_REPO_INFO_ARCH) + repo.srcrepo = srcrepo + return [repo, srcrepo] diff --git a/fedmod/cli.py b/fedmod/cli.py index 2eb8e96..3f2296c 100644 --- a/fedmod/cli.py +++ b/fedmod/cli.py @@ -3,7 +3,7 @@ import argparse import logging from .module_generator import ModuleGenerator -from . import _depchase +from . import _depchase, _repodata # TODO: Switch this over to click (already a dependency for progress bars) @@ -36,7 +36,7 @@ class ModtoolsCLI(object): ) parser_metadata = subparsers.add_parser( - 'metadata', parents=[base_parser], + 'fetch-metadata', parents=[base_parser], help="Fetches repository metadata", description="Caches needed repository metadata locally" ) @@ -65,8 +65,8 @@ class ModtoolsCLIHelper(object): if cli.args.cmd_name == 'rpm2module': mg = ModuleGenerator(cli.args.pkgs) mg.run() - elif cli.args.cmd_name == 'metadata': - _depchase.download_repo_metadata() + elif cli.args.cmd_name == 'fetch-metadata': + _repodata.download_repo_metadata() except KeyboardInterrupt: print('\nInterrupted by user') From 94606f0df655a462953363b053fb871a7706ce2b Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 26 2017 07:31:00 +0000 Subject: [PATCH 2/3] Read repomd.xml for metadata download URLs --- diff --git a/Pipfile b/Pipfile index b1b8725..f075a93 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,5 @@ snakeviz = "*" [packages] fedmod = {editable = true, path = "."} -"beautifulsoup4" = "*" "requests-toolbelt" = "*" click = "*" \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index 7aaad48..ca737db 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7dac39326ba07082a5742f5f9f35da3c32a2ea834ffa241142deae5556189695" + "sha256": "f26d771028572e70de059c2822f07e77566bf4026dee272bf9b0e3e4fbaa17b4" }, "host-environment-markers": { "implementation_name": "cpython", @@ -9,9 +9,9 @@ "os_name": "posix", "platform_machine": "x86_64", "platform_python_implementation": "CPython", - "platform_release": "4.13.4-200.fc26.x86_64", + "platform_release": "4.13.5-200.fc26.x86_64", "platform_system": "Linux", - "platform_version": "#1 SMP Thu Sep 28 20:46:39 UTC 2017", + "platform_version": "#1 SMP Thu Oct 5 16:53:13 UTC 2017", "python_full_version": "3.6.2", "python_version": "3.6", "sys_platform": "linux" @@ -26,20 +26,6 @@ ] }, "default": { - "beanbag": { - "hashes": [ - "sha256:c6aa1e90ad229a6352e4e5f2d468b9cbf3522d74f7a6ac963e88cba769aaf780" - ], - "version": "==1.9.2" - }, - "beautifulsoup4": { - "hashes": [ - "sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11", - "sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76", - "sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89" - ], - "version": "==4.6.0" - }, "certifi": { "hashes": [ "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", @@ -72,24 +58,44 @@ ], "version": "==2.6" }, - "modulemd": { - "hashes": [ - "sha256:a305510e11087c939bd3b00a5ef2aa4d07fd101808e5fedbb33a7720376db041" - ], - "version": "==1.3.1" - }, - "pdc-client": { - "hashes": [ - "sha256:af3d6e520858fd54c0f3e74c136462f75f3ad2f5938dbb0605a813e91bdfeb64" - ], - "version": "==1.7.0" + "lxml": { + "hashes": [ + "sha256:7a8715539adb41c78129983ba69d852e0102a3f51d559eeb91dce1f6290c4ad0", + "sha256:d3a98dda9831a37ef7f55c5e69c0d276c278f24978f5b36b9fad7eac05a22bfc", + "sha256:1deacd52638da2d7fcb864c3949f0285638ec10e6aace93ce15c6a2e0ed91b95", + "sha256:1548247ea3b50014a3ea55ad9446108df191b6a6e51aa8f5953c95b663f382ff", + "sha256:5da6f5b31ea2b573cb20e88aefc6b49d849d07588ba60871342cae42f569b0d7", + "sha256:12e348eb57fb79ccf91a49b7b937c49a5bbe1d73ba75589674b76a56d064bda0", + "sha256:b9e1735918fc1e83c522b9f1048e6bc5af38af958e4efc843046e4b0075a021b", + "sha256:fafeb4b190bd63ba2bcee2496d99cb7345fafbace6b999403010abdff8c05b72", + "sha256:9007da6fb1b96fb1c9d7bd65e97bbbad60295abc19833d7e67e05314c1868f58", + "sha256:cfbf0b956f33cda3af2a1438a2541549b69a7a240e71de7d8ca819b8f1547aac", + "sha256:0a103253a94cdad86028d273aaebb8b30c75fdf009c23e52cdc8ce88429fd326", + "sha256:b3d5a0ecf0c2c31c404246b6706b2e477159ee07b73be5102389ab250dd67701", + "sha256:e92af0fd08c7d2176ad4be4a7c47fd800d6ee05046b41e36ed579c01fb106c25", + "sha256:feb2144c2ae4035ad57165dd22bdc93b1389158a985c0497a096d39e2b2cd67b", + "sha256:a4433655219b84a360dbdf2c34d9625c3988a272e6fc028222d528ad5902f6a2", + "sha256:db98287cb1488eb103930a64444542f6ffe83694ef392f801aa56d648d905663", + "sha256:307d325ee143b60b9c82912e96e9f4345200c33c8ae00b04b001e4c85fb5f146", + "sha256:5caec9b174dbf927034d588669c62d2a9d0ce447365b20a3463f4daab1e4f03b", + "sha256:fb816595494ce21191764572215f56edfbc6d9fbebd1491c8466502892989689", + "sha256:10399bececdb67f0d9251ecf2dda2abf6ddeee6096741754356f1a3715c8c830", + "sha256:c263fd15d27f3be93485fcd83a495cbbc35352512d9e31644d49a54504a1be2a", + "sha256:c10ad53216d5af2b3ba63e65db793cb7dd7e598e17826938045e32f38b0e4814", + "sha256:d42a5182d4b0953d02e5f46c9f0dc304be736fbaa1c0d2f11326182b9684b5f4", + "sha256:dd7c22bf890d266e72c5e5c8c44555ffbfe4ca2a329da785e7d8b1972fc3ff74", + "sha256:93df9805146980e83834ea9320baa6a56d8aea45f63d7d3cc721f71eb1a1bac6", + "sha256:7ba1b62fe9414d73d493241011df952b72074808debc3a2d6d8a64fb9944edf6", + "sha256:d2c121f5f77bed1e1eddeee23ee76fee8a3d48fa7a3aab589d12942f87778a9e", + "sha256:be3aaeb5f468a49f523f16736ccff7d82af2b4b303292ba3d052b5b28f3fbe47" + ], + "version": "==4.1.0" }, - "pykerberos": { + "modulemd": { "hashes": [ - "sha256:60f84e1b606d58cc457b186914c195072ebebf5d5a05e75c0136541808e06523" + "sha256:59e66590f4a2e301c629cc83f3696723b1da19ee93a038311dc97bf5713dcccd" ], - "markers": "sys_platform != 'win32'", - "version": "==1.1.14" + "version": "==1.3.2" }, "python-dateutil": { "hashes": [ @@ -124,13 +130,6 @@ ], "version": "==2.18.4" }, - "requests-kerberos": { - "hashes": [ - "sha256:6c1dc3c7c0774ac104487112fc564525a838aa507982b7c4785e41e5cb6893c9", - "sha256:ae734f71f46a7b205a74fb90e160c7ba4cc4e0dff2d4f3129cf74806b51b94ba" - ], - "version": "==0.11.0" - }, "requests-toolbelt": { "hashes": [ "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", diff --git a/fedmod/_repodata.py b/fedmod/_repodata.py index 9898788..7c940ca 100644 --- a/fedmod/_repodata.py +++ b/fedmod/_repodata.py @@ -1,4 +1,6 @@ """Helpers for metadata management""" +import sys +import tempfile import os.path import requests import click @@ -9,7 +11,6 @@ import solv from requests_toolbelt.downloadutils.tee import tee_to_file from fnmatch import fnmatch from urllib.parse import urljoin -from bs4 import BeautifulSoup, SoupStrainer from lxml import etree XDG_CACHE_HOME = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") @@ -21,20 +22,29 @@ FALLBACK_STREAM = 'master' STREAM = 'f27' ARCH = 'x86_64' REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -REPO_METADATA_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/repodata/") -REPO_METADATA_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/repodata/") +REPO_URL_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/") +REPO_URL_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/") +REPO_METADATA_ARCH = os.path.join(REPO_URL_ARCH, "repodata/") +REPO_METADATA_SOURCE = os.path.join(REPO_URL_SOURCE, "repodata/") LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27") LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH) LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source") -METADATA_FILES = ("*-filelists.xml.gz", "*-primary.xml.gz", "*-modules.yaml.gz", "repomd.xml") +METADATA_SECTIONS = ("filelists", "*primary", "modules") + +_REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} +def _read_repomd_location(repomd_xml, section): + location = repomd_xml.find(f"rpm:data[@type='{section}']/rpm:location", _REPOMD_XML_NAMESPACE) + if location is not None: + return location.attrib["href"] + return None def _download_one_file(remote_url, filename): if os.path.exists(filename) and not filename.endswith("repomd.xml"): - print(f"Skipping download; {filename} already exists") + print(f" Skipping download; {filename} already exists") return with requests.get(remote_url, stream=True) as response: - print(f"Downloading {remote_url}") + print(f" Downloading {remote_url}") chunksize = 65536 expected_chunks = int(response.headers["content-length"]) / chunksize downloader = tee_to_file(response, filename=filename, chunksize=chunksize) @@ -42,41 +52,43 @@ def _download_one_file(remote_url, filename): with show_progress: for chunk in show_progress: pass - print(f"Added {filename} to cache") + print(f" Added {filename} to cache") def _download_metadata_files(metadata_url, local_path): - os.makedirs(os.path.join(local_path, "repodata"), exist_ok=True) - response = requests.get(metadata_url) + local_repodata_path = os.path.join(local_path, "repodata") + os.makedirs(local_repodata_path, exist_ok=True) + repomd_url = urljoin(metadata_url, "repodata/repomd.xml") + print(f"Remote metadata: {repomd_url}") + response = requests.get(repomd_url) response.raise_for_status() - link_filter = SoupStrainer("a", href=True) - metadata_links = BeautifulSoup(response.text, parse_only=link_filter, features="lxml") - patterns_to_check = set(METADATA_FILES) + repomd_filename = os.path.join(local_repodata_path, "repomd.xml") + with open(repomd_filename, "wb") as f: + f.write(response.content) + print(f" Cached metadata in {repomd_filename}") + repomd_xml = etree.parse(repomd_filename) files_to_fetch = set() - for link in metadata_links.find_all("a"): - href = link["href"] - for pattern in patterns_to_check: - if fnmatch(href, pattern): - patterns_to_check.remove(pattern) - files_to_fetch.add(href) - break # Go to next file + for section in METADATA_SECTIONS: + relative_href = _read_repomd_location(repomd_xml, section) + if relative_href is not None: + files_to_fetch.add(relative_href) predownload = set(os.listdir(local_path)) for relative_href in files_to_fetch: absolute_href = urljoin(metadata_url, relative_href) - filename = os.path.join(local_path, "repodata", relative_href) + filename = os.path.join(local_path, relative_href) # This could be parallelised with concurrent.futures, but # probably not worth it (it makes the progress bars trickier) _download_one_file(absolute_href, filename) postdownload = set(os.listdir(local_path)) # Prune any old metadata files automatically - if len(postdownload) >= (len(predownload) + len(METADATA_FILES)): + if len(postdownload) >= (len(predownload) + len(METADATA_SECTIONS)): # TODO: Actually prune old metadata files pass def download_repo_metadata(): """Downloads the latest repo metadata""" - _download_metadata_files(REPO_METADATA_ARCH, LOCAL_REPO_INFO_ARCH) - _download_metadata_files(REPO_METADATA_SOURCE, LOCAL_REPO_INFO_SOURCE) + _download_metadata_files(REPO_URL_ARCH, LOCAL_REPO_INFO_ARCH) + _download_metadata_files(REPO_URL_SOURCE, LOCAL_REPO_INFO_SOURCE) _SRPM_REVERSE_LOOKUP = {} # SRPM name : module name _RPM_REVERSE_LOOKUP = {} # RPM name : module name @@ -85,10 +97,10 @@ def _populate_module_reverse_lookup(): metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") repomd_xml = etree.parse(repomd_fname) - repo_modulemd = repomd_xml.find("rpm:data[@type='modules']/rpm:location", {"rpm": "http://linux.duke.edu/metadata/repo"}) - if repo_modulemd is None: + repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") + if repo_relative_modulemd is None: raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?") - repo_modulemd_fname = os.path.join(metadata_dir, repo_modulemd.attrib["href"]) + repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: modules_yaml = modules_yaml_gz.read() modules = modulemd.loads_all(modules_yaml) diff --git a/setup.py b/setup.py index e95f0d2..5757383 100644 --- a/setup.py +++ b/setup.py @@ -26,9 +26,9 @@ setup( install_requires=[ 'modulemd', 'click', - 'beautifulsoup4', 'requests', 'requests-toolbelt', + 'lxml', ], packages=find_packages(), ) From e94f4d04f48c4ea5944c38bb1cd6a2198e563175 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Oct 26 2017 10:12:32 +0000 Subject: [PATCH 3/3] Use full F27 metadata for package dependencies Allows draft modulemd generation for a much larger set of packages. Also made the generation process more tolerant of dependency resolution errors by switching them to printed warnings, rather than failing the generation process. That way, warnings solely due to bugs in the generator will have no effect, while those that indicate real problems will show up again when attempting to build the resulting module definition. --- diff --git a/Pipfile b/Pipfile index f075a93..cc24c95 100644 --- a/Pipfile +++ b/Pipfile @@ -12,4 +12,5 @@ snakeviz = "*" fedmod = {editable = true, path = "."} "requests-toolbelt" = "*" -click = "*" \ No newline at end of file +click = "*" +attrs = "*" \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index ca737db..f2bc15d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f26d771028572e70de059c2822f07e77566bf4026dee272bf9b0e3e4fbaa17b4" + "sha256": "eb22a789500751e490310d5c8e96e2d6d51d9f89adcf0f5d88ea79641e1fa8b3" }, "host-environment-markers": { "implementation_name": "cpython", @@ -26,6 +26,13 @@ ] }, "default": { + "attrs": { + "hashes": [ + "sha256:a7e0d9183f6457de12df7ba6a81f6569c7d6b25f67ad509b5ad52e8545970a2f", + "sha256:5d4d1b99f94d69338f485984127e4473b3ab9e20f43821b0e546cc3b2302fd11" + ], + "version": "==17.2.0" + }, "certifi": { "hashes": [ "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", diff --git a/fedmod/_depchase.py b/fedmod/_depchase.py index 25a52ff..3323f95 100644 --- a/fedmod/_depchase.py +++ b/fedmod/_depchase.py @@ -115,16 +115,25 @@ def _solve(solver, pkgnames): # Initial jobs, no conflicting packages for n in pkgnames: sel = pool.select(n, solv.Selection.SELECTION_NAME | solv.Selection.SELECTION_DOTARCH) - assert not sel.isempty(), "Could not find package for {}".format(n) + if sel.isempty(): + log.warn("Could not find package for {}".format(n)) + continue jobs += sel.jobs(solv.Job.SOLVER_INSTALL) problems = solver.solve(jobs) if problems: for problem in problems: - print(problem) - sys.exit(1) + log.warn(problem) print_transaction(pool, solver.transaction()) - return {s for s in solver.transaction().newpackages() if s.arch not in ("src", "nosrc")} + result = set() + for s in solver.transaction().newpackages(): + if s.name.startswith("fedora-release"): + # Relying on the F27 metadata injects irrelevant fedora-release deps + continue + if s.arch in ("src", "nosrc"): + continue + result.add(s) + return result def ensure_buildable(pkgset, pool=None): """Given a set of solvables, returns a set of source packages & build deps""" @@ -132,7 +141,7 @@ def ensure_buildable(pkgset, pool=None): # That's OK, since other modules will provide those packages # The goal of *this* method is to report the SRPMs that need to be # built, and their build dependencies - sources = set(get_sourcepkg(s) for s in pkgset) + sources = set(get_sourcepkg(s, only_name=True) for s in pkgset) builddeps = ensure_installable(sources, pool=pool) return sources, builddeps diff --git a/fedmod/_repodata.py b/fedmod/_repodata.py index 7c940ca..8eaa7ed 100644 --- a/fedmod/_repodata.py +++ b/fedmod/_repodata.py @@ -8,6 +8,7 @@ import gzip import logging import modulemd import solv +from attr import attributes, attrib from requests_toolbelt.downloadutils.tee import tee_to_file from fnmatch import fnmatch from urllib.parse import urljoin @@ -21,16 +22,53 @@ log = logging.getLogger(__name__) FALLBACK_STREAM = 'master' STREAM = 'f27' ARCH = 'x86_64' -REPO_URL_PREFIX = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" -REPO_URL_ARCH = os.path.join(REPO_URL_PREFIX, ARCH, "os/") -REPO_URL_SOURCE = os.path.join(REPO_URL_PREFIX, "source/tree/") -REPO_METADATA_ARCH = os.path.join(REPO_URL_ARCH, "repodata/") -REPO_METADATA_SOURCE = os.path.join(REPO_URL_SOURCE, "repodata/") -LOCAL_REPO_PATH = os.path.join(CACHEDIR, "repos", "f27") -LOCAL_REPO_INFO_ARCH = os.path.join(LOCAL_REPO_PATH, ARCH) -LOCAL_REPO_INFO_SOURCE = os.path.join(LOCAL_REPO_PATH, "source") +_F27_BIKESHED_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/modular/development/bikeshed/Server/" +_F27_MAIN_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/development/27/Everything/" +_F27_UPDATES_REPO = "https://dl.fedoraproject.org/pub/fedora/linux/updates/27/" -METADATA_SECTIONS = ("filelists", "*primary", "modules") +@attributes +class RepoPaths: + remote_repo_url = attrib(str) + remote_metadata_url = attrib(str) + local_cache_path = attrib(str) + local_metadata_path = attrib(str) + +def _define_repo(remote_prefix, local_cache_name, arch=None): + if arch is None: + local_arch_path = "source" + if "updates" in remote_prefix: + remote_arch_path = "SRPMS" + else: + remote_arch_path = "source/tree/" + else: + local_arch_path = arch + if "updates" in remote_prefix: + remote_arch_path = arch + else: + remote_arch_path = os.path.join(arch, "os/") + remote_repo_url = os.path.join(remote_prefix, remote_arch_path) + remote_metadata_url = os.path.join(remote_repo_url, "repodata/") + local_cache_path = os.path.join(CACHEDIR, "repos", local_cache_name, local_arch_path) + local_metadata_path = os.path.join(local_cache_path, "repodata/") + return RepoPaths(remote_repo_url, remote_metadata_url, + local_cache_path, local_metadata_path) + +_x86_64_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules", ARCH) +_SOURCE_MODULE_INFO = _define_repo(_F27_BIKESHED_REPO, "f27-modules") +_x86_64_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages", ARCH) +_SOURCE_PACKAGE_INFO = _define_repo(_F27_MAIN_REPO, "f27-packages") +_x86_64_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates", ARCH) +_SOURCE_UPDATES_INFO = _define_repo(_F27_UPDATES_REPO, "f27-updates") +_ALL_REPOS = ( + _x86_64_MODULE_INFO, + _SOURCE_MODULE_INFO, + _x86_64_PACKAGE_INFO, + _SOURCE_PACKAGE_INFO, + _x86_64_UPDATES_INFO, + _SOURCE_UPDATES_INFO, +) + +METADATA_SECTIONS = ("filelists", "primary", "modules") _REPOMD_XML_NAMESPACE = {"rpm": "http://linux.duke.edu/metadata/repo"} def _read_repomd_location(repomd_xml, section): @@ -54,14 +92,15 @@ def _download_one_file(remote_url, filename): pass print(f" Added {filename} to cache") -def _download_metadata_files(metadata_url, local_path): - local_repodata_path = os.path.join(local_path, "repodata") - os.makedirs(local_repodata_path, exist_ok=True) - repomd_url = urljoin(metadata_url, "repodata/repomd.xml") +def _download_metadata_files(repo_paths): + local_path = repo_paths.local_cache_path + local_metadata_path = repo_paths.local_metadata_path + os.makedirs(local_metadata_path, exist_ok=True) + repomd_url = urljoin(repo_paths.remote_metadata_url, "repomd.xml") print(f"Remote metadata: {repomd_url}") response = requests.get(repomd_url) response.raise_for_status() - repomd_filename = os.path.join(local_repodata_path, "repomd.xml") + repomd_filename = os.path.join(local_metadata_path, "repomd.xml") with open(repomd_filename, "wb") as f: f.write(response.content) print(f" Cached metadata in {repomd_filename}") @@ -73,7 +112,7 @@ def _download_metadata_files(metadata_url, local_path): files_to_fetch.add(relative_href) predownload = set(os.listdir(local_path)) for relative_href in files_to_fetch: - absolute_href = urljoin(metadata_url, relative_href) + absolute_href = urljoin(repo_paths.remote_repo_url, relative_href) filename = os.path.join(local_path, relative_href) # This could be parallelised with concurrent.futures, but # probably not worth it (it makes the progress bars trickier) @@ -87,19 +126,19 @@ def _download_metadata_files(metadata_url, local_path): def download_repo_metadata(): """Downloads the latest repo metadata""" - _download_metadata_files(REPO_URL_ARCH, LOCAL_REPO_INFO_ARCH) - _download_metadata_files(REPO_URL_SOURCE, LOCAL_REPO_INFO_SOURCE) + for repo_definition in _ALL_REPOS: + _download_metadata_files(repo_definition) _SRPM_REVERSE_LOOKUP = {} # SRPM name : module name _RPM_REVERSE_LOOKUP = {} # RPM name : module name def _populate_module_reverse_lookup(): # TODO: Cache the reverse mapping in _depchase.Repo instances, as with the solver data - metadata_dir = os.path.join(LOCAL_REPO_INFO_ARCH) + metadata_dir = os.path.join(_x86_64_MODULE_INFO.local_cache_path) repomd_fname = os.path.join(metadata_dir, "repodata", "repomd.xml") repomd_xml = etree.parse(repomd_fname) repo_relative_modulemd = _read_repomd_location(repomd_xml, "modules") if repo_relative_modulemd is None: - raise RuntimeError("No 'modules' entry found in repomd.xml. Is the metadata for a non-modular repo?") + raise RuntimeError(f"No 'modules' entry found in {repomd_fname}. Is the metadata for a non-modular repo?") repo_modulemd_fname = os.path.join(metadata_dir, repo_relative_modulemd) with gzip.open(repo_modulemd_fname, "r") as modules_yaml_gz: modules_yaml = modules_yaml_gz.read() @@ -345,7 +384,10 @@ def load_stub(repodata): def setup_repos(): - srcrepo = Repo("f27-source", LOCAL_REPO_INFO_SOURCE) - repo = Repo("f27", LOCAL_REPO_INFO_ARCH) + srcrepo = Repo("f27-source", _SOURCE_PACKAGE_INFO.local_cache_path) + repo = Repo("f27", _x86_64_PACKAGE_INFO.local_cache_path) repo.srcrepo = srcrepo - return [repo, srcrepo] + updates_srcrepo = Repo("f27-updates-source", _SOURCE_UPDATES_INFO.local_cache_path) + updates_repo = Repo("f27-updates", _x86_64_UPDATES_INFO.local_cache_path) + updates_repo.srcrepo = updates_srcrepo + return [repo, srcrepo, updates_repo, updates_srcrepo] diff --git a/fedmod/cli.py b/fedmod/cli.py index 3f2296c..49b3206 100644 --- a/fedmod/cli.py +++ b/fedmod/cli.py @@ -71,5 +71,5 @@ class ModtoolsCLIHelper(object): except KeyboardInterrupt: print('\nInterrupted by user') except Exception as e: - print(e) + logging.exception("Unexpected exception") sys.exit(1) diff --git a/fedmod/module_generator.py b/fedmod/module_generator.py index 6050cc3..33836db 100644 --- a/fedmod/module_generator.py +++ b/fedmod/module_generator.py @@ -5,21 +5,21 @@ import logging import dnf from . import _depchase +def _name_only(rpm_name): + name, version, release = rpm_name.rsplit("-", 2) + return name + def _categorise_deps(pool, all_rpm_deps): module_deps = set() remaining_rpm_deps = set() for pkg in all_rpm_deps: - modname = _depchase.get_module_for_rpm(pkg) + modname = _depchase.get_module_for_rpm(pkg.name) if modname is not None: module_deps.add(modname) else: remaining_rpm_deps.add(pkg) return module_deps, remaining_rpm_deps -def _name_only(rpm_name): - name, version, release = rpm_name.rsplit("-", 2) - return name - class ModuleGenerator(object): def __init__(self, pkgs): @@ -32,7 +32,7 @@ class ModuleGenerator(object): pkgs = self.pkgs pool = _depchase.make_pool("x86_64") self.api_srpms = {_name_only(_depchase.get_srpm_for_rpm(pool, dep)) for dep in pkgs} - run_deps = {s.name:s for s in _depchase.ensure_installable(pkgs, pool=pool)} + run_deps = _depchase.ensure_installable(pkgs, pool=pool) module_run_deps, rpm_run_deps = _categorise_deps(pool, run_deps) self.module_run_deps = module_run_deps run_srpms, build_deps = _depchase.ensure_buildable(rpm_run_deps) @@ -56,8 +56,8 @@ class ModuleGenerator(object): logging.warn("Failed to close out build dependencies after 10 iteration") self.module_run_deps = module_run_deps self.module_build_deps = module_build_deps - run_srpm_names = {_name_only(s.name) for s in run_srpms} - build_srpm_names = {_name_only(s.name) for s in build_srpms} + run_srpm_names = {_name_only(n) for n in run_srpms} + build_srpm_names = {_name_only(n) for n in build_srpms} self.run_srpms = run_srpm_names - build_srpm_names self.build_srpms = build_srpm_names - run_srpm_names self.build_and_run_srpms = run_srpm_names & build_srpm_names diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py index f51fb59..f12ae8a 100644 --- a/tests/test_module_generator.py +++ b/tests/test_module_generator.py @@ -91,5 +91,5 @@ class TestMultiplePackageInput(object): # Expected module dependencies for grep + mariadb assert set(modmd.buildrequires) == set() - expected_modules = set(('platform', 'mariadb', 'perl', 'networking-base')) + expected_modules = set(('platform', 'mariadb', 'perl')) assert set(modmd.requires) == expected_modules