From 001031be44f5a029279ac1dae11b335e658b628b Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Feb 26 2019 14:25:24 +0000 Subject: summarize-module: optionally access MBS --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 86930e6..58a6dbe 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -2,7 +2,7 @@ # # modulemd_summarizer - Prints a summary of ModuleMD files # -# Copyright © 2018 Red Hat, Inc. +# Copyright © 2018, 2019 Red Hat, Inc. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,11 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# Author: +# Authors: # Rafael dos Santos +# Nils Philippsen -from collections import defaultdict -from fnmatch import fnmatch +import logging import tempfile from click import ClickException @@ -30,10 +30,14 @@ gi.require_version('Modulemd', '1.0') # noqa: E402 from gi.repository import Modulemd import smartcols -from . import _fetchrepodata, _repodata +from . import _fetchrepodata +from .config import runtime_config -def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree): +log = logging.getLogger(__name__) + + +def _print_summary(module_builds, module_defaults, as_tree): tb = smartcols.Table() cl = tb.new_column('Name') cl.tree = as_tree @@ -43,106 +47,56 @@ def _print_summary(profiles, sdefaults, pdefaults, deps, mfilter, as_tree): cl_prof = tb.new_column('Profiles') cl_deps = tb.new_column('Dependencies') parent_ln = {} - for nsvc, plist in sorted(profiles.items()): - if mfilter and not any((fnmatch(nsvc, p) for p in mfilter)): - continue - modname, sname, version, context = nsvc.split(':') + mnames_defaults = {d.props.module_name: d for d in module_defaults} + + for mbuild in sorted(module_builds, key=lambda mb: mb.props.name): - def is_def_strm(s): - return s == sdefaults.get(modname, '') + mbp = mbuild.props + mdp = mnames_defaults.get(mbp.name, + Modulemd.Defaults(module_name=mbp.name)).props if as_tree: - pl = parent_ln.get(modname, None) + pl = parent_ln.get(mbp.name, None) if pl is None: - pl = parent_ln.setdefault(modname, tb.new_line()) - pl[cl] = modname + pl = parent_ln.setdefault(mbp.name, tb.new_line()) + pl[cl] = mbp.name else: pl = None ln = tb.new_line(pl) - ln[cl] = modname - ln[cl_strm] = sname + ' [d]' * is_def_strm(sname) - ln[cl_ver] = version - ln[cl_ctxt] = context - - def is_def_prof(p): - return p in pdefaults.get(modname, {}).get(sname, []) - - ln[cl_prof] = ', '.join(p + ' [d]' if is_def_prof(p) else p - for p in plist) - - dlist = sorted(deps.get(nsvc, [])) - if len(dlist) == 0: - continue - if len(dlist) == 1: - dlist = str(dlist[0]) - else: - dlist = ','.join(dlist) - ln[cl_deps] = dlist + ln[cl] = mbp.name + ln[cl_strm] = mbp.stream + ( + ' [d]' if mbp.stream == mdp.default_stream else '') + ln[cl_ver] = str(mbp.version) + # old module builds don't have the context set in their modulemd + ln[cl_ctxt] = mbp.context or '00000000' + + streams_dprofiles = {k: v.get() + for k, v in mdp.profile_defaults.items()} + + ln[cl_prof] = ', '.join(sorted( + p + ' [d]' + if p in streams_dprofiles.get(mbp.stream, ()) + else p + for p in mbp.profiles + )) + + dlist = sorted( + f"{dname}:{dstream}" + for deps in mbp.dependencies + for dname, streams_ss in sorted(deps.props.requires.items(), + key=lambda x: x[0]) + for dstream in sorted(streams_ss.get()) + ) + + if dlist: + ln[cl_deps] = ", ".join(dlist) print(tb) print('\nHint: [d]efault') -def _parse_mmd(mmd_index, profiles, dstreams, dprofiles, deps): - """ - Parse module repodata information. This repodata is provided by a tool - (e.g MBS), so we can assume the information is well-formed - """ - - for index in mmd_index: - module_name = index.get_name() - for module in index.get_streams().values(): - nsvc = module.get_nsvc() - plist = list(module.get_profiles().keys()) - profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist)) - - for dep in module.get_dependencies(): - deplist = set(deps.get(nsvc, [])) - for m, s in dep.peek_requires().items(): - deplist.add(f"{m}:{','.join(s.get())}" - if len(s.get()) else m) - deps[nsvc] = sorted(deplist) - - defaults = index.get_defaults() - if not defaults: - continue - - # Local module metadata can overwrite metadata from repo - dstreams[module_name] = defaults.peek_default_stream() - for s, pset in defaults.peek_profile_defaults().items(): - dprofiles[module_name][s] = pset.get() - - -def _parse_objs(objects, profiles, dstreams, dprofiles, deps): - """ - Parse module metadata information. This metadata is probably provided by - packager-written yamls, so it might be missing information (like module - name or module defaults). - """ - - for obj in objects: - if isinstance(obj, Modulemd.Module): - # local modulemd files might miss some info like context or - # version. So let's make sure nsvc is consistent - ctxt = obj.get_context() or '' - nsvc = f'{obj.get_name()}:{obj.get_stream()}:' \ - f'{obj.get_version()}:{ctxt}' - profiles[nsvc] = sorted(obj.get_profiles().keys()) - deplist = [] - for dep in obj.get_dependencies(): - for m, s in dep.peek_requires().items(): - deplist.append(f'{m}:{",".join(s.get())}' - if len(s.get()) else m) - deps[nsvc] = sorted(deplist) - elif isinstance(obj, Modulemd.Defaults): - module_name = obj.peek_module_name() - dstreams[module_name] = obj.peek_default_stream() - for s, pset in obj.peek_profile_defaults().items(): - dprofiles[module_name][s] = pset.get() - - def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False): """ Load Modulemd objects from each yaml file in the `yamls` list or each url @@ -162,26 +116,46 @@ def summarize_modules(mfilter=None, yamls=None, urls=None, as_tree=False): *as_tree*: print the summary in tree format, grouping information. """ - profiles, dstreams, dprofiles, deps = {}, {}, defaultdict(dict), {} - if yamls: - for yaml in yamls: - try: - objects = Modulemd.objects_from_file(yaml) - except gi.repository.GLib.GError as gerror: - raise ClickException(f"Could not read {yaml}: {gerror}") from gerror - _parse_objs(objects, profiles, dstreams, dprofiles, deps) - elif urls: - for url in urls: - with tempfile.TemporaryDirectory() as local_path: - rp = _fetchrepodata.RepoPaths(url, local_path) - _fetchrepodata._download_metadata_files(rp) - indexes = _fetchrepodata._read_modules(rp) - _parse_mmd(indexes, profiles, dstreams, dprofiles, deps) + module_builds = [] + module_defaults = [] + + if yamls or urls: + if yamls: + for yaml in yamls: + try: + objects = Modulemd.objects_from_file(yaml) + except gi.repository.GLib.GError as gerror: + raise ClickException(f"Could not read {yaml}: {gerror}") from gerror + elif urls: + for url in urls: + with tempfile.TemporaryDirectory() as local_path: + rp = _fetchrepodata.RepoPaths(url, local_path) + _fetchrepodata._download_metadata_files(rp) + objects = _fetchrepodata._read_modules(rp) + + # distribute modules/builds and defaults into their own lists + for o in objects: + if isinstance(o, Modulemd.Module): + module_builds.append(o) + elif isinstance(o, Modulemd.Defaults): + module_defaults.append(o) + else: + log.warning(f"Encountered unknown object in YAML: {o!r}") else: - profiles = _repodata.get_modules_profiles_lookup() - dstreams = _repodata.get_modules_default_streams_lookup() - dprofiles = _repodata.get_modules_default_profiles_lookup() - deps = _repodata.get_modules_dependencies_lookup() + source = runtime_config['source'] + source_type = runtime_config['source_type'] + + if source_type == 'buildsys': + log.info("Querying build systems for information, this can take some time.") + + if mfilter: + module_patterns = [p.split(":", 1)[0] for p in mfilter] + else: + module_patterns = None + + module_builds = list(source.enumerate_module_builds(nsvc_patterns=mfilter)) + module_defaults = list(source.enumerate_module_defaults( + module_patterns=module_patterns + )) - mfilter = mfilter or [] - _print_summary(profiles, dstreams, dprofiles, deps, mfilter, as_tree) + _print_summary(module_builds, module_defaults, as_tree) diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index 6929c75..731f83c 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -5,7 +5,10 @@ import re import pytest +from _fedmod._repodata import dataset_name +from _fedmod.config import runtime_config from _fedmod.modulemd_summarizer import summarize_modules +from _fedmod.sources import Source testfiles_dir = os.path.join(os.path.dirname(__file__), 'files') @@ -15,6 +18,19 @@ spec_v2_yaml_path = os.path.join(testfiles_dir, 'spec.v2.yaml') @pytest.mark.needs_metadata class TestModuleSummary(object): + def setup(self): + self.runtime_config_backup = runtime_config.copy() + runtime_config.clear() + runtime_config.update({ + 'dataset': dataset_name, + 'source_type': 'rpmrepodata', + 'source': Source.from_type('rpmrepodata'), + }) + + def teardown(self): + runtime_config.clear() + runtime_config.update(self.runtime_config_backup) + def matches(self, mod, strm, ver, ctxt, prof, deps, out): mstr = fr'^{mod}\s+{strm}\s+{ver}\s+{ctxt}\s+{prof}\s+{deps}$' return re.search(mstr, out, re.M) is not None @@ -27,10 +43,10 @@ class TestModuleSummary(object): assert self.matches('reviewboard', '2.5', '.*', '083bce86', r'default \[d\], server', - 'django:1.6,platform:f29', out) + 'django:1.6, platform:f29', out) assert self.matches('reviewboard', '3.0', '.*', '083bce86', r'default \[d\], server', - 'django:1.6,platform:f29', out) + 'django:1.6, platform:f29', out) assert self.matches('testmodule', 'master', '.*', 'c2c572ec', 'default', 'platform:f29', out) @@ -40,10 +56,10 @@ class TestModuleSummary(object): assert self.matches('reviewboard', '2.5', '.*', '083bce86', r'default \[d\], server', - 'django:1.6,platform:f29', out) + 'django:1.6, platform:f29', out) assert self.matches('reviewboard', '3.0', '.*', '083bce86', r'default \[d\], server', - 'django:1.6,platform:f29', out) + 'django:1.6, platform:f29', out) assert self.matches('django', '1.6', '.*', '6c81f848', r'default \[d\], python2_development', 'platform:f29', out) @@ -56,7 +72,9 @@ class TestModuleSummary(object): assert self.matches('foo', 'stream-name', '.*', 'c0ffee43', 'buildroot, container, default, minimal, ' + - 'srpm-buildroot', 'compatible:v3,v4,extras,' + - 'moreextras:bar,foo,platform:-epel7,-f27,-f28,' + - 'platform:epel7,platform:f27,platform:f28,' + - 'runtime:a,b', out) + 'srpm-buildroot', + 'compatible:v3, compatible:v4, moreextras:bar, ' + 'moreextras:foo, platform:-epel7, platform:-f27, ' + 'platform:-f28, platform:epel7, platform:f27, ' + 'platform:f28, runtime:a, runtime:b', + out)