From 3fe7bcde9089b25e16890ca0c5fd4e836ba62baa Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 23 2018 13:52:47 +0000 Subject: [PATCH 1/6] Revert "repodata: make get_dataset public" This reverts commit a236b5b95d896b0f40ce1fa3c8fc3eed5195de36. I overlooked the _repodata.py file and mistakenly made get_dataset public altough it should never be. Instead some methods for retrieving the required data from the repodata should've been added. This commit fixes that. --- diff --git a/_fedmod/_repodata.py b/_fedmod/_repodata.py index 45a661f..b77ac19 100644 --- a/_fedmod/_repodata.py +++ b/_fedmod/_repodata.py @@ -21,7 +21,7 @@ def _load_dataset(): _ACTIVE_DATASET = load_cached_repodata(dataset_name) -def get_dataset(): +def _get_dataset(): if _ACTIVE_DATASET is None: _load_dataset() return _ACTIVE_DATASET @@ -44,20 +44,20 @@ def set_dataset_name(name): def list_modules(): - return get_dataset().module_to_packages.keys() + return _get_dataset().module_to_packages.keys() def get_rpms_in_module(module_name): - return get_dataset().module_to_packages.get(module_name, []) + return _get_dataset().module_to_packages.get(module_name, []) def get_modules_for_rpm(rpm_name): - result = get_dataset().rpm_to_modules.get(rpm_name) + result = _get_dataset().rpm_to_modules.get(rpm_name) return result def get_module_for_rpm(rpm_name): - result = get_dataset().rpm_to_modules.get(rpm_name) + result = _get_dataset().rpm_to_modules.get(rpm_name) if result is not None: if len(result) > 1: log.warn( @@ -67,7 +67,19 @@ def get_module_for_rpm(rpm_name): def get_rpm_reverse_lookup(): - return get_dataset().rpm_to_modules + return _get_dataset().rpm_to_modules + + +def get_modules_profiles_lookup(): + return _get_dataset().module_to_profiles + + +def get_modules_default_streams_lookup(): + return _get_dataset().stream_defaults + + +def get_modules_default_profiles_lookup(): + return _get_dataset().profile_defaults class Repo(object): @@ -100,7 +112,7 @@ class Repo(object): path = "{}-{}.solvx".format(path, ext) else: path = "{}.solv".format(path) - return os.path.join(get_dataset().cache_dir, path.replace("/", "_")) + return os.path.join(_get_dataset().cache_dir, path.replace("/", "_")) def usecachedrepo(self, ext, mark=False): try: @@ -145,7 +157,7 @@ class Repo(object): tmpname = None try: fd, tmpname = tempfile.mkstemp(prefix=".newsolv-", - dir=get_dataset().cache_dir) + dir=_get_dataset().cache_dir) os.fchmod(fd, 0o444) f = os.fdopen(fd, "wb+") f = solv.xfopen_fd(None, f.fileno()) @@ -311,7 +323,7 @@ def load_stub(repodata): def setup_repos(): - dataset = get_dataset() + dataset = _get_dataset() repos = [] for reponame, (arch_cache_path, src_cache_path) in ( diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index e95fda5..6bfdd8c 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -27,7 +27,7 @@ from gi.repository import Modulemd import smartcols from ._fetchrepodata import merge_modules -from ._repodata import get_dataset +from . import _repodata def print_summary(profiles, sdefaults=None, pdefaults=None, restrict_to=None): @@ -94,9 +94,9 @@ def summarize_modules(restrict_list=None, yaml_files=None): *yaml_files*: additional yaml files to parse and include in the summary """ - profiles = get_dataset().module_to_profiles - dstreams = get_dataset().stream_defaults - dprofiles = get_dataset().profile_defaults + profiles = _repodata.get_modules_profiles_lookup() + dstreams = _repodata.get_modules_default_streams_lookup() + dprofiles = _repodata.get_modules_default_profiles_lookup() if yaml_files: _add_module_metadata(yaml_files, profiles, dstreams, dprofiles) From e9343537748fef4c9fa9bd36ee657fb0e1d9cd45 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 26 2018 20:07:37 +0000 Subject: [PATCH 2/6] summarize-module: mark print_summary as internal Rename it to _print_summary to show it's an internal function and move the sanitization of the input for the outer function. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 6bfdd8c..b67edbf 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -30,11 +30,7 @@ from ._fetchrepodata import merge_modules from . import _repodata -def print_summary(profiles, sdefaults=None, pdefaults=None, restrict_to=None): - sdefaults = sdefaults or {} - pdefaults = pdefaults or {} - restrict_to = restrict_to or [] - +def _print_summary(profiles, sdefaults, pdefaults, restrict_to): tb = smartcols.Table() cl = tb.new_column('Name') cl_strm = tb.new_column('Stream') @@ -101,4 +97,5 @@ def summarize_modules(restrict_list=None, yaml_files=None): if yaml_files: _add_module_metadata(yaml_files, profiles, dstreams, dprofiles) - print_summary(profiles, dstreams, dprofiles, restrict_list) + restrict_list = restrict_list or [] + _print_summary(profiles, dstreams, dprofiles, restrict_list) From 8d394837770feaf86eda43e6d88ecd8f7ff5935c Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 26 2018 20:07:46 +0000 Subject: [PATCH 3/6] summarize-module: make shallow copy of lookup dicts Since we are modifying them by adding information from local modulemd files, make a copy so as to not change the cached metadata. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index b67edbf..6b44c7b 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -90,9 +90,9 @@ def summarize_modules(restrict_list=None, yaml_files=None): *yaml_files*: additional yaml files to parse and include in the summary """ - profiles = _repodata.get_modules_profiles_lookup() - dstreams = _repodata.get_modules_default_streams_lookup() - dprofiles = _repodata.get_modules_default_profiles_lookup() + profiles = _repodata.get_modules_profiles_lookup().copy() + dstreams = _repodata.get_modules_default_streams_lookup().copy() + dprofiles = _repodata.get_modules_default_profiles_lookup().copy() if yaml_files: _add_module_metadata(yaml_files, profiles, dstreams, dprofiles) From 17879768a52d80c6663fb3bfa92991b2ad7e3bc0 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 28 2018 10:04:28 +0000 Subject: [PATCH 4/6] Flatten the module profiles lookup dict This way we can add more information (version and context) without chaining more dictionaries inside it. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/_fetchrepodata.py b/_fedmod/_fetchrepodata.py index d75e685..e2bd9c9 100644 --- a/_fedmod/_fetchrepodata.py +++ b/_fedmod/_fetchrepodata.py @@ -327,8 +327,8 @@ def _write_lookup_caches(paths): module_forward_lookup = {} srpm_reverse_lookup = defaultdict(list) rpm_reverse_lookup = defaultdict(list) - # {module-name: {stream: [profiles]}} - profile_forward_lookup = defaultdict(dict) + # {'module-name:stream:version:context': [profiles]}} + profile_forward_lookup = defaultdict(list) # {module-name: stream} stream_defaults_forward_lookup = {} # {module-name: {stream : [profiles]}} @@ -337,10 +337,13 @@ def _write_lookup_caches(paths): for index_set in index_sets: for index in index_set: module_name = index.get_name() + for nsvc, module in index.get_streams().items(): + profiles = module.get_profiles().keys() + profile_forward_lookup[nsvc] = sorted(profiles) + # What we think of as module is a ModuleStream for libmodulemd # We are ignoring context and using the stream with highest version for module in merge_modules(index.get_streams().values()): - sname = module.get_stream() artifacts = module.props.rpm_artifacts.get() module_forward_lookup[module_name] = list(set(artifacts)) @@ -353,10 +356,6 @@ def _write_lookup_caches(paths): rpmprefix = rpmname.split(":", 1)[0].rsplit("-", 1)[0] rpm_reverse_lookup[rpmprefix].append(module_name) - # {'name': ModuleProfile} - profiles = list(module.get_profiles().keys()) # only names - profile_forward_lookup[module_name][sname] = profiles - defaults = index.get_defaults() if not defaults: continue diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 6b44c7b..31d66f5 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -26,7 +26,6 @@ from gi.repository import Modulemd import smartcols -from ._fetchrepodata import merge_modules from . import _repodata @@ -35,23 +34,24 @@ def _print_summary(profiles, sdefaults, pdefaults, restrict_to): cl = tb.new_column('Name') cl_strm = tb.new_column('Stream') cl_prof = tb.new_column('Profiles') - for modname, sdict in sorted(profiles.items()): + for nsvc, plist in sorted(profiles.items()): + modname, sname, version, context = nsvc.split(':') + if restrict_to and modname not in restrict_to: continue def is_def_strm(s): return s == sdefaults.get(modname, '') - for sname, plist in sorted(sdict.items()): - ln = tb.new_line() - ln[cl] = modname - ln[cl_strm] = sname + ' [d]' * is_def_strm(sname) + ln = tb.new_line() + ln[cl] = modname + ln[cl_strm] = sname + ' [d]' * is_def_strm(sname) - def is_def_prof(p): - return p in pdefaults.get(modname, {}).get(sname, []) + def is_def_prof(p): + return p in pdefaults.get(modname, {}).get(sname, []) - ln[cl_prof] = ', '.join(p + ' [d]' if is_def_prof(p) else p - for p in plist) + ln[cl_prof] = ', '.join(p + ' [d]' if is_def_prof(p) else p + for p in plist) print(tb) print('\nHint: [d]efault') @@ -64,11 +64,14 @@ def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles): mmd_index, failures = Modulemd.index_from_file(yaml) assert len(failures) == 0, failures for module_name, index in mmd_index.items(): - for module in merge_modules(index.get_streams().values()): - stream_name = module.get_stream() + for module in index.get_streams().values(): + # local modulemd files might miss some info like context or + # version. So let's make sure nsvc is consistent + ctxt = module.get_context() or '' + nsvc = f'{module_name}:{module.get_stream()}:' \ + f'{module.get_version()}:{ctxt}' plist = list(module.get_profiles().keys()) - profiles.setdefault(module_name, {}).setdefault( - stream_name, []).extend(plist) + profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist)) defaults = index.get_defaults() if not defaults: diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index e758293..9308d05 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -25,10 +25,10 @@ class TestModuleSummary(object): out, err = capfd.readouterr() assert self.matches('reviewboard', '2.5', - r'server, default \[d\]', + r'default \[d\], server', out) assert self.matches('reviewboard', '3.0', - r'server, default \[d\]', + r'default \[d\], server', out) assert self.matches('testmodule', 'master', 'default', out) @@ -37,13 +37,13 @@ class TestModuleSummary(object): out, err = capfd.readouterr() assert self.matches('reviewboard', '2.5', - r'server, default \[d\]', + r'default \[d\], server', out) assert self.matches('reviewboard', '3.0', - r'server, default \[d\]', + r'default \[d\], server', out) assert self.matches('django', '1.6', - r'python2_development, default \[d\]', + r'default \[d\], python2_development', out) assert not self.matches('testmodule', 'master', 'default', out) @@ -52,5 +52,5 @@ class TestModuleSummary(object): out, err = capfd.readouterr() assert self.matches('testmodule', 'master', 'default', out) - assert self.matches('foo', 'stream-name', 'minimal, buildroot, ' + - 'container, srpm-buildroot, default', out) + assert self.matches('foo', 'stream-name', 'buildroot, container, ' + + 'default, minimal, srpm-buildroot', out) From 12ba3c4b892c03d921fd971b43c9a63ab4f4bfc4 Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 28 2018 10:13:18 +0000 Subject: [PATCH 5/6] summarize-module: add version and context to output Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/cli.py b/_fedmod/cli.py index 26c7ac4..a70e90a 100644 --- a/_fedmod/cli.py +++ b/_fedmod/cli.py @@ -198,6 +198,8 @@ def lint(modulemd, min_level): type=click.Path(exists=True), help="Additional modulemd files to check." " Can be given multiple times.") -def summarize(modules, add_file): +@click.option("--tree", "-t", is_flag=True, default=False, + help="Print output as a tree") +def summarize(modules, add_file, tree): """Prints a summary of available modules""" - summarize_modules(modules, add_file) + summarize_modules(modules, add_file, tree) diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index 31d66f5..ad4f8d9 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -29,11 +29,15 @@ import smartcols from . import _repodata -def _print_summary(profiles, sdefaults, pdefaults, restrict_to): +def _print_summary(profiles, sdefaults, pdefaults, restrict_to, as_tree): tb = smartcols.Table() cl = tb.new_column('Name') + cl.tree = as_tree cl_strm = tb.new_column('Stream') + cl_ver = tb.new_column('Version') + cl_ctxt = tb.new_column('Context') cl_prof = tb.new_column('Profiles') + parent_ln = {} for nsvc, plist in sorted(profiles.items()): modname, sname, version, context = nsvc.split(':') @@ -43,9 +47,19 @@ def _print_summary(profiles, sdefaults, pdefaults, restrict_to): def is_def_strm(s): return s == sdefaults.get(modname, '') - ln = tb.new_line() + if as_tree: + pl = parent_ln.get(modname, None) + if pl is None: + pl = parent_ln.setdefault(modname, tb.new_line()) + pl[cl] = modname + else: + pl = None + + ln = tb.new_line(pl) ln[cl] = modname ln[cl_strm] = sname + ' [d]' * is_def_strm(sname) + ln[cl_ver] = version + ln[cl_ctxt] = context def is_def_prof(p): return p in pdefaults.get(modname, {}).get(sname, []) @@ -83,7 +97,7 @@ def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles): dprofiles[module_name][s] = pset.get() -def summarize_modules(restrict_list=None, yaml_files=None): +def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): """ Load Modulemd objects from each repository in repo_list and print a summary of the modules found with a summary of their streams and profiles. @@ -101,4 +115,4 @@ def summarize_modules(restrict_list=None, yaml_files=None): _add_module_metadata(yaml_files, profiles, dstreams, dprofiles) restrict_list = restrict_list or [] - _print_summary(profiles, dstreams, dprofiles, restrict_list) + _print_summary(profiles, dstreams, dprofiles, restrict_list, as_tree) diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index 9308d05..a05ca34 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -15,8 +15,9 @@ spec_v2_yaml_path = os.path.join(testfiles_dir, 'spec.v2.yaml') @pytest.mark.needs_metadata class TestModuleSummary(object): - def matches(self, mod, strm, prof, out): - return re.search(fr'^{mod}\s+{strm}\s+{prof}$', out, re.M) is not None + def matches(self, mod, strm, ver, ctxt, prof, out): + mstr = fr'^{mod}\s+{strm}\s+{ver}\s+{ctxt}\s+{prof}$' + return re.search(mstr, out, re.M) is not None # FIXME: we should mock the fetched metadata so that these tests do not # fail when the metadata changes @@ -24,33 +25,37 @@ class TestModuleSummary(object): summarize_modules() out, err = capfd.readouterr() - assert self.matches('reviewboard', '2.5', + assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86', r'default \[d\], server', out) - assert self.matches('reviewboard', '3.0', + assert self.matches('reviewboard', '3.0', '20180828143238', '083bce86', r'default \[d\], server', out) - assert self.matches('testmodule', 'master', 'default', out) + assert self.matches('testmodule', 'master', '20180405123256', + 'c2c572ec', 'default', out) def test_summarize_modules_restricted(self, capfd): summarize_modules(['reviewboard', 'django']) out, err = capfd.readouterr() - assert self.matches('reviewboard', '2.5', + assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86', r'default \[d\], server', out) - assert self.matches('reviewboard', '3.0', + assert self.matches('reviewboard', '3.0', '20180828143238', '083bce86', r'default \[d\], server', out) - assert self.matches('django', '1.6', + assert self.matches('django', '1.6', '20180828135711', '6c81f848', r'default \[d\], python2_development', out) - assert not self.matches('testmodule', 'master', 'default', out) + assert not self.matches('testmodule', 'master', '20180405123256', + 'c2c572ec', 'default', out) def test_summarize_modules_local_files(self, capfd): summarize_modules(yaml_files=[spec_v2_yaml_path]) out, err = capfd.readouterr() - assert self.matches('testmodule', 'master', 'default', out) - assert self.matches('foo', 'stream-name', 'buildroot, container, ' + - 'default, minimal, srpm-buildroot', out) + assert self.matches('testmodule', 'master', '20180405123256', + 'c2c572ec', 'default', out) + assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43', + 'buildroot, container, default, minimal, ' + + 'srpm-buildroot', out) From 1937421d5faf8a441cd0703ea61762ffb9da645a Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Date: Nov 28 2018 10:22:08 +0000 Subject: [PATCH 6/6] summarize-module: add modular dependency info For each module context, the dependency on other modules might change. Therefore it's important to add the dependency info to the output so developers can get a better outlook of the repository. Signed-off-by: Rafael dos Santos --- diff --git a/_fedmod/_fetchrepodata.py b/_fedmod/_fetchrepodata.py index e2bd9c9..731a59f 100644 --- a/_fedmod/_fetchrepodata.py +++ b/_fedmod/_fetchrepodata.py @@ -172,6 +172,7 @@ def _get_distro_paths(dataset_name): _MODULE_FORWARD_LOOKUP_CACHE = "module-contents" _PROFILE_FORWARD_LOOKUP_CACHE = "module-profiles" +_MODULE_DEPENDENCIES_FORWARD_LOOKUP_CACHE = "module-dependencies" _STREAM_DEFAULT_FORWARD_LOOKUP_CACHE = "stream-defaults" _PROFILE_DEFAULT_FORWARD_LOOKUP_CACHE = "profile-defaults" _SRPM_REVERSE_LOOKUP_CACHE = "srpm-to-module" @@ -179,6 +180,7 @@ _RPM_REVERSE_LOOKUP_CACHE = "rpm-to-module" _ALL_CACHES = [_MODULE_FORWARD_LOOKUP_CACHE, _PROFILE_FORWARD_LOOKUP_CACHE, + _MODULE_DEPENDENCIES_FORWARD_LOOKUP_CACHE, _STREAM_DEFAULT_FORWARD_LOOKUP_CACHE, _PROFILE_DEFAULT_FORWARD_LOOKUP_CACHE, _SRPM_REVERSE_LOOKUP_CACHE, @@ -333,6 +335,8 @@ def _write_lookup_caches(paths): stream_defaults_forward_lookup = {} # {module-name: {stream : [profiles]}} profile_defaults_forward_lookup = defaultdict(dict) + # {'module-name:stream:version:context': ['module-name:stream']} + module_dependencies_forward_lookup = defaultdict(list) for index_set in index_sets: for index in index_set: @@ -341,6 +345,13 @@ def _write_lookup_caches(paths): profiles = module.get_profiles().keys() profile_forward_lookup[nsvc] = sorted(profiles) + for dep in module.get_dependencies(): + dset = set() + for m, s in dep.peek_requires().items(): + dset.add(f"{m}:{','.join(s.get())}" + if len(s.get()) else m) + module_dependencies_forward_lookup[nsvc] = sorted(dset) + # What we think of as module is a ModuleStream for libmodulemd # We are ignoring context and using the stream with highest version for module in merge_modules(index.get_streams().values()): @@ -370,6 +381,8 @@ def _write_lookup_caches(paths): print("Caching lookup tables") _write_cache(paths, _MODULE_FORWARD_LOOKUP_CACHE, module_forward_lookup) _write_cache(paths, _PROFILE_FORWARD_LOOKUP_CACHE, profile_forward_lookup) + _write_cache(paths, _MODULE_DEPENDENCIES_FORWARD_LOOKUP_CACHE, + module_dependencies_forward_lookup) _write_cache(paths, _STREAM_DEFAULT_FORWARD_LOOKUP_CACHE, stream_defaults_forward_lookup) _write_cache(paths, _PROFILE_DEFAULT_FORWARD_LOOKUP_CACHE, @@ -396,6 +409,7 @@ class LocalMetadataCache: rpm_to_modules = attrib(type=dict) module_to_packages = attrib(type=dict) module_to_profiles = attrib(type=dict) + module_to_deps = attrib(type=dict) stream_defaults = attrib(type=dict) profile_defaults = attrib(type=dict) repo_cache_paths = attrib(type=dict) @@ -432,6 +446,8 @@ def load_cached_repodata(dataset_name): rpm_to_modules=_read_cache(paths, _RPM_REVERSE_LOOKUP_CACHE), module_to_packages=_read_cache(paths, _MODULE_FORWARD_LOOKUP_CACHE), module_to_profiles=_read_cache(paths, _PROFILE_FORWARD_LOOKUP_CACHE), + module_to_deps=_read_cache(paths, + _MODULE_DEPENDENCIES_FORWARD_LOOKUP_CACHE), stream_defaults=_read_cache(paths, _STREAM_DEFAULT_FORWARD_LOOKUP_CACHE), profile_defaults=_read_cache(paths, diff --git a/_fedmod/_repodata.py b/_fedmod/_repodata.py index b77ac19..d1b2612 100644 --- a/_fedmod/_repodata.py +++ b/_fedmod/_repodata.py @@ -74,6 +74,10 @@ def get_modules_profiles_lookup(): return _get_dataset().module_to_profiles +def get_modules_dependencies_lookup(): + return _get_dataset().module_to_deps + + def get_modules_default_streams_lookup(): return _get_dataset().stream_defaults diff --git a/_fedmod/modulemd_summarizer.py b/_fedmod/modulemd_summarizer.py index ad4f8d9..cdad351 100644 --- a/_fedmod/modulemd_summarizer.py +++ b/_fedmod/modulemd_summarizer.py @@ -29,7 +29,7 @@ import smartcols from . import _repodata -def _print_summary(profiles, sdefaults, pdefaults, restrict_to, as_tree): +def _print_summary(profiles, sdefaults, pdefaults, deps, restrict_to, as_tree): tb = smartcols.Table() cl = tb.new_column('Name') cl.tree = as_tree @@ -37,6 +37,7 @@ def _print_summary(profiles, sdefaults, pdefaults, restrict_to, as_tree): cl_ver = tb.new_column('Version') cl_ctxt = tb.new_column('Context') cl_prof = tb.new_column('Profiles') + cl_deps = tb.new_column('Dependencies') parent_ln = {} for nsvc, plist in sorted(profiles.items()): modname, sname, version, context = nsvc.split(':') @@ -67,16 +68,26 @@ def _print_summary(profiles, sdefaults, pdefaults, restrict_to, as_tree): ln[cl_prof] = ', '.join(p + ' [d]' if is_def_prof(p) else p for p in plist) + dlist = sorted(deps.get(nsvc, [])) + if len(dlist) == 0: + continue + if len(dlist) == 1: + dlist = str(dlist[0]) + else: + dlist = ','.join(dlist) + ln[cl_deps] = dlist + print(tb) print('\nHint: [d]efault') -def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles): +def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps): for yaml in yaml_files: assert yaml.endswith('.yaml'), "Not a yaml file: {}".format(yaml) mmd_index, failures = Modulemd.index_from_file(yaml) assert len(failures) == 0, failures + for module_name, index in mmd_index.items(): for module in index.get_streams().values(): # local modulemd files might miss some info like context or @@ -87,6 +98,13 @@ def _add_module_metadata(yaml_files, profiles, dstreams, dprofiles): plist = list(module.get_profiles().keys()) profiles[nsvc] = sorted(set(profiles.get(nsvc, []) + plist)) + for dep in module.get_dependencies(): + deplist = set(deps.get(nsvc, [])) + for m, s in dep.peek_requires().items(): + deplist.add(f"{m}:{','.join(s.get())}" + if len(s.get()) else m) + deps[nsvc] = sorted(deplist) + defaults = index.get_defaults() if not defaults: continue @@ -110,9 +128,10 @@ def summarize_modules(restrict_list=None, yaml_files=None, as_tree=False): profiles = _repodata.get_modules_profiles_lookup().copy() dstreams = _repodata.get_modules_default_streams_lookup().copy() dprofiles = _repodata.get_modules_default_profiles_lookup().copy() + deps = _repodata.get_modules_dependencies_lookup().copy() if yaml_files: - _add_module_metadata(yaml_files, profiles, dstreams, dprofiles) + _add_module_metadata(yaml_files, profiles, dstreams, dprofiles, deps) restrict_list = restrict_list or [] - _print_summary(profiles, dstreams, dprofiles, restrict_list, as_tree) + _print_summary(profiles, dstreams, dprofiles, deps, restrict_list, as_tree) diff --git a/tests/test_module_summary.py b/tests/test_module_summary.py index a05ca34..f3df520 100644 --- a/tests/test_module_summary.py +++ b/tests/test_module_summary.py @@ -15,8 +15,8 @@ spec_v2_yaml_path = os.path.join(testfiles_dir, 'spec.v2.yaml') @pytest.mark.needs_metadata class TestModuleSummary(object): - def matches(self, mod, strm, ver, ctxt, prof, out): - mstr = fr'^{mod}\s+{strm}\s+{ver}\s+{ctxt}\s+{prof}$' + def matches(self, mod, strm, ver, ctxt, prof, deps, out): + mstr = fr'^{mod}\s+{strm}\s+{ver}\s+{ctxt}\s+{prof}\s+{deps}$' return re.search(mstr, out, re.M) is not None # FIXME: we should mock the fetched metadata so that these tests do not @@ -27,12 +27,12 @@ class TestModuleSummary(object): assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86', r'default \[d\], server', - out) + 'django:1.6,platform:f29', out) assert self.matches('reviewboard', '3.0', '20180828143238', '083bce86', r'default \[d\], server', - out) + 'django:1.6,platform:f29', out) assert self.matches('testmodule', 'master', '20180405123256', - 'c2c572ec', 'default', out) + 'c2c572ec', 'default', 'platform:f29', out) def test_summarize_modules_restricted(self, capfd): summarize_modules(['reviewboard', 'django']) @@ -40,22 +40,25 @@ class TestModuleSummary(object): assert self.matches('reviewboard', '2.5', '20180828143308', '083bce86', r'default \[d\], server', - out) + 'django:1.6,platform:f29', out) assert self.matches('reviewboard', '3.0', '20180828143238', '083bce86', r'default \[d\], server', - out) + 'django:1.6,platform:f29', out) assert self.matches('django', '1.6', '20180828135711', '6c81f848', r'default \[d\], python2_development', - out) + 'platform:f29', out) assert not self.matches('testmodule', 'master', '20180405123256', - 'c2c572ec', 'default', out) + 'c2c572ec', 'default', 'platform:f29', out) def test_summarize_modules_local_files(self, capfd): summarize_modules(yaml_files=[spec_v2_yaml_path]) out, err = capfd.readouterr() assert self.matches('testmodule', 'master', '20180405123256', - 'c2c572ec', 'default', out) + 'c2c572ec', 'default', 'platform:f29', out) assert self.matches('foo', 'stream-name', '20160927144203', 'c0ffee43', 'buildroot, container, default, minimal, ' + - 'srpm-buildroot', out) + 'srpm-buildroot', 'compatible:v3,v4,extras,' + + 'moreextras:bar,foo,platform:-epel7,-f27,-f28,' + + 'platform:epel7,platform:f27,platform:f28,' + + 'runtime:a,b', out)