From cc25d58a887214674ffce825193e7d1791d5b69f Mon Sep 17 00:00:00 2001 From: CentOS CI Date: Oct 31 2019 20:02:30 +0000 Subject: [PATCH 1/2] Convert remaining tests to libmodulemd v2 Signed-off-by: CentOS CI --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f166652 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*~ +.idea diff --git a/tests/exclusions.txt b/tests/exclusions.txt index f2cc370..8562f65 100644 --- a/tests/exclusions.txt +++ b/tests/exclusions.txt @@ -10,6 +10,7 @@ README.md # Skip any .gitignore files overrides/.gitignore +.gitignore # Skip the validator files tests/common_tests.sh @@ -24,4 +25,5 @@ tests/empty.yaml tests/intents.yaml tests/missingstream.yaml tests/module.yaml +tests/modulewithnames.yaml tests/nodejs.yaml diff --git a/tests/modulewithnames.yaml b/tests/modulewithnames.yaml new file mode 100644 index 0000000..9807322 --- /dev/null +++ b/tests/modulewithnames.yaml @@ -0,0 +1,53 @@ +--- +document: modulemd +version: 2 +data: + name: nodejs + stream: testing + summary: Javascript runtime + description: >- + Node.js is a platform built on Chrome''s JavaScript runtime for easily building + fast, scalable network applications. Node.js uses an event-driven, non-blocking + I/O model that makes it lightweight and efficient, perfect for data-intensive + real-time applications that run across distributed devices. + license: + module: + - MIT + dependencies: + - buildrequires: + platform: [] + requires: + platform: [] + references: + community: http://nodejs.org + documentation: http://nodejs.org/en/docs + tracker: https://github.com/nodejs/node/issues + profiles: + default: + rpms: + - nodejs + - npm + development: + rpms: + - nodejs + - nodejs-devel + - npm + minimal: + rpms: + - nodejs + api: + rpms: + - nodejs + - nodejs-devel + - npm + components: + rpms: + libuv: + rationale: Platform abstraction layer for Node.js + ref: 1.20 + buildorder: 0 + nodejs: + rationale: Javascript runtime and npm package manager. + ref: 10 + buildorder: 10 +... diff --git a/tests/validate.py b/tests/validate.py index 2ffb240..6abc13c 100755 --- a/tests/validate.py +++ b/tests/validate.py @@ -5,77 +5,77 @@ import gi import git import os import sys +import logging +from logging import error, info from gi.repository import GLib -gi.require_version('Modulemd', '1.0') -from gi.repository import Modulemd +gi.require_version('Modulemd', '2.0') +from gi.repository import Modulemd # noqa + +logging.getLogger().setLevel(logging.INFO) + def do_validate(filename): # Valid filenames must end in ".yaml" to be properly included by Pungi if not filename.endswith(".yaml"): - print("%s does not end with .yaml. It will not be included by " + error("{} does not end with .yaml. It will not be included by " "Pungi. If this file does not contain defaults, it should " - "be added to the tests/exclusions.txt file." % filename, - file=sys.stderr,) + "be added to the tests/exclusions.txt file.".format(filename)) return False, None # The files must parse correctly + idx = Modulemd.ModuleIndex() try: - (objects, failures) = Modulemd.objects_from_file_ext(filename) + (objects, failures) = idx.update_from_file(filename, True) except GLib.Error as e: - print(e.message, file=sys.stderr) + error("{}".format(e.message)) return False, None - if (failures): + if failures: for failure in failures: - print("Failed subdocument (%s): \n%s\n", failure.message, - failure.doc, file=sys.stderr) + error("Failed subdocument ({}): \n{}\n".format(failure.get_gerror().message, + failure.get_yaml())) return False, None # There must be exactly one object per file - if len(objects) != 1: - print("There must be exactly one subdocument", file=sys.stderr) + names = idx.get_module_names() + if len(names) != 1: + error("There must be exactly one module represented by this file") return False, None # The files must have exactly one Modulemd.Defaults object - try: - (default, def_failures) = \ - Modulemd.Defaults.new_from_file_ext (filename) - except GLib.Error as e: - print(e.message, file=sys.stderr) + module = idx.get_module(names[0]) + defaults = module.get_defaults() + + if defaults is None: + error("No defaults document provided for {}".format(module.props.module_name)) + return False, None + + # The files must not contain any streams + if len(module.get_stream_names()) != 0: + error("File included a stream document.") return False, None # Filenames must match their contents expected_name = os.path.basename(filename).rsplit('.', maxsplit=1)[0] - if expected_name != default.props.module_name: - print("Module name \"%s\" doesn't match filename \"%s.yaml\"" % ( - default.props.module_name, expected_name), - file=sys.stderr) + if expected_name != defaults.props.module_name: + error("Module name \"{}\" doesn't match filename \"{}.yaml\"".format( + defaults.props.module_name, expected_name)) return False, None - if default.props.default_stream: + default_stream = defaults.get_default_stream() + if default_stream: # Default streams must also appear in the profiles list - if not default.props.default_stream in default.props.profile_defaults: - print("Stream '%s' missing from profiles" % ( - default.props.default_stream), - file=sys.stderr) + if not defaults.get_default_profiles_for_stream(default_stream): + error("Stream '{}' is missing from the profiles for '{}'".format(default_stream, defaults.get_module_name())) return False, None # Modules in Fedora must not specify "Intents" - try: - if (default.props.intents): - print ('Module name "%s" includes the intents section, which is ' - 'not currently permitted in Fedora.' % ( - default.props.module_name), - file=sys.stderr) - return False, None - except AttributeError: - # The available version of libmodulemd does not support intents - pass + # TODO: This needs a new interface exposed in libmodulemd v2 - print("%s is valid" % filename) - return (True, default) + info("{} is valid".format(filename)) + return (True, idx) def main(): @@ -104,7 +104,7 @@ def main(): exclusions.append(line.strip()) # Validate all of the files - prioritizer = Modulemd.Prioritizer() + merger = Modulemd.ModuleIndexMerger() for file in files: excluded = False for excl in exclusions: @@ -112,15 +112,15 @@ def main(): excluded = True break if not excluded: - (valid, obj) = do_validate(file) + (valid, idx) = do_validate(file) if not valid: - print("%s failed to validate" % file, file=sys.stderr) + error("{} failed to validate".format(file)) result = os.EX_DATAERR else: try: - prioritizer.add([obj,], 0) + merger.associate_index(idx, 0) except GLib.Error as e: - print("Could not merge %s with other defaults: %s" % ( + error("Could not merge {} with other defaults: {}".format( file, e.message)) result = os.EX_DATAERR @@ -131,11 +131,14 @@ def main(): # conflicts arise that weren't detected by the above tests. This should be # impossible. try: - prioritizer.resolve() + idx = merger.resolve_ext(strict_default_streams=True) except GLib.Error as e: - print("Could not merge all defaults: %s" % e.message) + error("Could not merge all defaults: {}".format(e.message)) result = os.EX_DATAERR + if result == os.EX_OK: + info("Merging all of the documents encountered no errors.") + return result From c5a8d1b927dc4a5faf2f11597c1a9258641224fa Mon Sep 17 00:00:00 2001 From: CentOS CI Date: Oct 31 2019 20:07:16 +0000 Subject: [PATCH 2/2] Reformat python code with black Signed-off-by: CentOS CI --- diff --git a/tests/compare_defaults.py b/tests/compare_defaults.py index 5b1d9b9..742046b 100755 --- a/tests/compare_defaults.py +++ b/tests/compare_defaults.py @@ -6,36 +6,41 @@ import os import sys from gi.repository import GLib -gi.require_version('Modulemd', '2.0') + +gi.require_version("Modulemd", "2.0") from gi.repository import Modulemd + def print_failure(failure): print("Error: %s" % failure.get_gerror()) print("Offending YAML: \n%s" % failure.get_yaml()) + def unusable_baseline(): print("Baseline was unusable. Skipping test.") return os.EX_OK -def get_index_and_defaults (repo, filename, commit): + +def get_index_and_defaults(repo, filename, commit): try: - yaml = repo.git.show('%s:%s' % (commit, filename)) + yaml = repo.git.show("%s:%s" % (commit, filename)) except git.exc.GitCommandError as e: # This file didn't exist in the commit, so return # FileNotFoundError to skip checking it. - raise FileNotFoundError("{} does not exist at commit {}".format(filename, commit)) - + raise FileNotFoundError( + "{} does not exist at commit {}".format(filename, commit) + ) index = Modulemd.ModuleIndex.new() try: ret, failures = index.update_from_string(yaml, True) if ret != True: for failure in failures: - print_failure (failure) + print_failure(failure) raise IOError("Invalid modulemd-defaults document") except gi.repository.GLib.Error as e: - print ("Error: %s" % e) + print("Error: %s" % e) raise IOError("Invalid YAML document") module_names = index.get_module_names() @@ -50,6 +55,7 @@ def get_index_and_defaults (repo, filename, commit): return index, defaults + def main(): filename = sys.argv[1] baseline_commit = sys.argv[2] @@ -57,13 +63,13 @@ def main(): script_dir = os.path.dirname(os.path.realpath(__file__)) - repo = git.Repo(script_dir, - search_parent_directories=True) + repo = git.Repo(script_dir, search_parent_directories=True) # First get the defaults from the baseline commit try: - baseline_index, baseline_defaults = get_index_and_defaults ( - repo, filename, baseline_commit) + baseline_index, baseline_defaults = get_index_and_defaults( + repo, filename, baseline_commit + ) except FileNotFoundError as e: # The baseline file didn't exist, so there's no valid original # to compare to. Check it only with validate.py. @@ -78,11 +84,17 @@ def main(): try: updated_commit = updated_commit - updated_index, updated_defaults = get_index_and_defaults ( - repo, filename, updated_commit) + updated_index, updated_defaults = get_index_and_defaults( + repo, filename, updated_commit + ) except FileNotFoundError as e: # The PR is removing this file. Assume that this is acceptable - print("{} is being removed. Not performing any comparison tests.".format(filename), file=sys.stderr) + print( + "{} is being removed. Not performing any comparison tests.".format( + filename + ), + file=sys.stderr, + ) return os.EX_OK except IOError as e: # If we hit this, the patch is broken. @@ -90,10 +102,12 @@ def main(): return os.EX_DATAERR if updated_defaults.get_modified() <= baseline_defaults.get_modified(): - print ("%s has changed but the 'modified' field has not increased." % - filename) - print ("Baseline modified: {}".format(baseline_defaults.get_modified())) - print ("Updated modified: {}".format(updated_defaults.get_modified())) + print( + "%s has changed but the 'modified' field has not increased." + % filename + ) + print("Baseline modified: {}".format(baseline_defaults.get_modified())) + print("Updated modified: {}".format(updated_defaults.get_modified())) return os.EX_DATAERR # Confirm that these two sets of defaults merge cleanly @@ -104,11 +118,11 @@ def main(): try: merger.resolve() except gi.repository.GLib.Error as e: - print ("Merge Error: %s" % e, file=sys.stderr) + print("Merge Error: %s" % e, file=sys.stderr) return os.EX_DATAERR return os.EX_OK + if __name__ == "__main__": sys.exit(main()) - diff --git a/tests/validate.py b/tests/validate.py index 6abc13c..680d0c6 100755 --- a/tests/validate.py +++ b/tests/validate.py @@ -9,7 +9,8 @@ import logging from logging import error, info from gi.repository import GLib -gi.require_version('Modulemd', '2.0') + +gi.require_version("Modulemd", "2.0") from gi.repository import Modulemd # noqa logging.getLogger().setLevel(logging.INFO) @@ -18,9 +19,11 @@ logging.getLogger().setLevel(logging.INFO) def do_validate(filename): # Valid filenames must end in ".yaml" to be properly included by Pungi if not filename.endswith(".yaml"): - error("{} does not end with .yaml. It will not be included by " - "Pungi. If this file does not contain defaults, it should " - "be added to the tests/exclusions.txt file.".format(filename)) + error( + "{} does not end with .yaml. It will not be included by " + "Pungi. If this file does not contain defaults, it should " + "be added to the tests/exclusions.txt file.".format(filename) + ) return False, None # The files must parse correctly @@ -33,8 +36,11 @@ def do_validate(filename): if failures: for failure in failures: - error("Failed subdocument ({}): \n{}\n".format(failure.get_gerror().message, - failure.get_yaml())) + error( + "Failed subdocument ({}): \n{}\n".format( + failure.get_gerror().message, failure.get_yaml() + ) + ) return False, None # There must be exactly one object per file @@ -48,7 +54,11 @@ def do_validate(filename): defaults = module.get_defaults() if defaults is None: - error("No defaults document provided for {}".format(module.props.module_name)) + error( + "No defaults document provided for {}".format( + module.props.module_name + ) + ) return False, None # The files must not contain any streams @@ -57,18 +67,25 @@ def do_validate(filename): return False, None # Filenames must match their contents - expected_name = os.path.basename(filename).rsplit('.', maxsplit=1)[0] + expected_name = os.path.basename(filename).rsplit(".", maxsplit=1)[0] if expected_name != defaults.props.module_name: - error("Module name \"{}\" doesn't match filename \"{}.yaml\"".format( - defaults.props.module_name, expected_name)) + error( + 'Module name "{}" doesn\'t match filename "{}.yaml"'.format( + defaults.props.module_name, expected_name + ) + ) return False, None default_stream = defaults.get_default_stream() if default_stream: # Default streams must also appear in the profiles list if not defaults.get_default_profiles_for_stream(default_stream): - error("Stream '{}' is missing from the profiles for '{}'".format(default_stream, defaults.get_module_name())) + error( + "Stream '{}' is missing from the profiles for '{}'".format( + default_stream, defaults.get_module_name() + ) + ) return False, None # Modules in Fedora must not specify "Intents" @@ -87,8 +104,7 @@ def main(): script_dir = os.path.dirname(os.path.realpath(__file__)) # Get the repo we're running in - repo = git.Repo(script_dir, - search_parent_directories=True) + repo = git.Repo(script_dir, search_parent_directories=True) # Get the list of files in this repository files = [x for (x, y) in repo.index.entries.keys()] @@ -108,7 +124,7 @@ def main(): for file in files: excluded = False for excl in exclusions: - if (file.startswith(excl)): + if file.startswith(excl): excluded = True break if not excluded: @@ -120,8 +136,11 @@ def main(): try: merger.associate_index(idx, 0) except GLib.Error as e: - error("Could not merge {} with other defaults: {}".format( - file, e.message)) + error( + "Could not merge {} with other defaults: {}".format( + file, e.message + ) + ) result = os.EX_DATAERR if result == os.EX_DATAERR: @@ -139,8 +158,8 @@ def main(): if result == os.EX_OK: info("Merging all of the documents encountered no errors.") - return result + if __name__ == "__main__": sys.exit(main())