PR#10: Module metadata diff - module_diff

module_diff

#10 Module metadata diff

Merged 7 years ago by ralph. Opened 7 years ago by fivaldi.

fivaldi/module_diff fivaldi_diff_modulemd into master

Imitate .copy() due to Python 2.7

Filip Valder • 7 years ago

636e4c4

Add --no-colors option

Filip Valder • 7 years ago

765102a

New: Module metadata diff

Filip Valder • 7 years ago

db2cf3f

README.md

file modified

		`@@ -2,6 +2,8 @@`

		`Creates diffs between two versions of a module. Written in Python 3, but is compatible with Python 2.7.`

		+ For full support of module metadata diffs, check that `libmodulemd` is installed on your system.
		`+`
		`## Development`

		`To install the tool do:`

bin/module_diff

file modified

+10 -6

		`@@ -5,22 +5,26 @@`


		`def main():`
		`- args = parse_arguments()`
		`- module_diff(args.src_nsvc, args.dst_nsvc, json=args.json, dist=args.dist, reused=args.reused)`
		`+ args = parse_arguments().__dict__`
		`+ module_diff(args.pop("src_nsvc_id"), args.pop("dst_nsvc_id"), **args)`


		`def parse_arguments():`
		`parser = argparse.ArgumentParser()`
		`- parser.add_argument("src_nsvc", action="store",`
		`- help="The NSVC of a source module to compare 'from'")`
		`- parser.add_argument("dst_nsvc", action="store",`
		`- help="The NSVC of a destination module to compare 'to'")`
		`+ parser.add_argument("src_nsvc_id", action="store",`
		`+ help="The NSVC/ID of a source module to compare 'from'")`
		`+ parser.add_argument("dst_nsvc_id", action="store",`
		`+ help="The NSVC/ID of a destination module to compare 'to'")`
		`parser.add_argument("-j", "--json", action="store_true",`
		`help="The module diff will be returned as JSON")`
		`parser.add_argument("-d", "--dist", action="store_true",`
		`help="Will display also RPMs with changed only dist tag")`
		`parser.add_argument("-r", "--reused", action="store_true",`
		`help="Will display also RPMs which where reused between versions")`
		`+ parser.add_argument("--mmd-diff", action="store_true",`
		`+ help="Will display also module metadata diff")`
		`+ parser.add_argument("--no-colors", action="store_true",`
		`+ help="Will disable colors in the output")`

		`return parser.parse_args()`

module_diff/__init__.py

file modified

+38 -20

		`@@ -5,29 +5,35 @@`


		`class CliColors:`
		`- HEADER = '\033[95m'`
		`- OKBLUE = '\033[94m'`
		`- OKGREEN = '\033[92m'`
		`- WARNING = '\033[93m'`
		`- FAIL = '\033[91m'`
		`- ENDC = '\033[0m'`
		`- BOLD = '\033[1m'`
		`- UNDERLINE = '\033[4m'`
		`+ def __init__(self, no_colors=False):`
		`+ self.ansi_colors = {`
		`+ 'BLUE': '\033[94m',`
		`+ 'GREEN': '\033[92m',`
		`+ 'RED': '\033[91m',`
		`+ 'RESET': '\033[0m'}`
		`+ self.no_colors = no_colors`

		`+ def __getattr__(self, attr):`
		`+ if self.no_colors:`
		`+ return ''`
		`+ return self.ansi_colors[attr]`

		`- def module_diff(src_nsvc, dst_nsvc, json=False, dist=False, reused=False):`
		`+`
		`+ def module_diff(src_nsvc_id, dst_nsvc_id, **kwargs):`
		`mbs = MBS()`
		`- src_module = Module(mbs.get_module(src_nsvc), dist, reused)`
		`- dst_module = Module(mbs.get_module(dst_nsvc), dist, reused)`
		`+ src_module = Module(mbs.get_module(src_nsvc_id), **kwargs)`
		`+ dst_module = Module(mbs.get_module(dst_nsvc_id), **kwargs)`

		`diff = src_module.get_diff(dst_module)`
		`- if json:`
		`+ if kwargs["json"]:`
		`jsonify_diff(diff)`
		`else:`
		`- human_readable_diff(diff, reused=reused, dist=dist)`
		`+ human_readable_diff(diff, **kwargs)`
		`+`

		`+ def human_readable_diff(diff, **kwargs):`
		`+ cli_colors = CliColors(no_colors=kwargs["no_colors"])`

		`- def human_readable_diff(diff, reused=False, dist=False):`
		`print("---- Stats ----\n")`
		`stats = diff["stats"]`
		`print("Compared modules:")`
		`@@ -40,34 +46,46 @@`
		`print("# of RPMs upgraded: %s" % stats["num_upgraded"])`
		`print("# of RPMs downgraded: %s" % stats["num_downgraded"])`

		`- if reused:`
		`+ if kwargs["reused"]:`
		`print("# of RPMs reused: %s" % stats["num_reused"])`

		`- if dist:`
		`+ if kwargs["dist"]:`
		`print("# of RPMs which dist tag changed: %s" % stats["num_changed_dist"])`

		`print("\n---- RPMs Added ----\n")`
		`for rpm in diff["added_rpms"]:`
		`- print("%s+ %s%s" % (CliColors.OKGREEN, rpm, CliColors.ENDC))`
		`+ print("%s+ %s%s" % (cli_colors.GREEN, rpm, cli_colors.RESET))`
		`print("\n---- RPMs Removed ----\n")`
		`for rpm in diff["removed_rpms"]:`
		`- print("%s- %s%s" % (CliColors.FAIL, rpm, CliColors.ENDC))`
		`+ print("%s- %s%s" % (cli_colors.RED, rpm, cli_colors.RESET))`
		`print("\n---- RPMs Upgraded ----\n")`
		`for rpm in diff["upgraded_rpms"]:`
		`print("U %s -> %s" % (rpm[0], rpm[1]))`
		`print("\n---- RPMs Downgraded ----\n")`
		`for rpm in diff["downgraded_rpms"]:`
		`print("D %s -> %s" % (rpm[0], rpm[1]))`
		`- if reused:`
		`+ if kwargs["reused"]:`
		`print("\n---- RPMs Reused ----\n")`
		`for rpm in diff["reused_rpms"]:`
		`print("R %s" % rpm)`

		`- if dist:`
		`+ if kwargs["dist"]:`
		`print("\n---- RPMs which dist tag changed ----\n")`
		`for rpm in diff["changed_distag_rpms"]:`
		`print("C %s -> %s" % (rpm[0], rpm[1]))`

		`+ if kwargs["mmd_diff"]:`
		`+ print("\n---- Module metadata diff ----\n")`
		`+ for line in diff["mmd_diff"]:`
		`+ if line.startswith("+"):`
		`+ print("%s%s%s" % (cli_colors.GREEN, line, cli_colors.RESET))`
		`+ elif line.startswith("-"):`
		`+ print("%s%s%s" % (cli_colors.RED, line, cli_colors.RESET))`
		`+ elif line.startswith("?"):`
		`+ print("%s%s%s" % (cli_colors.BLUE, line, cli_colors.RESET))`
		`+ else:`
		`+ print(line)`
		`+`

		`def jsonify_diff(diff):`
		`print(json.dumps(diff))`

module_diff/mbs.py

file modified

+34 -19

		`@@ -3,20 +3,28 @@`

		`class MBS(object):`

		`- def __init__(self):`
		`- self.mbs_url = ('https://mbs.fedoraproject.org'`
		`- '/module-build-service/1/module-builds/')`
		`- self.params = {'verbose': 'true'}`
		`-`
		`- def get_module(self, nsvc):`
		`- sep = ':' if ':' in nsvc else '-'`
		`- name = nsvc.split(sep)[0:-3]`
		`- stream = nsvc.split(sep)[-3]`
		`- version = nsvc.split(sep)[-2]`
		`- context = nsvc.split(sep)[-1]`
		`- params = dict(self.params)`
		`- params.update({'name': name, 'stream': stream, 'version': version, 'context': context})`
		`- res = requests.get(self.mbs_url, params=params)`
		`+ mbs_url = ('https://mbs.fedoraproject.org'`
		`+ '/module-build-service/1/module-builds/')`
		`+ default_params = {'verbose': 'true'}`
		`+`
		`+ def get_module(self, nsvc_or_id):`
		`+ params = dict(self.default_params)`
		`+ url = self.mbs_url`
		`+`
		`+ if nsvc_or_id.isdigit():`
		`+ module_id = int(nsvc_or_id)`
		`+ url = url.rstrip("/") + "/%s" % module_id`
		`+ else:`
		`+ sep = ':' if ':' in nsvc_or_id else '-'`
		`+ name = nsvc_or_id.split(sep)[0:-3]`
		`+ stream = nsvc_or_id.split(sep)[-3]`
		`+ version = nsvc_or_id.split(sep)[-2]`
		`+ context = nsvc_or_id.split(sep)[-1]`
		`+`
		`+ params.update({'name': name, 'stream': stream, 'version': version,`
		`+ 'context': context})`
		`+`
		`+ res = requests.get(url, params=params)`

		`if not res.ok:`
		`raise requests.HTTPError(`
		`@@ -26,10 +34,17 @@`
		`module_data = res.json()`
		`except Exception:`
		`raise RuntimeError(`
		`- "Couldn't process module data. Check the response: %s" % res.text)`
		`+ "Couldn't extract module data. Check the response: %s" % res.text)`
		`+`
		`+ if all(k in module_data for k in ["id", "name", "stream", "version"]):`
		`+ return module_data`

		`- if not module_data['meta']['total'] == 1:`
		`- raise RuntimeError('We need exactly one module in response from MBS.'`
		`- ' Check the URL: %s' % res.url)`
		`+ elif all(k in module_data for k in ["meta", "items"]):`
		`+ if not module_data['meta']['total'] == 1:`
		`+ raise RuntimeError('We need exactly one module in response from MBS.'`
		`+ ' Check the URL: %s' % res.url)`
		`+ return module_data["items"][0]`

		`- return module_data["items"][0]`
		`+ else:`
		`+ raise RuntimeError(`
		`+ "Couldn't process module_data: %s" % module_data)`

module_diff/module.py

file modified

+46 -9

		`@@ -1,23 +1,34 @@`
		`+ import json`
		`import re`
		`+ import yaml`

		`+ from json import JSONEncoder`
		`+ from difflib import ndiff`
		`from itertools import product`
		`from kobo.rpmlib import parse_nvr, compare_nvr`
		`- from modulemd import ModuleMetadata`


		`- DISTAG_PATTERN = r"\.(module\_[0-9a-f]+)$"`
		`+ DISTAG_PATTERN = r"\.(module\_[\w\+]+)$"`
		`+`
		`+`
		`+ class MMDEncoder(JSONEncoder):`
		`+ def default(self, obj):`
		`+ if isinstance(obj, (list, dict, str, int, float, bool, type(None))):`
		`+ return JSONEncoder.default(self, obj)`
		`+ return repr(obj)`


		`class Module(object):`
		`""" Represents a Module for diff purposes. """`

		`- def __init__(self, module_data, dist=False, reused=False):`
		`+ def __init__(self, module_data, **kwargs):`
		`"""`
		`Args:`
		`module_data (dict): the module data from MBS response.`
		`"""`
		`- self.dist = dist`
		`- self.reused = reused`
		`+ self.dist = kwargs["dist"] if "dist" in kwargs else False`
		`+ self.reused = kwargs["reused"] if "reused" in kwargs else False`
		`+ self.mmd_diff = kwargs["mmd_diff"] if "mmd_diff" in kwargs else False`

		`self.name = module_data["name"]`
		`self.stream = module_data["stream"]`
		`@@ -38,9 +49,18 @@`
		`returns:`
		`(ModuleMetadata): Metadata of the current module.`
		`"""`
		`+`
		`+ try:`
		`+ import gi`
		`+ gi.require_version('Modulemd', '1.0')`
		`+ from gi.repository import Modulemd`
		`+ except Exception:`
		`+ raise RuntimeError(`
		`+ "Some requirements may be missing. Check that libmodulemd"`
		`+ " is installed. Inspect the traceback for further details.")`
		`+`
		`if not hasattr(self, "_mmd"):`
		`- self._mmd = ModuleMetadata()`
		`- self._mmd.loads(self.modulemd)`
		`+ self._mmd = Modulemd.Module.new_from_string(self.modulemd)`
		`return self._mmd`

		`def get_diff(self, dst_module):`
		`@@ -153,6 +173,9 @@`
		`diff["stats"]["num_changed_dist"] = len(changed_distag_rpms)`
		`diff["changed_distag_rpms"] = changed_distag_rpms`

		`+ if self.mmd_diff:`
		`+ diff["mmd_diff"] = self.modulemd_diff(dst_module)`
		`+`
		`return diff`

		`def get_rpm_md(self, rpm):`
		`@@ -177,5 +200,19 @@`

		`return md`

		`- def diff_modulemd(self):`
		`- raise NotImplementedError`
		`+ def modulemd_diff(self, dst_module):`
		`+ """`
		`+ Makes a module metadata diff.`
		`+ Args:`
		`+ dst_module (Module): Other Module object which we want to diff.`
		`+`
		`+ Returns:`
		`+ (str): returns a str with the results of a diff`
		`+ """`
		`+`
		`+ src_mmd_json = json.dumps(yaml.load(self.mmd.dumps()),`
		`+ indent=4, sort_keys=True, cls=MMDEncoder).splitlines()`
		`+ dst_mmd_json = json.dumps(yaml.load(dst_module.mmd.dumps()),`
		`+ indent=4, sort_keys=True, cls=MMDEncoder).splitlines()`
		`+`
		`+ return ndiff(src_mmd_json, dst_mmd_json)`

requirements.txt

file modified

+2 -1

		`@@ -1,4 +1,5 @@`
		`+ PyGObject`
		`+ PyYAML`
		`kobo`
		`koji`
		`- modulemd`
		`requests`

tests/conftest.py

file modified

+192

		`@@ -212,3 +212,195 @@`
		`''')`

		`return json.loads(mbs_data_text)`
		`+`
		`+`
		`+ @pytest.fixture`
		`+ def mbs_json_res_id_1795():`
		`+ """ Mock MBS response (JSON) - module ID 1795 """`
		`+`
		`+ mbs_data_text = (r'''`
		`+ {`
		`+ "build_context": "ef25c801bb4d5799a1453bd2d5798d3a597b9061",`
		`+ "component_builds": [`
		`+ 99443,`
		`+ 99441,`
		`+ 99440`
		`+ ],`
		`+ "context": "6c81f848",`
		`+ "id": 1795,`
		`+ "koji_tag": "module-nodejs-10-20180525120815-6c81f848",`
		+ "modulemd": "---\ndocument: modulemd\nversion: 2\ndata:\n name: nodejs\n stream: 10\n version: 20180525120815\n context: 6c81f848\n summary: Javascript runtime\n description: >-\n Node.js is a platform built on Chrome''s JavaScript runtime for easily building\n fast, scalable network applications. Node.js uses an event-driven, non-blocking\n I/O model that makes it lightweight and efficient, perfect for data-intensive\n real-time applications that run across distributed devices.\n license:\n module:\n - MIT\n xmd:\n mbs:\n scmurl: https://src.fedoraproject.org/modules/nodejs.git?#3f5a84799a6ff9fd5a24b3820f546f675ffaafc4\n buildrequires:\n platform:\n version: 4\n ref: f29\n stream: f29\n context: 00000000\n filtered_rpms: []\n mse: TRUE\n rpms:\n libuv:\n ref: db8ffab1c1559b150fb0ba58fcaece69c0a5c23f\n nodejs:\n ref: ad36c3e56371ec8d28e415840d8dae56ab452142\n commit: 3f5a84799a6ff9fd5a24b3820f546f675ffaafc4\n dependencies:\n - buildrequires:\n platform: [f29]\n requires:\n platform: [f29]\n references:\n community: http://nodejs.org\n documentation: http://nodejs.org/en/docs\n tracker: https://github.com/nodejs/node/issues\n profiles:\n default:\n rpms:\n - nodejs\n - npm\n development:\n rpms:\n - nodejs\n - nodejs-devel\n - npm\n minimal:\n rpms:\n - nodejs\n api:\n rpms:\n - nodejs\n - nodejs-devel\n - npm\n components:\n rpms:\n libuv:\n rationale: Platform abstraction layer for Node.js\n repository: git://pkgs.fedoraproject.org/rpms/libuv\n cache: http://pkgs.fedoraproject.org/repo/pkgs/libuv\n ref: 1\n nodejs:\n rationale: Javascript runtime and npm package manager.\n repository: git://pkgs.fedoraproject.org/rpms/nodejs\n cache: http://pkgs.fedoraproject.org/repo/pkgs/nodejs\n ref: 10\n buildorder: 10\n...\n",''' # noqa: E501
		`+ '''`
		`+ "name": "nodejs",`
		`+ "owner": "sgallagh",`
		`+ "rebuild_strategy": "only-changed",`
		`+ "ref_build_context": "ef25c801bb4d5799a1453bd2d5798d3a597b9061",`
		`+ "runtime_context": "d64e1e812ab9b003a6055ee0ba3da6d3e6237ed7",`
		`+ "scmurl": "https://src.fedoraproject.org/modules/nodejs.git?#3f5a84799a6ff9fd5a24b3820f546f675ffaafc4",''' # noqa: E501`
		`+ '''`
		`+ "siblings": [`
		`+ 1794`
		`+ ],`
		`+ "state": 5,`
		`+ "state_name": "ready",`
		`+ "state_reason": null,`
		`+ "state_trace": [`
		`+ {`
		`+ "reason": null,`
		`+ "state": 0,`
		`+ "state_name": "init",`
		`+ "time": "2018-05-25T12:08:54Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 1,`
		`+ "state_name": "wait",`
		`+ "time": "2018-05-25T12:08:55Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 2,`
		`+ "state_name": "build",`
		`+ "time": "2018-05-25T12:09:02Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 3,`
		`+ "state_name": "done",`
		`+ "time": "2018-05-25T20:02:54Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 5,`
		`+ "state_name": "ready",`
		`+ "time": "2018-05-25T20:03:00Z"`
		`+ }`
		`+ ],`
		`+ "state_url": "/module-build-service/1/module-builds/1795",`
		`+ "stream": "10",`
		`+ "tasks": {`
		`+ "rpms": {`
		`+ "libuv": {`
		`+ "nvr": "libuv-1.20.3-1.module_1743+72f89101",`
		`+ "state": 1,`
		`+ "state_reason": "Reused component from previous module build",`
		`+ "task_id": 26877622`
		`+ },`
		`+ "module-build-macros": {`
		`+ "nvr": "module-build-macros-0.1-1.module_1794+ffed6513",`
		`+ "state": 1,`
		`+ "state_reason": "",`
		`+ "task_id": 27187177`
		`+ },`
		`+ "nodejs": {`
		`+ "nvr": "nodejs-10.2.1-1.module_1794+ffed6513",`
		`+ "state": 1,`
		`+ "state_reason": "",`
		`+ "task_id": 27187302`
		`+ }`
		`+ }`
		`+ },`
		`+ "time_completed": "2018-05-25T20:02:54Z",`
		`+ "time_modified": "2018-05-25T20:03:00Z",`
		`+ "time_submitted": "2018-05-25T12:08:54Z",`
		`+ "version": "20180525120815"`
		`+ }`
		`+ ''')`
		`+`
		`+ return json.loads(mbs_data_text)`
		`+`
		`+`
		`+ @pytest.fixture`
		`+ def mbs_json_res_id_1874():`
		`+ """ Mock MBS response (JSON) - module ID 1874 """`
		`+`
		`+ mbs_data_text = (r'''`
		`+ {`
		`+ "build_context": "ef25c801bb4d5799a1453bd2d5798d3a597b9061",`
		`+ "component_builds": [`
		`+ 100324,`
		`+ 100327,`
		`+ 100325`
		`+ ],`
		`+ "context": "6c81f848",`
		`+ "id": 1874,`
		`+ "koji_tag": "module-nodejs-10-20180622010510-6c81f848",`
		+ "modulemd": "---\ndocument: modulemd\nversion: 2\ndata:\n name: nodejs\n stream: 10\n version: 20180622010510\n context: 6c81f848\n summary: Javascript runtime\n description: >-\n Node.js is a platform built on Chrome''s JavaScript runtime for easily building\n fast, scalable network applications. Node.js uses an event-driven, non-blocking\n I/O model that makes it lightweight and efficient, perfect for data-intensive\n real-time applications that run across distributed devices.\n license:\n module:\n - MIT\n xmd:\n mbs:\n scmurl: https://src.fedoraproject.org/modules/nodejs.git?#f41bdea38b920d67fda578a643a2e42da4f03b18\n buildrequires:\n platform:\n version: 4\n ref: f29\n stream: f29\n context: 00000000\n filtered_rpms: []\n mse: TRUE\n rpms:\n libuv:\n ref: db8ffab1c1559b150fb0ba58fcaece69c0a5c23f\n nodejs:\n ref: 011878379b079f9dc7e8845e468bd2375b2d228b\n commit: f41bdea38b920d67fda578a643a2e42da4f03b18\n dependencies:\n - buildrequires:\n platform: [f29]\n requires:\n platform: [f29]\n references:\n community: http://nodejs.org\n documentation: http://nodejs.org/en/docs\n tracker: https://github.com/nodejs/node/issues\n profiles:\n default:\n rpms:\n - nodejs\n - npm\n development:\n rpms:\n - nodejs\n - nodejs-devel\n - npm\n minimal:\n rpms:\n - nodejs\n api:\n rpms:\n - nodejs\n - nodejs-devel\n - npm\n components:\n rpms:\n libuv:\n rationale: Platform abstraction layer for Node.js\n repository: git://pkgs.fedoraproject.org/rpms/libuv\n cache: http://pkgs.fedoraproject.org/repo/pkgs/libuv\n ref: 1\n nodejs:\n rationale: Javascript runtime and npm package manager.\n repository: git://pkgs.fedoraproject.org/rpms/nodejs\n cache: http://pkgs.fedoraproject.org/repo/pkgs/nodejs\n ref: 10\n buildorder: 10\n...\n",''' # noqa: E501
		`+ '''`
		`+ "name": "nodejs",`
		`+ "owner": "sgallagh",`
		`+ "rebuild_strategy": "only-changed",`
		`+ "ref_build_context": "ef25c801bb4d5799a1453bd2d5798d3a597b9061",`
		`+ "runtime_context": "d64e1e812ab9b003a6055ee0ba3da6d3e6237ed7",`
		`+ "scmurl": "https://src.fedoraproject.org/modules/nodejs.git?#f41bdea38b920d67fda578a643a2e42da4f03b18",''' # noqa: E501`
		`+ '''`
		`+ "siblings": [`
		`+ 1873`
		`+ ],`
		`+ "state": 5,`
		`+ "state_name": "ready",`
		`+ "state_reason": null,`
		`+ "state_trace": [`
		`+ {`
		`+ "reason": null,`
		`+ "state": 0,`
		`+ "state_name": "init",`
		`+ "time": "2018-06-22T01:05:37Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 1,`
		`+ "state_name": "wait",`
		`+ "time": "2018-06-22T01:05:39Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 2,`
		`+ "state_name": "build",`
		`+ "time": "2018-06-22T01:05:56Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 3,`
		`+ "state_name": "done",`
		`+ "time": "2018-06-22T04:12:34Z"`
		`+ },`
		`+ {`
		`+ "reason": null,`
		`+ "state": 5,`
		`+ "state_name": "ready",`
		`+ "time": "2018-06-22T04:12:41Z"`
		`+ }`
		`+ ],`
		`+ "state_url": "/module-build-service/1/module-builds/1874",`
		`+ "stream": "10",`
		`+ "tasks": {`
		`+ "rpms": {`
		`+ "libuv": {`
		`+ "nvr": "libuv-1.20.3-1.module_1743+72f89101",`
		`+ "state": 1,`
		`+ "state_reason": "Reused component from previous module build",`
		`+ "task_id": 26877622`
		`+ },`
		`+ "module-build-macros": {`
		`+ "nvr": "module-build-macros-0.1-1.module_1873+3ca07718",`
		`+ "state": 1,`
		`+ "state_reason": "",`
		`+ "task_id": 27773131`
		`+ },`
		`+ "nodejs": {`
		`+ "nvr": "nodejs-10.5.0-1.module_1873+3ca07718",`
		`+ "state": 1,`
		`+ "state_reason": "",`
		`+ "task_id": 27773337`
		`+ }`
		`+ }`
		`+ },`
		`+ "time_completed": "2018-06-22T04:12:34Z",`
		`+ "time_modified": "2018-06-22T04:12:41Z",`
		`+ "time_submitted": "2018-06-22T01:05:37Z",`
		`+ "version": "20180622010510"`
		`+ }`
		`+ ''')`
		`+`
		`+ return json.loads(mbs_data_text)`

tests/test_cli.py

file modified

+60 -4

		`@@ -6,14 +6,17 @@`
		`imp.load_source("module_diff_bin", "bin/module_diff") # noqa`
		`import requests_mock`

		`+ from module_diff.mbs import MBS`
		`from module_diff_bin import main`


		`- module1_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=mariadb&stream=10.2&version=20171019133930&context=00000000")`
		`+ module1_url = (`
		`+ "{}?verbose=true&name=mariadb&stream=10.2&version=20171019133930&context=00000000".format(`
		`+ MBS.mbs_url))`

		`- module2_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=mariadb&stream=10.2&version=20171103103655&context=00000000")`
		`+ module2_url = (`
		`+ "{}?verbose=true&name=mariadb&stream=10.2&version=20171103103655&context=00000000".format(`
		`+ MBS.mbs_url))`


		`def generate_module_res(mbs_json_res):`
		`@@ -162,6 +165,30 @@`
		`assert "RPMs which dist tag changed" not in output`
		`assert "RPMs Reused" in output`

		`+ def test_cli_mmd_diff_option(self, mbs_json_res, capsys):`
		`+ """`
		`+ Testing option --mmd-diff which displays module metadata differences.`
		`+ """`
		`+ module1_res, module2_res = generate_module_res(mbs_json_res)`
		`+`
		`+ with requests_mock.mock() as mock_http:`
		`+ mock_http.register_uri("GET", module1_url, json=module1_res)`
		`+ mock_http.register_uri("GET", module2_url, json=module2_res)`
		`+`
		`+ cli_cmd = [`
		`+ 'module_diff',`
		`+ 'mariadb-10.2-20171019133930-00000000',`
		`+ 'mariadb-10.2-20171103103655-00000000',`
		`+ '--mmd-diff'`
		`+ ]`
		`+`
		`+ with mock.patch("sys.argv", cli_cmd):`
		`+ main()`
		`+ output = capsys.readouterr()[0]`
		`+`
		`+ check_common_output(output)`
		`+ assert "Module metadata diff" in output`
		`+`
		`def test_cli_json_option(self, mbs_json_res, capsys):`
		`"""`
		`Testing option --json which displays the output as a json string.`
		`@@ -186,3 +213,32 @@`
		`json_dict = json.loads(output)`
		`assert isinstance(json_dict, dict)`
		`assert len(json_dict)`
		`+`
		`+ def test_colors(self, mbs_json_res, capsys):`
		`+ """`
		`+ Test toggling of --no-colors option.`
		`+ """`
		`+ module1_res, module2_res = generate_module_res(mbs_json_res)`
		`+`
		`+ with requests_mock.mock() as mock_http:`
		`+ mock_http.register_uri("GET", module1_url, json=module1_res)`
		`+ mock_http.register_uri("GET", module2_url, json=module2_res)`
		`+`
		`+ cli_cmd = [`
		`+ 'module_diff',`
		`+ 'mariadb-10.2-20171019133930-00000000',`
		`+ 'mariadb-10.2-20171103103655-00000000'`
		`+ ]`
		`+`
		`+ cli_cmd_no_colors = list(cli_cmd)`
		`+ cli_cmd_no_colors.append('--no-colors')`
		`+`
		`+ with mock.patch("sys.argv", cli_cmd):`
		`+ main()`
		`+ output = capsys.readouterr()[0]`
		`+ assert '\x1b[' in output`
		`+`
		`+ with mock.patch("sys.argv", cli_cmd_no_colors):`
		`+ main()`
		`+ output = capsys.readouterr()[0]`
		`+ assert '\x1b[' not in output`

tests/test_mbs.py

file modified

+16 -4

		`@@ -7,8 +7,9 @@`
		`class TestMBS(object):`

		`def test_get_module(self, mbs_json_res):`
		`- api_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=mariadb&stream=f26&version=20170517122119&context=00000000")`
		`+ api_url = (`
		`+ "{}?verbose=true&name=mariadb&stream=f26&version=20170517122119"`
		`+ "&context=00000000".format(MBS.mbs_url))`

		`with requests_mock.mock() as mock_http:`
		`mock_http.get(api_url, json=mbs_json_res)`
		`@@ -17,9 +18,20 @@`

		`assert module == mbs_json_res["items"][0]`

		`+ def test_get_module_by_id(self, mbs_json_res):`
		`+ api_url = "{}476?verbose=true".format(MBS.mbs_url)`
		`+`
		`+ with requests_mock.mock() as mock_http:`
		`+ mock_http.get(api_url, json=mbs_json_res)`
		`+ mbs = MBS()`
		`+ module = mbs.get_module("476")`
		`+`
		`+ assert module == mbs_json_res["items"][0]`
		`+`
		`def test_get_module_no_match(self, mbs_json_res_no_match):`
		`- api_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=True&name=mariadb&stream=f26&version=20010101000000&context=00000000")`
		`+ api_url = (`
		`+ "{}?verbose=True&name=mariadb&stream=f26&version=20010101000000"`
		`+ "&context=00000000".format(MBS.mbs_url))`

		`with requests_mock.mock() as mock_http:`
		`mock_http.get(api_url, json=mbs_json_res_no_match)`

tests/test_module.py

file modified

+40 -6

		`@@ -6,15 +6,21 @@`
		`from module_diff.module import Module`


		`- module1_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=mariadb&stream=10.2&version=20171019133930&context=00000000")`
		`+ module1_url = (`
		`+ "{}?verbose=true&name=mariadb&stream=10.2&version=20171019133930&context=00000000".format(`
		`+ MBS.mbs_url))`

		`- module2_url = ("https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=mariadb&stream=10.2&version=20171103103655&context=00000000")`
		`+ module2_url = (`
		`+ "{}?verbose=true&name=mariadb&stream=10.2&version=20171103103655&context=00000000".format(`
		`+ MBS.mbs_url))`

		`module_diff_ns_url = (`
		`- "https://mbs.fedoraproject.org/module-build-service/1/module-builds/"`
		`- "?verbose=true&name=platform&stream=f27&version=20171011150016&context=00000000")`
		`+ "{}?verbose=true&name=platform&stream=f27&version=20171011150016&context=00000000".format(`
		`+ MBS.mbs_url))`
		`+`
		`+ module_id_1795_url = "{}1795?verbose=true".format(MBS.mbs_url)`
		`+`
		`+ module_id_1874_url = "{}1874?verbose=true".format(MBS.mbs_url)`


		`@pytest.mark.usefixtures("mbs_json_res")`
		`@@ -232,6 +238,34 @@`
		`("mariadb-debugsource-3:10.2.9-3.module_629c669e",`
		`"mariadb-debugsource-3:10.2.9-3.module_15d35d06")]`

		`+ def test_get_mmd_diff(self, mbs_json_res_id_1795, mbs_json_res_id_1874):`
		`+ """`
		`+ Testing module metadata differences`
		`+ """`
		`+ module1_res = copy.deepcopy(mbs_json_res_id_1795)`
		`+ module2_res = copy.deepcopy(mbs_json_res_id_1874)`
		`+`
		`+ with requests_mock.mock() as mock_http:`
		`+ mock_http.register_uri("GET", module_id_1795_url, json=module1_res)`
		`+ mock_http.register_uri("GET", module_id_1874_url, json=module2_res)`
		`+ mbs = MBS()`
		`+ module1 = Module(mbs.get_module("1795"), mmd_diff=True)`
		`+ module2 = Module(mbs.get_module("1874"), mmd_diff=True)`
		`+`
		`+ diff = module1.get_diff(module2)`
		`+`
		`+ assert diff["added_rpms"] == []`
		`+ assert diff["removed_rpms"] == []`
		`+ assert diff["upgraded_rpms"] == [('nodejs-10.2.1-1.module_1794+ffed6513',`
		`+ 'nodejs-10.5.0-1.module_1873+3ca07718')]`
		`+ assert diff["downgraded_rpms"] == []`
		`+`
		`+ lines = [l.strip() for l in diff["mmd_diff"]]`
		`+ assert '- "version": 20180525120815,' in lines`
		`+ assert '+ "version": 20180622010510,' in lines`
		`+ assert '- "commit": "3f5a84799a6ff9fd5a24b3820f546f675ffaafc4",' in lines`
		`+ assert '+ "commit": "f41bdea38b920d67fda578a643a2e42da4f03b18",' in lines`
		`+`
		`def test_get_rpm_md(self, mbs_json_res):`
		`"""`
		`Testing to get correct metadata from name of a rpm file.`

fivaldi commented 7 years ago

This PR is meant to be merged after PR#9 as it already includes its changes.
This PR targets issues #3, #5.

The main feature is module metadata diff support. This is a JSON interpretation of the metadata which is suitable for comparing between module versions. Modulemd has been removed and superseded with libmodulemd, which extends functionality of module_diff to the modulemd specification version 2.

The MBS API now supports getting module by ID, which has also been brought to the CLI.

rebased onto e728f8d

7 years ago

mcurlej commented on line 16 of bin/module_diff 7 years ago

so this is nsvc or and id of an nsvc?

mcurlej commented on line 4 of module_diff/__init__.py 7 years ago

Make sure that the colors can be turned off, module_diff should be able to be used as a lib not only as a cli tool.

fivaldi commented on line 16 of bin/module_diff 7 years ago

It's either NSVC, or module ID.

1 new commit added

Add --no-colors option

7 years ago

fivaldi commented 7 years ago

@mcurlej Please check the latest commit dd67a00.

1 new commit added

Imitate .copy() due to Python 2.7

7 years ago

fivaldi commented 7 years ago

@mcurlej From my POV it's complete & ready to be merged. Should I fix something/add some minor related changes?

Edited 7 years ago by fivaldi

fivaldi commented 7 years ago

@mcurlej I'd like to remind this PR... Thx!

ralph commented 7 years ago

@fivaldi, pagure says "The pull-request cannot be merged due to conflicts". Can you rebase on master?

fivaldi commented 7 years ago

@ralph Thx, conflicts fixed, rebased & re-tested....

  py27: commands succeeded
  py36: commands succeeded
  flake8: commands succeeded
  congratulations :)

rebased onto db2cf3f

7 years ago

ralph commented 7 years ago

Thanks Filip!

Pull-Request has been merged by ralph

7 years ago

Metadata

Assignee

None

Tags

No Tags

Changes Summary 10

module_diff/__init__.py

module_diff/module.py

module_diff

Source Code

#10 Module metadata diff Merged 7 years ago by ralph. Opened 7 years ago by fivaldi. fivaldi/module_diff fivaldi_diff_modulemd into master

Metadata

Changes Summary 10

#10 Module metadata diff

Merged 7 years ago by ralph. Opened 7 years ago by fivaldi.

fivaldi/module_diff fivaldi_diff_modulemd into master