From 2f7dfee0568537e8433194e90fbf494e0021c1de Mon Sep 17 00:00:00 2001 From: Lubomír Sedlář Date: Oct 15 2018 07:11:18 +0000 Subject: Add command to print summary of compose contents JIRA: COMPOSE-2524 Signed-off-by: Lubomír Sedlář --- diff --git a/README.rst b/README.rst index dfd1d2a..e8cd97e 100644 --- a/README.rst +++ b/README.rst @@ -31,6 +31,9 @@ Contents copy parts of compose (filtering by variant or architecture) to another location via ``rsync``. +**compose-print-essentials** + print basic information about a compose + **compose-report-package-moves** identify rpm packages that moved between related variants between two composes. Useful for identification of packages which were moved from diff --git a/bin/compose-print-essentials b/bin/compose-print-essentials new file mode 100755 index 0000000..16b6892 --- /dev/null +++ b/bin/compose-print-essentials @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import os +import sys + +here = sys.path[0] +if here != "/usr/bin": + sys.path[0] = os.path.dirname(here) + +from compose_utils import essentials + +if __name__ == "__main__": + essentials.main() diff --git a/compose_utils/essentials.py b/compose_utils/essentials.py new file mode 100644 index 0000000..02f49da --- /dev/null +++ b/compose_utils/essentials.py @@ -0,0 +1,94 @@ +# -*- encoding: utf-8 -*- + +import argparse +import collections +import json +import os + +import productmd.common +import productmd.compose + + +def get_package_version(compose, pkg): + """Find all SRPMs with given name.""" + kernels = set() + for variant in compose.rpms.rpms: + for arch in compose.rpms.rpms[variant]: + for build in compose.rpms.rpms[variant][arch]: + n, v, r = build.rsplit("-", 2) + if n == pkg: + kernels.add(build.replace(".src", "")) + if kernels: + return "%s: %s" % (pkg.capitalize(), ", ".join(sorted(kernels))) + + +def get_containers(compose): + """Get details about containers built in OSBS: the NVR, name label and + registry where it can be retrieved from. + """ + metadata_uri = os.path.join(compose.compose_path, "metadata/osbs.json") + try: + # This should probably be exposed by productmd in some better way. + # Unlike regular open(), this will handle HTTP urls transaprently. + with productmd.common._open_file_obj(metadata_uri) as f: + metadata = json.load(f) + except IOError: + # Failed to open metadata, there are no containers. + return None + containers = collections.defaultdict(list) + for variant in metadata: + for arch in metadata[variant]: + for container in metadata[variant][arch]: + containers[ + ( + container["nvr"], + container["docker"]["config"]["config"]["Labels"]["name"], + ) + ].append(container["docker"]["repositories"][0]) + if containers: + return "Containers:\n" + "\n".join( + _format_container(c[0], c[1], r) for c, r in containers.items() + ) + + +def _format_container(nvr, name, repos): + return " * %s (%s)\n%s" % (nvr, name, "\n".join(" - %s" % r for r in repos)) + + +def get_images(compose, types=["qcow2"]): + """Get list of image filenames for given types.""" + images = set() + for variant in compose.images.images: + for arch in compose.images.images[variant]: + for image in compose.images.images[variant][arch]: + if image.type in types: + images.add(os.path.basename(image.path)) + if images: + return "Images:\n%s" % "\n".join(" * %s" % x for x in sorted(images)) + + +def get_essentials(compose_path): + compose = productmd.compose.Compose(compose_path) + return filter( + None, + [ + get_package_version(compose, "kernel"), + get_package_version(compose, "lorax"), + get_package_version(compose, "anaconda"), + get_containers(compose), + get_images(compose) + ], + ) + + +def print_details(details): + for detail in details: + print(detail) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument("COMPOSE", help="compose to inspect") + opts = parser.parse_args(args) + details = get_essentials(opts.COMPOSE) + print_details(details) diff --git a/doc/compose-print-essentials.1 b/doc/compose-print-essentials.1 new file mode 100644 index 0000000..07f223f --- /dev/null +++ b/doc/compose-print-essentials.1 @@ -0,0 +1,23 @@ +.TH compose-print-essentials 1 +.SH NAME +compose-print-essentials \- print basic information about a compose +.SH SYNOPSIS +.B compose-print-essentials +[\fIOPTIONS\fR...] +\fICOMPOSE_PATH\fR +.SH DESCRIPTION +.B compose-print-essentials +loads metadata from a compose and prints a summary of basic information: which +kernel version is used, what containers were built and to which registries they +were pushed, and a list of other images that were created in the compose. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Print help and exit. +.SH EXIT CODE +Exit code is always 0 on success and non-zero if there was a problem loading +the compose. +.SH BUGS +Please report bugs at +.br +https://pagure.io/compose-utils/issues diff --git a/setup.py b/setup.py index 96d19c3..d8dd0a8 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ setup( 'bin/compose-latest-symlink', 'bin/compose-list', 'bin/compose-partial-copy', + 'bin/compose-print-essentials', 'bin/compose-report-package-moves', ], install_requires=[ @@ -39,6 +40,7 @@ setup( 'doc/compose-has-build.1', 'doc/compose-list.1', 'doc/compose-partial-copy.1', + 'doc/compose-print-essentials.1', 'doc/compose-report-package-moves.1', ]), ], diff --git a/tests/composes/DP-1.0-20181012.t.0/COMPOSE_ID b/tests/composes/DP-1.0-20181012.t.0/COMPOSE_ID new file mode 100644 index 0000000..c6ac206 --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/COMPOSE_ID @@ -0,0 +1 @@ +DP-1.0-20160315.t.0 \ No newline at end of file diff --git a/tests/composes/DP-1.0-20181012.t.0/STATUS b/tests/composes/DP-1.0-20181012.t.0/STATUS new file mode 100644 index 0000000..23351c7 --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/STATUS @@ -0,0 +1 @@ +FINISHED diff --git a/tests/composes/DP-1.0-20181012.t.0/compose/metadata/composeinfo.json b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/composeinfo.json new file mode 100644 index 0000000..9dbf7f8 --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/composeinfo.json @@ -0,0 +1,46 @@ +{ + "header": { + "version": "1.0" + }, + "payload": { + "compose": { + "date": "20181012", + "id": "DP-1.0-20181012.t.0", + "respin": 0, + "type": "test" + }, + "release": { + "name": "Dummy Product", + "short": "DP", + "version": "1.0" + }, + "variants": { + "Client": { + "arches": [ + "i386", + "x86_64" + ], + "id": "Client", + "name": "Client", + "paths": { + }, + "type": "variant", + "uid": "Client", + "variants": [] + }, + "Server": { + "arches": [ + "s390x", + "x86_64" + ], + "id": "Server", + "name": "Server", + "paths": { + }, + "type": "variant", + "uid": "Server", + "variants": [] + } + } + } +} diff --git a/tests/composes/DP-1.0-20181012.t.0/compose/metadata/images.json b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/images.json new file mode 100644 index 0000000..47f2539 --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/images.json @@ -0,0 +1,61 @@ +{ + "header": { + "version": "1.0" + }, + "payload": { + "compose": { + "date": "20181012", + "id": "DP-1.0-20181012.t.0", + "respin": 0, + "type": "test" + }, + "images": { + "Client": { + "i386": [ + { + "arch": "i386", + "bootable": false, + "checksums": { + "md5": "9b32efc55699d38638f3d083bcf198c7", + "sha1": "1261a10b716966264ff8aa3c4772d97226604235", + "sha256": "9ddfe8bb8fde9be0452d533bcd89b3682fe2a425629bafeab0e969d101b80a85" + }, + "disc_count": 1, + "disc_number": 1, + "format": "qcow2", + "implant_md5": null, + "mtime": 1458031335, + "path": "Client/i386/images/DP-1.0-20181012.t.0-Client-i386.qcow2", + "size": 507904, + "type": "qcow2", + "volume_id": null, + "subvariant": "Client" + } + ] + }, + "Server": { + "x86_64": [ + { + "arch": "x86_64", + "bootable": false, + "checksums": { + "md5": "1f6a0052212667abe02dedf5787269a3", + "sha1": "f361070b2b681f976863f7aa1ba52bbc1f6a8b53", + "sha256": "cb47ccafe3357a63bb14c066dcadc1e4621d7331d20f34c735b90ec620e747cc" + }, + "disc_count": 1, + "disc_number": 1, + "format": "qcow2", + "implant_md5": null, + "mtime": 1458031335, + "path": "Server/x86_64/images/DP-1.0-20181012.t.0-Server-x86_64.qcow2", + "size": 577536, + "type": "qcow2", + "volume_id": null, + "subvariant": "Server" + } + ] + } + } + } +} diff --git a/tests/composes/DP-1.0-20181012.t.0/compose/metadata/osbs.json b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/osbs.json new file mode 100644 index 0000000..35fa72f --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/osbs.json @@ -0,0 +1,40 @@ +{ + "Server": { + "aarch64": [ + { + "nvr": "dp-1.0-1", + "docker": { + "config": { + "config": { + "Labels": { + "name": "base" + } + } + }, + "repositories": [ + "registry.example.com/dp:latest-aarch64", + "registry.example.com/dp:cafe" + ] + } + } + ], + "x86_64": [ + { + "nvr": "dp-1.0-1", + "docker": { + "config": { + "config": { + "Labels": { + "name": "base" + } + } + }, + "repositories": [ + "registry.example.com/dp:latest-x86_64", + "registry.example.com/dp:beef" + ] + } + } + ] + } +} diff --git a/tests/composes/DP-1.0-20181012.t.0/compose/metadata/rpms.json b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/rpms.json new file mode 100644 index 0000000..330d10e --- /dev/null +++ b/tests/composes/DP-1.0-20181012.t.0/compose/metadata/rpms.json @@ -0,0 +1,60 @@ +{ + "header": { + "version": "1.0" + }, + "payload": { + "compose": { + "date": "20181012", + "id": "DP-1.0-20181012.t.0", + "respin": 0, + "type": "test" + }, + "rpms": { + "Client": { + "i386": { + "Dummy-firefox-0:16.0.1-1.src": { + "Dummy-firefox-0:16.0.1-1.i686": { + "category": "binary", + "path": "Client/i386/os/Packages/d/Dummy-firefox-16.0.1-1.i686.rpm", + "sigkey": null + } + }, + "kernel-1.0-1.src": { + "Dummy-xulrunner-0:16.0.1-1.i686": { + "category": "binary", + "path": "Client/i386/os/Packages/d/Dummy-xulrunner-16.0.1-1.i686.rpm", + "sigkey": null + } + } + }, + "x86_64": { + "kernel-1.0-1.src": { + "Dummy-firefox-0:16.0.1-1.src": { + "category": "source", + "path": "Client/source/tree/Packages/d/Dummy-firefox-16.0.1-1.src.rpm", + "sigkey": null + } + }, + "dummy-filesystem-0:4.2.37-6.src": { + "dummy-filesystem-0:4.2.37-6.x86_64": { + "category": "binary", + "path": "Client/x86_64/os/Packages/d/dummy-filesystem-4.2.37-6.x86_64.rpm", + "sigkey": null + } + } + } + }, + "Server": { + "s390x": { + "kernel-special-2.0-1.src": { + "dummy-basesystem-0:10.0-6.src": { + "category": "source", + "path": "Server/source/tree/Packages/d/dummy-basesystem-10.0-6.src.rpm", + "sigkey": null + } + } + } + } + } + } +} diff --git a/tests/test_essentials.py b/tests/test_essentials.py new file mode 100644 index 0000000..d133e8b --- /dev/null +++ b/tests/test_essentials.py @@ -0,0 +1,83 @@ +# -*- encoding: utf-8 -*- + +from textwrap import dedent + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import mock +from six import StringIO + +from .helpers import get_compose, get_compose_path + +from compose_utils import essentials + + +class GetPackageVersionTest(unittest.TestCase): + def test_no_match(self): + compose = get_compose("DP-1.0-20160315.t.0") + self.assertIsNone(essentials.get_package_version(compose, "kernel")) + + def test_with_kernel(self): + compose = get_compose("DP-1.0-20181012.t.0") + self.assertEqual( + essentials.get_package_version(compose, "kernel"), + "Kernel: kernel-1.0-1", + ) + + +class GetContainersTest(unittest.TestCase): + def test_no_metadata(self): + compose = get_compose("DP-1.0-20160315.t.0") + self.assertIsNone(essentials.get_containers(compose)) + + def test_with_containers(self): + compose = get_compose("DP-1.0-20181012.t.0") + self.maxDiff = None + self.assertEqual( + essentials.get_containers(compose), + dedent( + """\ + Containers: + * dp-1.0-1 (base) + - registry.example.com/dp:latest-aarch64 + - registry.example.com/dp:latest-x86_64""" + ), + ) + + +class GetImagesTest(unittest.TestCase): + def test_no_metadata(self): + compose = get_compose("DP-1.0-20160315.t.0") + self.assertIsNone(essentials.get_images(compose)) + + def test_with_images(self): + compose = get_compose("DP-1.0-20181012.t.0") + self.assertEqual( + essentials.get_images(compose), + dedent( + """\ + Images: + * DP-1.0-20181012.t.0-Client-i386.qcow2 + * DP-1.0-20181012.t.0-Server-x86_64.qcow2""" + ), + ) + + +class GetEssentialsTest(unittest.TestCase): + def test_no_metadata(self): + compose_path = get_compose_path("DP-1.0-20160315.t.0") + self.assertEqual(essentials.get_essentials(compose_path), []) + + def test_with_metadata(self): + compose_path = get_compose_path("DP-1.0-20181012.t.0") + self.assertEqual(len(essentials.get_essentials(compose_path)), 3) + + +class PrintDetailsTest(unittest.TestCase): + def test_printing(self): + with mock.patch("sys.stdout", new_callable=StringIO) as out: + essentials.print_details(["foo", "bar"]) + self.assertEqual(out.getvalue(), "foo\nbar\n")