PR#87: Exclude modular packages from product listings - compose-utils

compose-utils

#87 Exclude modular packages from product listings

Merged 5 years ago by lsedlar. Opened 5 years ago by lsedlar.

lsedlar/compose-utils modular-packages-in-listing into master

Exclude modular packages from product listings

Lubomír Sedlář • 5 years ago

b4a3b9e

bin/compose-dump-listings

file modified

+7 -1

		`@@ -19,12 +19,18 @@`
		`if opts.compose.endswith("rpms.json"):`
		`rpms = productmd.Rpms()`
		`rpms.load(opts.compose)`
		`+ modular_packages = {}`
		`else:`
		`compose = productmd.compose.Compose(opts.compose)`
		`rpms = compose.rpms`
		`+ modular_packages = compose_utils.get_modular_packages(compose)`

		`listing = compose_utils.get_listing(`
		`- rpms, variants=opts.variant, arches=opts.arch, pkg_arches=opts.pkg_arch`
		`+ rpms,`
		`+ variants=opts.variant,`
		`+ arches=opts.arch,`
		`+ pkg_arches=opts.pkg_arch,`
		`+ skip_nevras=modular_packages,`
		`)`
		`json.dump(listing, sys.stdout, sort_keys=True, indent=2)`

compose_utils/__init__.py

file modified

+17 -1

		`@@ -57,7 +57,7 @@`
		`sys.exit(1)`


		`- def get_listing(rpms, variants=[], arches=[], pkg_arches=[]):`
		`+ def get_listing(rpms, variants=[], arches=[], pkg_arches=[], skip_nevras={}):`
		`"""Create product listing suitable for using as prepopulate file in Pungi."""`
		`listing = {}`

		`@@ -67,10 +67,15 @@`
		`for arch in rpms.rpms[variant]:`
		`if arches and arch not in arches:`
		`continue`
		`+ to_skip = skip_nevras.get(variant, {}).get(arch, set())`
		`for srpm_nevra in rpms.rpms[variant][arch]:`
		`pkgs = []`
		`srpm_data = rpms.rpms[variant][arch][srpm_nevra]`
		`srpm_nvr = kobo.rpmlib.parse_nvra(srpm_nevra)`
		`+ if set(srpm_data.keys()) & to_skip:`
		`+ # If something from the build should be skipped, whole`
		`+ # build should be skipped.`
		`+ continue`
		`for nevra in srpm_data:`
		`nvra = kobo.rpmlib.parse_nvra(nevra)`
		`if pkg_arches and nvra["arch"] not in pkg_arches:`
		`@@ -85,3 +90,14 @@`
		`] = sorted(pkgs)`

		`return listing`
		`+`
		`+`
		`+ def get_modular_packages(compose):`
		`+ """Return a mapping from variant and arch to a set of modular package NVRAs."""`
		`+ mapping = {}`
		`+ for variant in compose.modules.modules:`
		`+ for arch in compose.modules.modules[variant]:`
		`+ for nsvc in compose.modules.modules[variant][arch]:`
		`+ module = compose.modules.modules[variant][arch][nsvc]`
		`+ mapping.setdefault(variant, {}).setdefault(arch, set()).update(module["rpms"])`
		`+ return mapping`

tests/composes/DP-1.0-20160315.t.0/compose/metadata/modules.json

file added

+72

		`@@ -0,0 +1,72 @@`
		`+ {`
		`+ "header": {`
		`+ "type": "productmd.modules",`
		`+ "version": "1.2"`
		`+ },`
		`+ "payload": {`
		`+ "compose": {`
		`+ "date": "20190117",`
		`+ "id": "Fedora-Rawhide-20190117.n.0",`
		`+ "respin": 0,`
		`+ "type": "nightly"`
		`+ },`
		`+ "modules": {`
		`+ "Server": {`
		`+ "aarch64": {`
		`+ "fish:3:3020190116191836:602da195": {`
		`+ "metadata": {`
		`+ "context": "602da195",`
		`+ "koji_tag": "module-fish-3-3020190116191836-602da195",`
		`+ "name": "fish",`
		`+ "stream": "3",`
		`+ "uid": "fish:3:3020190116191836:602da195",`
		`+ "version": "3020190116191836"`
		`+ },`
		`+ "modulemd_path": {`
		`+ "binary": "Modular/aarch64/os/repodata/895b54407639b9836d7d2d397d7e1962f5f6c9d92e056a2e4d8a102fe1163826-modules.yaml.gz"`
		`+ },`
		`+ "rpms": [`
		`+ "fish-0:3.0.0-1.module_f30+2710+8dbb4023.aarch64"`
		`+ ]`
		`+ },`
		`+ "fish:master:3020190116160550:602da195": {`
		`+ "metadata": {`
		`+ "context": "602da195",`
		`+ "koji_tag": "module-fish-master-3020190116160550-602da195",`
		`+ "name": "fish",`
		`+ "stream": "master",`
		`+ "uid": "fish:master:3020190116160550:602da195",`
		`+ "version": "3020190116160550"`
		`+ },`
		`+ "modulemd_path": {`
		`+ "binary": "Modular/aarch64/os/repodata/895b54407639b9836d7d2d397d7e1962f5f6c9d92e056a2e4d8a102fe1163826-modules.yaml.gz"`
		`+ },`
		`+ "rpms": [`
		`+ "fish-0:3.0.0-1.module_f30+2703+82e92e7e.aarch64"`
		`+ ]`
		`+ }`
		`+ },`
		`+ "x86_64": {`
		`+ "testmodule:master:3020190115184641:a5b0195c": {`
		`+ "metadata": {`
		`+ "context": "a5b0195c",`
		`+ "koji_tag": "module-testmodule-master-3020190115184641-a5b0195c",`
		`+ "name": "testmodule",`
		`+ "stream": "master",`
		`+ "uid": "testmodule:master:3020190115184641:a5b0195c",`
		`+ "version": "3020190115184641"`
		`+ },`
		`+ "modulemd_path": {`
		`+ "binary": "Modular/x86_64/os/repodata/5ff336ba62aa170e033efc2dc9e9b372b641900eb98cd9af53738e1838d3c27a-modules.yaml.gz"`
		`+ },`
		`+ "rpms": [`
		`+ "perl-List-Compare-0:0.53-11.module_f30+2696+4b2456b0.noarch",`
		`+ "dwm-user-0:6.1-8.module_f30+2696+4b2456b0.x86_64",`
		`+ "dwm-0:6.1-8.module_f30+2696+4b2456b0.x86_64"`
		`+ ]`
		`+ }`
		`+ }`
		`+ }`
		`+ }`
		`+ }`
		`+ }`

tests/fixtures/filtered-listing.json

file added

+131

		`@@ -0,0 +1,131 @@`
		`+ {`
		`+ "Client": {`
		`+ "i386": {`
		`+ "Dummy-firefox": [`
		`+ "Dummy-firefox-debuginfo.i686",`
		`+ "Dummy-firefox.i686"`
		`+ ],`
		`+ "Dummy-xulrunner": [`
		`+ "Dummy-xulrunner-debuginfo.i686",`
		`+ "Dummy-xulrunner.i686"`
		`+ ],`
		`+ "dummy-basesystem": [`
		`+ "dummy-basesystem.noarch"`
		`+ ],`
		`+ "dummy-bash": [`
		`+ "dummy-bash-debuginfo.i686",`
		`+ "dummy-bash.i686"`
		`+ ],`
		`+ "dummy-filesystem": [`
		`+ "dummy-filesystem.i686"`
		`+ ],`
		`+ "dummy-glibc": [`
		`+ "dummy-glibc-common.i686",`
		`+ "dummy-glibc-debuginfo-common.i686",`
		`+ "dummy-glibc-debuginfo.i686",`
		`+ "dummy-glibc.i686"`
		`+ ],`
		`+ "dummy-lvm2": [`
		`+ "dummy-lvm2-debuginfo.i686",`
		`+ "dummy-lvm2-libs.i686",`
		`+ "dummy-lvm2.i686"`
		`+ ],`
		`+ "dummy-tftp": [`
		`+ "dummy-tftp-debuginfo.i686",`
		`+ "dummy-tftp.i686"`
		`+ ]`
		`+ },`
		`+ "x86_64": {`
		`+ "Dummy-xulrunner": [`
		`+ "Dummy-xulrunner-debuginfo.x86_64",`
		`+ "Dummy-xulrunner.x86_64"`
		`+ ],`
		`+ "dummy-basesystem": [`
		`+ "dummy-basesystem.noarch"`
		`+ ],`
		`+ "dummy-bash": [`
		`+ "dummy-bash-debuginfo.x86_64",`
		`+ "dummy-bash.x86_64"`
		`+ ],`
		`+ "dummy-filesystem": [`
		`+ "dummy-filesystem.x86_64"`
		`+ ],`
		`+ "dummy-glibc": [`
		`+ "dummy-glibc-common.x86_64",`
		`+ "dummy-glibc-debuginfo-common.i686",`
		`+ "dummy-glibc-debuginfo-common.x86_64",`
		`+ "dummy-glibc-debuginfo.i686",`
		`+ "dummy-glibc-debuginfo.x86_64",`
		`+ "dummy-glibc.i686",`
		`+ "dummy-glibc.x86_64"`
		`+ ],`
		`+ "dummy-lvm2": [`
		`+ "dummy-lvm2-debuginfo.x86_64",`
		`+ "dummy-lvm2-libs.x86_64",`
		`+ "dummy-lvm2.x86_64"`
		`+ ],`
		`+ "dummy-tftp": [`
		`+ "dummy-tftp-debuginfo.x86_64",`
		`+ "dummy-tftp.x86_64"`
		`+ ]`
		`+ }`
		`+ },`
		`+ "Server": {`
		`+ "s390x": {`
		`+ "dummy-bash": [`
		`+ "dummy-bash-debuginfo.s390x",`
		`+ "dummy-bash.s390x"`
		`+ ],`
		`+ "dummy-filesystem": [`
		`+ "dummy-filesystem.s390x"`
		`+ ],`
		`+ "dummy-glibc": [`
		`+ "dummy-glibc-common.s390x",`
		`+ "dummy-glibc-debuginfo-common.s390x",`
		`+ "dummy-glibc-debuginfo.s390x",`
		`+ "dummy-glibc.s390x"`
		`+ ],`
		`+ "dummy-lvm2": [`
		`+ "dummy-lvm2-debuginfo.s390x",`
		`+ "dummy-lvm2-libs.s390x",`
		`+ "dummy-lvm2.s390x"`
		`+ ],`
		`+ "dummy-tftp": [`
		`+ "dummy-tftp-debuginfo.s390x",`
		`+ "dummy-tftp.s390x"`
		`+ ]`
		`+ },`
		`+ "x86_64": {`
		`+ "dummy-basesystem": [`
		`+ "dummy-basesystem.noarch"`
		`+ ],`
		`+ "dummy-bash": [`
		`+ "dummy-bash-debuginfo.x86_64",`
		`+ "dummy-bash.x86_64"`
		`+ ],`
		`+ "dummy-filesystem": [`
		`+ "dummy-filesystem.x86_64"`
		`+ ],`
		`+ "dummy-glibc": [`
		`+ "dummy-glibc-common.x86_64",`
		`+ "dummy-glibc-debuginfo-common.i686",`
		`+ "dummy-glibc-debuginfo-common.x86_64",`
		`+ "dummy-glibc-debuginfo.i686",`
		`+ "dummy-glibc-debuginfo.x86_64",`
		`+ "dummy-glibc.i686",`
		`+ "dummy-glibc.x86_64"`
		`+ ],`
		`+ "dummy-lvm2": [`
		`+ "dummy-lvm2-debuginfo.x86_64",`
		`+ "dummy-lvm2-devel.i686",`
		`+ "dummy-lvm2-devel.x86_64",`
		`+ "dummy-lvm2-libs.x86_64",`
		`+ "dummy-lvm2.x86_64"`
		`+ ],`
		`+ "dummy-tftp": [`
		`+ "dummy-tftp-debuginfo.x86_64",`
		`+ "dummy-tftp.x86_64"`
		`+ ]`
		`+ }`
		`+ }`
		`+ }`

tests/test_listings.py

file modified

+43 -1

		`@@ -9,7 +9,7 @@`

		`from .helpers import get_compose, get_fixture`

		`- from compose_utils import get_listing`
		`+ from compose_utils import get_listing, get_modular_packages`


		`class ListingTest(unittest.TestCase):`
		`@@ -58,3 +58,45 @@`
		`listing = get_listing(compose.rpms, pkg_arches=["i686"], arches=["x86_64"])`

		`self.assertEqual(listing, expected)`
		`+`
		`+ def test_get_listing_skip_packages(self):`
		`+ compose = get_compose("DP-1.0-20160315.t.0")`
		`+ with open(get_fixture("filtered-listing.json")) as fh:`
		`+ expected = json.load(fh)`
		`+`
		`+ self.maxDiff = None`
		`+ skip_nevras = {`
		`+ "Server": {`
		`+ "s390x": set(["dummy-basesystem-0:10.0-6.noarch"]),`
		`+ },`
		`+ "Client": {`
		`+ "x86_64": set(["Dummy-firefox-0:16.0.1-1.x86_64"]),`
		`+ },`
		`+ }`
		`+`
		`+ listing = get_listing(compose.rpms, skip_nevras=skip_nevras)`
		`+`
		`+ self.assertEqual(listing, expected)`
		`+`
		`+`
		`+ class GetModularPackagesTest(unittest.TestCase):`
		`+`
		`+ def test_load(self):`
		`+ compose = get_compose("DP-1.0-20160315.t.0")`
		`+`
		`+ self.assertEqual(`
		`+ get_modular_packages(compose),`
		`+ {`
		`+ "Server": {`
		`+ "aarch64": set([`
		`+ "fish-0:3.0.0-1.module_f30+2710+8dbb4023.aarch64",`
		`+ "fish-0:3.0.0-1.module_f30+2703+82e92e7e.aarch64",`
		`+ ]),`
		`+ "x86_64": set([`
		`+ "perl-List-Compare-0:0.53-11.module_f30+2696+4b2456b0.noarch",`
		`+ "dwm-user-0:6.1-8.module_f30+2696+4b2456b0.x86_64",`
		`+ "dwm-0:6.1-8.module_f30+2696+4b2456b0.x86_64"`
		`+ ]),`
		`+ }`
		`+ }`
		`+ )`

lsedlar commented 5 years ago

It does not make much sense to include them. The listing is meant as input for Pungi's prepopulate list, and that will not handle modular packages nicely.