From 5d346f8dd37e005a3eb3d395ac5135c406ef6d20 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Apr 26 2019 13:18:50 +0000 Subject: Fix -debuginfo/-debugsource packages handling in KojiContentGenerator. Our current code has following issues with -debuginfo/-debugsource handling: - The foo-debuginfo is included in the MMD only when foo is included there, but some special packages contain custom -debuginfo sub-packages like foo-common-debuginfo without matching foo-common sub-package. With our current code, the foo-common-debuginfo is never included in the final MMD. - The foo-debugsource is included in the MMD only when foo.src.rpm, but some special packages contain custom -debuginfo sub-package. This commit changes the handling of -debuginfo/-debugsource like this: - The RPMs to include in the final MMD are evaluated in particular order now. At first we evaluate non-debug RPMs and then debug RPMs. - When handling the foo-debuginfo/foo-debugsource RPM, we include it in final MMD only in one of these cases: - The "foo" is included in the MMD file (it means it is not filtered out). - The "foo" package does not exist at all (it means only foo-debuginfo exists and we need to include this package unless filtered out) and in the same time the SRPM from which this -debuginfo/-debugsource RPM has been built is included in a final MMD (it means that there is at least some package from this build included - this handles case when only foo.src.rpm and foo-debugsource.rpm would be included in a final MMD, which would be wrong.) We also respect filters here, so it is possible to explicitely filter out also -debuginfo/-debugsource packages. --- diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index eefc009..3fe9da8 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -34,7 +34,6 @@ import subprocess import tempfile import time from io import open -from collections import defaultdict from itertools import chain import kobo.rpmlib @@ -564,25 +563,46 @@ class KojiContentGenerator(object): # Modulemd.SimpleSet into which we will add licenses of all RPMs. rpm_licenses = Modulemd.SimpleSet() - # RPM name suffixes that imply relationship to another RPM - group_suffixes = ("-debuginfo", "-debugsource") + # RPM name suffixes for debug RPMs. + debug_suffixes = ("-debuginfo", "-debugsource") - # Grouping of "main" RPM with the corresponding debuginfo and debugsource RPMs - grouped_rpms = defaultdict(list) - source_rpms = {} + # The Name:NEVRA of source RPMs which are included in final MMD. non_devel_source_rpms = {} + + # Map source RPM name to NEVRA. + source_rpms = {} + + # Names of binary RPMs in the Koji tag. + binary_rpm_names = set() + + # Names of binary RPMs for which the `self._should_include()` method returned True. + included_rpm_names = set() + + # Names source RPMs which have some RPM built from this SRPM included in a final MMD. + included_srpm_names = set() + + # We need to evaluate the non-debug RPMs at first to find out which are included + # in the final MMD and then decide whether to include the debug RPMs based on that. + # In order to do that, we need to group debug RPMs and non-debug RPMs. + # We also fill in the `binary_rpm_names` and `source_rpms` here. + debug_rpms = {} + non_debug_rpms = {} for nevra, rpm in self.rpms_dict.items(): - name = strip_suffixes(rpm["name"], group_suffixes) - grouped_rpms[name].append((nevra, rpm)) if rpm["arch"] == "src": - source_rpms[name] = nevra + source_rpms[rpm["name"]] = nevra + else: + binary_rpm_names.add(rpm["name"]) + if rpm["name"].endswith(debug_suffixes): + debug_rpms[nevra] = rpm + else: + non_debug_rpms[nevra] = rpm - # Check each RPM in `self.rpms_dict` to find out if it can be included in mmd + # Check each RPM in Koji tag to find out if it can be included in mmd # for this architecture. - for nevra, rpm in self.rpms_dict.items(): - # Filter out debug and source RPMs, these will later be included if - # the "main" RPM is included - if rpm["name"].endswith(group_suffixes) or rpm["arch"] == "src": + for nevra, rpm in chain(non_debug_rpms.items(), debug_rpms.items()): + # Filter out source RPMs, these will later be included if + # the "main" RPM is included. + if rpm["arch"] == "src": continue # Filter out RPMs which will never end up in final modulemd: @@ -599,7 +619,27 @@ class KojiContentGenerator(object): if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(exclusive_arches): continue - should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches) + # The debug RPMs are handled differently ... + if rpm["name"].endswith(debug_suffixes): + # We include foo-debuginfo/foo-debugsource RPMs only in one of these cases: + # - The "foo" is included in a MMD file (it means it is not filtered out). + # - The "foo" package does not exist at all (it means only foo-debuginfo exists + # and we need to include this package unless filtered out) and in the same time + # the SRPM from which this -debuginfo/-debugsource RPM has been built is included + # in a final MMD (it means that there is at least some package from this build + # included - this handles case when only foo.src.rpm and foo-debugsource.rpm + # would be included in the final MMD, which would be wrong.) + # We also respect filters here, so it is possible to explicitely filter out also + # -debuginfo/-debugsource packages. + main_rpm_name = strip_suffixes(rpm["name"], debug_suffixes) + if (main_rpm_name in included_rpm_names or ( + main_rpm_name not in binary_rpm_names + and rpm["srpm_name"] in included_srpm_names)): + should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches) + else: + should_include = False + else: + should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches) # A source RPM should be included in a -devel module only if all the # RPMs built from this source RPM are included in a -devel module. @@ -608,6 +648,8 @@ class KojiContentGenerator(object): # list for -devel modules. if should_include: non_devel_source_rpms[rpm["name"]] = rpm["srpm_nevra"] + included_rpm_names.add(rpm["name"]) + included_srpm_names.add(rpm["srpm_name"]) if self.devel and should_include: # In case this is a -devel module, we want to skip any RPMs which would normally be @@ -619,16 +661,11 @@ class KojiContentGenerator(object): # really should include and skip the others. continue - # Add RPM and its related RPMs to packages. - for related_nevra, related_rpm in grouped_rpms[rpm["name"]]: - if related_rpm["arch"] != rpm["arch"]: - continue - - rpm_artifacts.add(related_nevra) - # Not all RPMs have licenses (for example debuginfo packages). - license = related_rpm.get("license") - if license: - rpm_licenses.add(license) + rpm_artifacts.add(nevra) + # Not all RPMs have licenses (for example debuginfo packages). + license = rpm.get("license") + if license: + rpm_licenses.add(license) if self.devel: for source_nevra in set(source_rpms.values()) - set(non_devel_source_rpms.values()): diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 4853368..6bcd2a0 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -991,3 +991,185 @@ class TestBuild: side_effect=koji.GenericError("Build already exists asdv")) self.cg.koji_import() tagger.assert_called() + + def test_fill_in_rpms_list_debuginfo_deps(self): + """ + Tests that -debuginfo RPM required by other -debuginfo RPM is included in a RPM list. + + The python3-pymongo has matching python3-pymongo-debuginfo RPM which requires + python-pymongo-debuginfo RPM. All of them should appear in RPM list + """ + self._add_test_rpm( + "python-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64", + "python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src") + + self._add_test_rpm( + "python3-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64", + "python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src") + + self._add_test_rpm( + "python3-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64", + "python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src") + + self._add_test_rpm( + "python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src", + "python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src") + + mmd = self.cg.module.mmd() + mmd = self.cg._fill_in_rpms_list(mmd, "x86_64") + + assert set(mmd.get_rpm_artifacts().get()) == set([ + 'python-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64', + 'python3-pymongo-debuginfo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64', + 'python-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.src', + 'python3-pymongo-3.6.1-9.module+f29.1.0+2993+d789589b.x86_64']) + + def test_fill_in_rpms_list_debuginfo_deps_psycopg2(self): + """ + Tests that -debuginfo RPM required by other -debuginfo RPM is included in a RPM list + with the psycopg2 RPM test-case, because psycopg2 RPMs are built in kind of special + way... + """ + self._add_test_rpm( + "python2-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-debug-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-tests-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-debug-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + mmd = self.cg.module.mmd() + mmd = self.cg._fill_in_rpms_list(mmd, "x86_64") + + assert set(mmd.get_rpm_artifacts().get()) == set([ + "python2-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python2-psycopg2-debug-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-debuginfo-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python2-psycopg2-tests-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python2-psycopg2-debug-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src"]) + + def test_fill_in_rpms_list_debugsource_for_non_srpm(self): + self._add_test_rpm( + "python2-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + mmd = self.cg.module.mmd() + mmd = self.cg._fill_in_rpms_list(mmd, "x86_64") + + assert set(mmd.get_rpm_artifacts().get()) == set([ + "python2-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src"]) + + def test_fill_in_rpms_list_debuginfo_deps_glibc(self): + self._add_test_rpm( + "glibc-common-2.29.9000-16.fc31.x86_64", + "glibc-2.29.9000-16.fc31.src") + + self._add_test_rpm( + "glibc-2.29.9000-16.fc31.x86_64", + "glibc-2.29.9000-16.fc31.src") + + self._add_test_rpm( + "glibc-debuginfo-common-2.29.9000-16.fc31.x86_64", + "glibc-2.29.9000-16.fc31.src") + + self._add_test_rpm( + "glibc-debuginfo-2.29.9000-16.fc31.x86_64", + "glibc-2.29.9000-16.fc31.src") + + self._add_test_rpm( + "glibc-2.29.9000-16.fc31.src", + "glibc-2.29.9000-16.fc31.src") + + mmd = self.cg.module.mmd() + mmd = self.cg._fill_in_rpms_list(mmd, "x86_64") + + assert set(mmd.get_rpm_artifacts().get()) == set([ + "glibc-common-2.29.9000-16.fc31.x86_64", + "glibc-2.29.9000-16.fc31.src", + "glibc-2.29.9000-16.fc31.x86_64", + "glibc-debuginfo-common-2.29.9000-16.fc31.x86_64", + "glibc-debuginfo-2.29.9000-16.fc31.x86_64"]) + + def test_fill_in_rpms_list_debuginfo_deps_kernel(self): + self._add_test_rpm( + "kernel-debuginfo-common-aarch64-5.0.9-301.fc30.aarch64", + "kernel-5.0.9-301.fc30.src") + + self._add_test_rpm( + "kernel-debuginfo-5.0.9-301.fc30.aarch64", + "kernel-5.0.9-301.fc30.src") + + self._add_test_rpm( + "kernel-5.0.9-301.fc30.aarch64", + "kernel-5.0.9-301.fc30.src") + + self._add_test_rpm( + "kernel-5.0.9-301.fc30.src", + "kernel-5.0.9-301.fc30.src") + + mmd = self.cg.module.mmd() + mmd = self.cg._fill_in_rpms_list(mmd, "aarch64") + + assert set(mmd.get_rpm_artifacts().get()) == set([ + "kernel-debuginfo-common-aarch64-5.0.9-301.fc30.aarch64", + "kernel-5.0.9-301.fc30.src", + "kernel-debuginfo-5.0.9-301.fc30.aarch64", + "kernel-5.0.9-301.fc30.aarch64"]) + + def test_fill_in_rpms_list_debugsource_not_included(self): + self._add_test_rpm( + "python-psycopg2-debugsource-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python2-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.x86_64", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + self._add_test_rpm( + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src", + "python-psycopg2-2.7.5-7.module+f29.0.0+2961+596d0223.src") + + mmd = self.cg.module.mmd() + filter_list = Modulemd.SimpleSet() + filter_list.add("python2-psycopg2") + mmd.set_rpm_filter(filter_list) + mmd = self.cg._fill_in_rpms_list(mmd, "x86_64") + + assert set(mmd.get_rpm_artifacts().get()) == set([])