PR#1062: Import -devel CG build with RPMs which are filtered out of the current real CG build. - fm-orchestrator

fm-orchestrator

#1062 Import -devel CG build with RPMs which are filtered out of the current real CG build.

Merged 5 years ago by mprahl. Opened 5 years ago by jkaluza.

jkaluza/fm-orchestrator cg-devel-build into master

Import -devel CG build with RPMs which are filtered out of the current real CG build.

Jan Kaluza • 5 years ago

13ab184

module_build_service/builder/KojiContentGenerator.py

file modified

+92 -57

		`@@ -74,6 +74,7 @@`
		`self.module_name = module.name`
		`self.mmd = module.modulemd`
		`self.config = config`
		`+ self.devel = False`
		`# List of architectures the module is built for.`
		`self.arches = []`
		`# List of RPMs tagged in module.koji_tag as returned by Koji.`
		`@@ -261,6 +262,8 @@`
		`def _get_build(self):`
		`ret = {}`
		`ret[u'name'] = self.module.name`
		`+ if self.devel:`
		`+ ret['name'] += "-devel"`
		`ret[u'version'] = self.module.stream.replace("-", "_")`
		`# Append the context to the version to make NVRs of modules unique in the event of`
		`# module stream expansion`
		`@@ -276,7 +279,7 @@`
		`u"module_build_service_id": self.module.id,`
		`u"content_koji_tag": self.module.koji_tag,`
		`u"modulemd_str": self.module.modulemd,`
		`- u"name": self.module.name,`
		`+ u"name": ret['name'],`
		`u"stream": self.module.stream,`
		`u"version": self.module.version,`
		`u"context": self.module.context`
		`@@ -429,6 +432,68 @@`

		`return ret`

		`+ def _should_include_rpm(self, rpm, mmd, arch, multilib_arches):`
jkaluza commented 5 years ago This code is just moved from the `_fill_in_rpms_list` below.
		`+ """`
		+ Helper method for `_fill_in_rpms_list` returning True if the RPM object
		`+ should be included in a final MMD file for given arch.`
		`+ """`
		`+ # Check the "whitelist" buildopts section of MMD.`
		`+ # When "whitelist" is defined, it overrides component names in`
		+ # `mmd.get_rpm_components()`. The whitelist is used when module needs to build
		`+ # package with different SRPM name than the package name. This is case for example`
		`+ # for software collections where SRPM name can be "httpd24-httpd", but package name`
		`+ # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the`
		`+ # rpm["srpm_name"] would be "httpd24-httpd".`
		`+ srpm = rpm["srpm_name"]`
		`+ whitelist = None`
		`+ buildopts = mmd.get_buildopts()`
		`+ if buildopts:`
		`+ whitelist = buildopts.get_rpm_whitelist()`
		`+ if whitelist:`
		`+ if srpm not in whitelist:`
		`+ # Package is not in the whitelist, skip it.`
		`+ return False`
		`+`
		`+ # If there is no whitelist, just check that the SRPM name we have here`
		`+ # exists in the list of components.`
		`+ # In theory, there should never be situation where modular tag contains`
		`+ # some RPM built from SRPM not included in get_rpm_components() or in whitelist,`
		`+ # but the original Pungi code checked for this case.`
		`+ if not whitelist and srpm not in mmd.get_rpm_components().keys():`
		`+ return False`
		`+`
		`+ # Do not include this RPM if it is filtered.`
		`+ if rpm["name"] in mmd.get_rpm_filter().get():`
		`+ return False`
		`+`
		`+ # Skip the rpm if it's built for multilib arch, but`
		`+ # multilib is not enabled for this srpm in MMD.`
		`+ try:`
		`+ mmd_component = mmd.get_rpm_components()[srpm]`
		`+ multilib = mmd_component.get_multilib()`
		`+ multilib = multilib.get() if multilib else set()`
		+ # The `multilib` set defines the list of architectures for which
		`+ # the multilib is enabled.`
		`+ #`
		`+ # Filter out RPMs from multilib architectures if multilib is not`
		`+ # enabled for current arch. Keep the RPMs from non-multilib compatible`
		`+ # architectures.`
		`+ if arch not in multilib and rpm["arch"] in multilib_arches:`
		`+ return False`
		`+ except KeyError:`
		`+ # TODO: This exception is raised only when "whitelist" is used.`
		`+ # Since components in whitelist have different names than ones in`
		`+ # components list, we won't find them there.`
		`+ # We would need to track the RPMs srpm_name from whitelist back to`
		`+ # original package name used in MMD's components list. This is possible`
		`+ # but original Pungi code is not doing that. This is TODO for future`
		`+ # improvements.`
		`+`
		`+ # No such component, disable any multilib`
		`+ if rpm["arch"] not in ("noarch", arch):`
		`+ return False`
		`+ return True`
		`+`
		`def _fill_in_rpms_list(self, mmd, arch):`
		`"""`
		Fills in the list of built RPMs in architecture specific `mmd` for `arch`
		`@@ -473,8 +538,6 @@`
		`rpm["arch"] not in [arch, "noarch", "src"]):`
		`continue`

		`- srpm = rpm["srpm_name"]`
		`-`
		`# Skip the RPM if it is excluded on this arch or exclusive`
		`# for different arch.`
		`if rpm["excludearch"] and set(rpm["excludearch"]) & set(exclusive_arches):`
		`@@ -482,61 +545,17 @@`
		`if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(exclusive_arches):`
		`continue`

		`- # Check the "whitelist" buildopts section of MMD.`
		`- # When "whitelist" is defined, it overrides component names in`
		- # `mmd.get_rpm_components()`. The whitelist is used when module needs to build
		`- # package with different SRPM name than the package name. This is case for example`
		`- # for software collections where SRPM name can be "httpd24-httpd", but package name`
		`- # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the`
		`- # rpm["srpm_name"] would be "httpd24-httpd".`
		`- whitelist = None`
		`- buildopts = mmd.get_buildopts()`
		`- if buildopts:`
		`- whitelist = buildopts.get_rpm_whitelist()`
		`- if whitelist:`
		`- if srpm not in whitelist:`
		`- # Package is not in the whitelist, skip it.`
		`- continue`
		`-`
		`- # If there is no whitelist, just check that the SRPM name we have here`
		`- # exists in the list of components.`
		`- # In theory, there should never be situation where modular tag contains`
		`- # some RPM built from SRPM not included in get_rpm_components() or in whitelist,`
		`- # but the original Pungi code checked for this case.`
		`- if not whitelist and srpm not in mmd.get_rpm_components().keys():`
		`+ should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)`
		`+ if self.devel and should_include:`
		`+ # In case this is -devel module, we want to skip any RPMs which would be normally`
		`+ # include in a module and only keep those which wouldn't be included, because`
mprahl commented 5 years ago `include` => `included`
		`+ # -devel is complement to normal module build.`
		`continue`
		`-`
		`- # Do not include this RPM if it is filtered.`
		`- if rpm["name"] in mmd.get_rpm_filter().get():`
		`+ elif not self.devel and not should_include:`
		`+ # In chase this is normal (non-devel) module, include only packages which we`
mprahl commented 5 years ago `chase` => `case`
		`+ # really should include and skip the others.`
		`continue`

		`- # Skip the rpm if it's built for multilib arch, but`
		`- # multilib is not enabled for this srpm in MMD.`
		`- try:`
		`- mmd_component = mmd.get_rpm_components()[srpm]`
		`- multilib = mmd_component.get_multilib()`
		`- multilib = multilib.get() if multilib else set()`
		- # The `multilib` set defines the list of architectures for which
		`- # the multilib is enabled.`
		`- #`
		`- # Filter out RPMs from multilib architectures if multilib is not`
		`- # enabled for current arch. Keep the RPMs from non-multilib compatible`
		`- # architectures.`
		`- if arch not in multilib and rpm["arch"] in multilib_arches:`
		`- continue`
		`- except KeyError:`
		`- # TODO: This exception is raised only when "whitelist" is used.`
		`- # Since components in whitelist have different names than ones in`
		`- # components list, we won't find them there.`
		`- # We would need to track the RPMs srpm_name from whitelist back to`
		`- # original package name used in MMD's components list. This is possible`
		`- # but original Pungi code is not doing that. This is TODO for future`
		`- # improvements.`
		`-`
		`- # No such component, disable any multilib`
		`- if rpm["arch"] not in ("noarch", arch):`
		`- continue`
		`-`
		`# Add RPM to packages.`
		`rpm_artifacts.add(nevra)`

		`@@ -558,6 +577,9 @@`
		`:return: Finalized modulemd string.`
		`"""`
		`mmd = self.module.mmd()`
		`+ if self.devel:`
		`+ mmd.set_name(mmd.get_name() + "-devel")`
		`+`
		`# Set the "Arch" field in mmd.`
		`mmd.set_arch(pungi.arch.tree_arch_to_yum_arch(arch))`
		`# Fill in the list of built RPMs.`
		`@@ -655,16 +677,29 @@`
		`session.tagBuild(tag_info["id"], nvr)`

		`def _load_koji_tag(self, koji_session):`
		`+ # Do not load Koji tag data if this method is called again. This would`
		`+ # waste resources, because the Koji tag content is always the same`
		`+ # for already built module.`
		`+ if self.arches and self.rpms and self.rpms_dict:`
		`+ return`
		`+`
		`tag = koji_session.getTag(self.module.koji_tag)`
		`self.arches = tag["arches"].split(" ") if tag["arches"] else []`
		`self.rpms = self._koji_rpms_in_tag(self.module.koji_tag)`
		`self.rpms_dict = {kobo.rpmlib.make_nvra(rpm, force_epoch=True): rpm for rpm in self.rpms}`

		`- def koji_import(self):`
		`+ def koji_import(self, devel=False):`
		`"""This method imports given module into the configured koji instance as`
		`a content generator based build`

		`- Raises an exception when error is encountered during import"""`
		`+ Raises an exception when error is encountered during import`
		`+`
		`+ :param bool devel: True if the "-devel" module should be created and imported.`
		`+ The "-devel" module build contains only the RPMs which are normally filtered`
		`+ from the module build. If set to False, normal module build respecting the`
		`+ filters is created and imported.`
		`+ """`
		`+ self.devel = devel`
		`session = get_session(self.config, self.owner)`
		`self._load_koji_tag(session)`

module_build_service/builder/KojiModuleBuilder.py

file modified

		`@@ -1176,6 +1176,7 @@`
		`if self.config.koji_enable_content_generator and self.module.state == 3:`
		`cg = KojiContentGenerator(self.module, self.config)`
		`cg.koji_import()`
		`+ cg.koji_import(devel=True)`

		`@staticmethod`
		`def get_rpm_module_tag(rpm):`

tests/test_content_generator.py

file modified

+73 -30

		`@@ -83,8 +83,9 @@`
		`@patch("platform.machine")`
		`@patch(("module_build_service.builder.KojiContentGenerator.KojiContentGenerator."`
		`"_koji_rpms_in_tag"))`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, coutput, popen,`
		`- get_session):`
		`+ get_session, devel):`
		`""" Test generation of content generator json """`
		`koji_session = MagicMock()`
		`koji_session.getUser.return_value = GET_USER_RV`
		`@@ -117,11 +118,17 @@`
		`build_logs.start(self.cg.module)`
		`build_logs.stop(self.cg.module)`

		`+ self.cg.devel = devel`
		`self.cg._load_koji_tag(koji_session)`
		`file_dir = self.cg._prepare_file_directory()`
		`ret = self.cg._get_content_generator_metadata(file_dir)`
		`rpms_in_tag.assert_called_once()`
		`- assert expected_output == ret`
		`+ if not devel:`
		`+ assert expected_output == ret`
		`+ else:`
		`+ # For devel, only check that the name has -devel suffix.`
		`+ assert ret["build"]["name"] == "nginx-devel"`
		`+ assert ret["build"]["extra"]["typeinfo"]["module"]["name"] == "nginx-devel"`

		`@patch("module_build_service.builder.KojiContentGenerator.get_session")`
		`@patch("subprocess.Popen")`
		`@@ -429,7 +436,8 @@`
		`self.cg.module.modulemd = mmd.dumps()`
		`self.cg.modulemd = mmd.dumps()`

		`- def test_fill_in_rpms_list(self):`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`+ def test_fill_in_rpms_list(self, devel):`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.s390x", "dhcp")`
		`@@ -439,15 +447,23 @@`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.s390x", "perl-Tangerine")`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.src", "perl-Tangerine")`

		`+ self.cg.devel = devel`
		`mmd = self.cg.module.mmd()`
		`mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")`

		`- # Only x86_64 packages should be filled in, because we requested x86_64 arch.`
		`- assert set(mmd.get_rpm_artifacts().get()) == set([`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.src",`
		`- "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",`
		`- "perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])`
		`+ if not devel:`
		`+ # Only x86_64 packages should be filled in, because we requested x86_64 arch.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.src",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])`
		`+ else:`
		`+ # The i686 packages are filtered out in normal packages, because multilib`
		`+ # is not enabled for them - therefore we want to include them in -devel.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])`

		`def test_fill_in_rpms_exclusivearch(self):`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",`
		`@@ -476,7 +492,8 @@`
		`assert set(mmd.get_rpm_artifacts().get()) == set([`
		`"perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch"])`

		`- def test_fill_in_rpms_rpm_whitelist(self):`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`+ def test_fill_in_rpms_rpm_whitelist(self, devel):`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",`
		`koji_srpm_name="python27-dhcp")`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",`
		`@@ -486,6 +503,7 @@`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",`
		`koji_srpm_name="foo-perl-Tangerine")`

		`+ self.cg.devel = devel`
		`mmd = self.cg.module.mmd()`
		`opts = mmd.get_buildopts()`
		`opts.set_rpm_whitelist(["python27-dhcp"])`
		`@@ -493,17 +511,25 @@`

		`mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")`

		`- # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted`
		`- # srpm name.`
		`- assert set(mmd.get_rpm_artifacts().get()) == set([`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ if not devel:`
		`+ # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted`
		`+ # srpm name.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ else:`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])`

		`- def test_fill_in_rpms_list_filters(self):`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`+ def test_fill_in_rpms_list_filters(self, devel):`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine")`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")`

		`+ self.cg.devel = devel`
		`mmd = self.cg.module.mmd()`
		`filter_list = Modulemd.SimpleSet()`
		`filter_list.add("dhcp-libs")`
		`@@ -511,11 +537,18 @@`

		`mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")`

		`- # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.`
		`- assert set(mmd.get_rpm_artifacts().get()) == set([`
		`- "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ if not devel:`
		`+ # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ else:`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])`

		`- def test_fill_in_rpms_list_multilib(self):`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`+ def test_fill_in_rpms_list_multilib(self, devel):`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",`
		`multilib=["x86_64"])`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",`
		`@@ -525,15 +558,20 @@`
		`self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",`
		`multilib=["ppc64le"])`

		`+ self.cg.devel = devel`
		`mmd = self.cg.module.mmd()`
		`mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")`

		`- # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have`
		`- # multilib set.`
		`- assert set(mmd.get_rpm_artifacts().get()) == set([`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",`
		`- "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ if not devel:`
		`+ # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have`
		`+ # multilib set.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])`
		`+ else:`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])`

		`@pytest.mark.parametrize(`
		`"licenses, expected", (`
		`@@ -556,17 +594,22 @@`
		`# Only x86_64 packages should be filled in, because we requested x86_64 arch.`
		`assert set(mmd.get_content_licenses().get()) == set(expected)`

		`- def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self):`
		`+ @pytest.mark.parametrize("devel", (False, True))`
		`+ def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self, devel):`
		`# A build has ExcludeArch: i686 (because it only works on 64 bit arches).`
		`# A noarch package is built there, and this noarch packages should be`
		`# included in x86_64 repo.`
		`self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",`
		`excludearch=["i686"])`

		`+ self.cg.devel = devel`
		`mmd = self.cg.module.mmd()`
		`mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")`

		`- # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have`
		`- # multilib set.`
		`- assert set(mmd.get_rpm_artifacts().get()) == set([`
		`- "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])`
		`+ if not devel:`
		`+ # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have`
		`+ # multilib set.`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([`
		`+ "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])`
		`+ else:`
		`+ assert set(mmd.get_rpm_artifacts().get()) == set([])`