#1062 Import -devel CG build with RPMs which are filtered out of the current real CG build.
Merged 9 months ago by mprahl. Opened 9 months ago by jkaluza.
jkaluza/fm-orchestrator cg-devel-build  into  master

@@ -74,6 +74,7 @@ 

          self.module_name = module.name

          self.mmd = module.modulemd

          self.config = config

+         self.devel = False

          # List of architectures the module is built for.

          self.arches = []

          # List of RPMs tagged in module.koji_tag as returned by Koji.

@@ -261,6 +262,8 @@ 

      def _get_build(self):

          ret = {}

          ret[u'name'] = self.module.name

+         if self.devel:

+             ret['name'] += "-devel"

          ret[u'version'] = self.module.stream.replace("-", "_")

          # Append the context to the version to make NVRs of modules unique in the event of

          # module stream expansion

@@ -276,7 +279,7 @@ 

                      u"module_build_service_id": self.module.id,

                      u"content_koji_tag": self.module.koji_tag,

                      u"modulemd_str": self.module.modulemd,

-                     u"name": self.module.name,

+                     u"name": ret['name'],

                      u"stream": self.module.stream,

                      u"version": self.module.version,

                      u"context": self.module.context

@@ -429,6 +432,68 @@ 

  

          return ret

  

+     def _should_include_rpm(self, rpm, mmd, arch, multilib_arches):

This code is just moved from the _fill_in_rpms_list below.

+         """

+         Helper method for `_fill_in_rpms_list` returning True if the RPM object

+         should be included in a final MMD file for given arch.

+         """

+         # Check the "whitelist" buildopts section of MMD.

+         # When "whitelist" is defined, it overrides component names in

+         # `mmd.get_rpm_components()`. The whitelist is used when module needs to build

+         # package with different SRPM name than the package name. This is case for example

+         # for software collections where SRPM name can be "httpd24-httpd", but package name

+         # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the

+         # rpm["srpm_name"] would be "httpd24-httpd".

+         srpm = rpm["srpm_name"]

+         whitelist = None

+         buildopts = mmd.get_buildopts()

+         if buildopts:

+             whitelist = buildopts.get_rpm_whitelist()

+             if whitelist:

+                 if srpm not in whitelist:

+                     # Package is not in the whitelist, skip it.

+                     return False

+ 

+         # If there is no whitelist, just check that the SRPM name we have here

+         # exists in the list of components.

+         # In theory, there should never be situation where modular tag contains

+         # some RPM built from SRPM not included in get_rpm_components() or in whitelist,

+         # but the original Pungi code checked for this case.

+         if not whitelist and srpm not in mmd.get_rpm_components().keys():

+             return False

+ 

+         # Do not include this RPM if it is filtered.

+         if rpm["name"] in mmd.get_rpm_filter().get():

+             return False

+ 

+         # Skip the rpm if it's built for multilib arch, but

+         # multilib is not enabled for this srpm in MMD.

+         try:

+             mmd_component = mmd.get_rpm_components()[srpm]

+             multilib = mmd_component.get_multilib()

+             multilib = multilib.get() if multilib else set()

+             # The `multilib` set defines the list of architectures for which

+             # the multilib is enabled.

+             #

+             # Filter out RPMs from multilib architectures if multilib is not

+             # enabled for current arch. Keep the RPMs from non-multilib compatible

+             # architectures.

+             if arch not in multilib and rpm["arch"] in multilib_arches:

+                 return False

+         except KeyError:

+             # TODO: This exception is raised only when "whitelist" is used.

+             # Since components in whitelist have different names than ones in

+             # components list, we won't find them there.

+             # We would need to track the RPMs srpm_name from whitelist back to

+             # original package name used in MMD's components list. This is possible

+             # but original Pungi code is not doing that. This is TODO for future

+             # improvements.

+ 

+             # No such component, disable any multilib

+             if rpm["arch"] not in ("noarch", arch):

+                 return False

+         return True

+ 

      def _fill_in_rpms_list(self, mmd, arch):

          """

          Fills in the list of built RPMs in architecture specific `mmd` for `arch`

@@ -473,8 +538,6 @@ 

                      rpm["arch"] not in [arch, "noarch", "src"]):

                  continue

  

-             srpm = rpm["srpm_name"]

- 

              # Skip the RPM if it is excluded on this arch or exclusive

              # for different arch.

              if rpm["excludearch"] and set(rpm["excludearch"]) & set(exclusive_arches):

@@ -482,61 +545,17 @@ 

              if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(exclusive_arches):

                  continue

  

-             # Check the "whitelist" buildopts section of MMD.

-             # When "whitelist" is defined, it overrides component names in

-             # `mmd.get_rpm_components()`. The whitelist is used when module needs to build

-             # package with different SRPM name than the package name. This is case for example

-             # for software collections where SRPM name can be "httpd24-httpd", but package name

-             # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the

-             # rpm["srpm_name"] would be "httpd24-httpd".

-             whitelist = None

-             buildopts = mmd.get_buildopts()

-             if buildopts:

-                 whitelist = buildopts.get_rpm_whitelist()

-                 if whitelist:

-                     if srpm not in whitelist:

-                         # Package is not in the whitelist, skip it.

-                         continue

- 

-             # If there is no whitelist, just check that the SRPM name we have here

-             # exists in the list of components.

-             # In theory, there should never be situation where modular tag contains

-             # some RPM built from SRPM not included in get_rpm_components() or in whitelist,

-             # but the original Pungi code checked for this case.

-             if not whitelist and srpm not in mmd.get_rpm_components().keys():

+             should_include = self._should_include_rpm(rpm, mmd, arch, multilib_arches)

+             if self.devel and should_include:

+                 # In case this is -devel module, we want to skip any RPMs which would be normally

+                 # include in a module and only keep those which wouldn't be included, because

include => included

+                 # -devel is complement to normal module build.

                  continue

- 

-             # Do not include this RPM if it is filtered.

-             if rpm["name"] in mmd.get_rpm_filter().get():

+             elif not self.devel and not should_include:

+                 # In chase this is normal (non-devel) module, include only packages which we

chase => case

+                 # really should include and skip the others.

                  continue

  

-             # Skip the rpm if it's built for multilib arch, but

-             # multilib is not enabled for this srpm in MMD.

-             try:

-                 mmd_component = mmd.get_rpm_components()[srpm]

-                 multilib = mmd_component.get_multilib()

-                 multilib = multilib.get() if multilib else set()

-                 # The `multilib` set defines the list of architectures for which

-                 # the multilib is enabled.

-                 #

-                 # Filter out RPMs from multilib architectures if multilib is not

-                 # enabled for current arch. Keep the RPMs from non-multilib compatible

-                 # architectures.

-                 if arch not in multilib and rpm["arch"] in multilib_arches:

-                     continue

-             except KeyError:

-                 # TODO: This exception is raised only when "whitelist" is used.

-                 # Since components in whitelist have different names than ones in

-                 # components list, we won't find them there.

-                 # We would need to track the RPMs srpm_name from whitelist back to

-                 # original package name used in MMD's components list. This is possible

-                 # but original Pungi code is not doing that. This is TODO for future

-                 # improvements.

- 

-                 # No such component, disable any multilib

-                 if rpm["arch"] not in ("noarch", arch):

-                     continue

- 

              # Add RPM to packages.

              rpm_artifacts.add(nevra)

  

@@ -558,6 +577,9 @@ 

          :return: Finalized modulemd string.

          """

          mmd = self.module.mmd()

+         if self.devel:

+             mmd.set_name(mmd.get_name() + "-devel")

+ 

          # Set the "Arch" field in mmd.

          mmd.set_arch(pungi.arch.tree_arch_to_yum_arch(arch))

          # Fill in the list of built RPMs.

@@ -655,16 +677,29 @@ 

          session.tagBuild(tag_info["id"], nvr)

  

      def _load_koji_tag(self, koji_session):

+         # Do not load Koji tag data if this method is called again. This would

+         # waste resources, because the Koji tag content is always the same

+         # for already built module.

+         if self.arches and self.rpms and self.rpms_dict:

+             return

+ 

          tag = koji_session.getTag(self.module.koji_tag)

          self.arches = tag["arches"].split(" ") if tag["arches"] else []

          self.rpms = self._koji_rpms_in_tag(self.module.koji_tag)

          self.rpms_dict = {kobo.rpmlib.make_nvra(rpm, force_epoch=True): rpm for rpm in self.rpms}

  

-     def koji_import(self):

+     def koji_import(self, devel=False):

          """This method imports given module into the configured koji instance as

          a content generator based build

  

-         Raises an exception when error is encountered during import"""

+         Raises an exception when error is encountered during import

+ 

+         :param bool devel: True if the "-devel" module should be created and imported.

+             The "-devel" module build contains only the RPMs which are normally filtered

+             from the module build. If set to False, normal module build respecting the

+             filters is created and imported.

+         """

+         self.devel = devel

          session = get_session(self.config, self.owner)

          self._load_koji_tag(session)

  

@@ -1176,6 +1176,7 @@ 

          if self.config.koji_enable_content_generator and self.module.state == 3:

              cg = KojiContentGenerator(self.module, self.config)

              cg.koji_import()

+             cg.koji_import(devel=True)

  

      @staticmethod

      def get_rpm_module_tag(rpm):

file modified
+73 -30

@@ -83,8 +83,9 @@ 

      @patch("platform.machine")

      @patch(("module_build_service.builder.KojiContentGenerator.KojiContentGenerator."

             "_koji_rpms_in_tag"))

+     @pytest.mark.parametrize("devel", (False, True))

      def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, coutput, popen,

-                                 get_session):

+                                 get_session, devel):

          """ Test generation of content generator json """

          koji_session = MagicMock()

          koji_session.getUser.return_value = GET_USER_RV

@@ -117,11 +118,17 @@ 

          build_logs.start(self.cg.module)

          build_logs.stop(self.cg.module)

  

+         self.cg.devel = devel

          self.cg._load_koji_tag(koji_session)

          file_dir = self.cg._prepare_file_directory()

          ret = self.cg._get_content_generator_metadata(file_dir)

          rpms_in_tag.assert_called_once()

-         assert expected_output == ret

+         if not devel:

+             assert expected_output == ret

+         else:

+             # For devel, only check that the name has -devel suffix.

+             assert ret["build"]["name"] == "nginx-devel"

+             assert ret["build"]["extra"]["typeinfo"]["module"]["name"] == "nginx-devel"

  

      @patch("module_build_service.builder.KojiContentGenerator.get_session")

      @patch("subprocess.Popen")

@@ -429,7 +436,8 @@ 

              self.cg.module.modulemd = mmd.dumps()

              self.cg.modulemd = mmd.dumps()

  

-     def test_fill_in_rpms_list(self):

+     @pytest.mark.parametrize("devel", (False, True))

+     def test_fill_in_rpms_list(self, devel):

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.s390x", "dhcp")

@@ -439,15 +447,23 @@ 

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.s390x", "perl-Tangerine")

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.src", "perl-Tangerine")

  

+         self.cg.devel = devel

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         # Only x86_64 packages should be filled in, because we requested x86_64 arch.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.src",

-             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",

-             "perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])

+         if not devel:

+             # Only x86_64 packages should be filled in, because we requested x86_64 arch.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.src",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.src"])

+         else:

+             # The i686 packages are filtered out in normal packages, because multilib

+             # is not enabled for them - therefore we want to include them in -devel.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])

  

      def test_fill_in_rpms_exclusivearch(self):

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",

@@ -476,7 +492,8 @@ 

          assert set(mmd.get_rpm_artifacts().get()) == set([

              "perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch"])

  

-     def test_fill_in_rpms_rpm_whitelist(self):

+     @pytest.mark.parametrize("devel", (False, True))

+     def test_fill_in_rpms_rpm_whitelist(self, devel):

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",

                             koji_srpm_name="python27-dhcp")

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",

@@ -486,6 +503,7 @@ 

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",

                             koji_srpm_name="foo-perl-Tangerine")

  

+         self.cg.devel = devel

          mmd = self.cg.module.mmd()

          opts = mmd.get_buildopts()

          opts.set_rpm_whitelist(["python27-dhcp"])

@@ -493,17 +511,25 @@ 

  

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted

-         # srpm name.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])

+         if not devel:

+             # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted

+             # srpm name.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])

+         else:

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])

  

-     def test_fill_in_rpms_list_filters(self):

+     @pytest.mark.parametrize("devel", (False, True))

+     def test_fill_in_rpms_list_filters(self, devel):

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine")

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")

  

+         self.cg.devel = devel

          mmd = self.cg.module.mmd()

          filter_list = Modulemd.SimpleSet()

          filter_list.add("dhcp-libs")

@@ -511,11 +537,18 @@ 

  

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

-             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+         if not devel:

+             # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+         else:

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])

  

-     def test_fill_in_rpms_list_multilib(self):

+     @pytest.mark.parametrize("devel", (False, True))

+     def test_fill_in_rpms_list_multilib(self, devel):

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",

                             multilib=["x86_64"])

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",

@@ -525,15 +558,20 @@ 

          self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",

                             multilib=["ppc64le"])

  

+         self.cg.devel = devel

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

-         # multilib set.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

-             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+         if not devel:

+             # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

+             # multilib set.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+         else:

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "perl-Tangerine-12:4.3.5-5.module_2118aef6.i686"])

  

      @pytest.mark.parametrize(

          "licenses, expected", (

@@ -556,17 +594,22 @@ 

          # Only x86_64 packages should be filled in, because we requested x86_64 arch.

          assert set(mmd.get_content_licenses().get()) == set(expected)

  

-     def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self):

+     @pytest.mark.parametrize("devel", (False, True))

+     def test_fill_in_rpms_list_noarch_filtering_not_influenced_by_multilib(self, devel):

          # A build has ExcludeArch: i686 (because it only works on 64 bit arches).

          # A noarch package is built there, and this noarch packages should be

          # included in x86_64 repo.

          self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",

                             excludearch=["i686"])

  

+         self.cg.devel = devel

          mmd = self.cg.module.mmd()

          mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

  

-         # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

-         # multilib set.

-         assert set(mmd.get_rpm_artifacts().get()) == set([

-             "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])

+         if not devel:

+             # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

+             # multilib set.

+             assert set(mmd.get_rpm_artifacts().get()) == set([

+                 "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])

+         else:

+             assert set(mmd.get_rpm_artifacts().get()) == set([])

For some modules, modularity-wg wants to be able to ship the "-devel" modules containing
the RPMs which are normally filtered out from the module.

This PR generates second Koji CG module with -devel suffix in a name with final modulemd
files containing the filtered out RPMs.

This code is just moved from the _fill_in_rpms_list below.

:thumbsup: Just a couple of typos to fix

I'll merge as is and fix the typos in a commit to master.

Commit 1c104d0 fixes this pull-request

Pull-Request has been merged by mprahl

9 months ago

Pull-Request has been merged by mprahl

9 months ago