#1007 Add list of built RPMs to architecture specific modulemd files in CG builds.
Merged 7 months ago by jkaluza. Opened 7 months ago by jkaluza.
jkaluza/fm-orchestrator cg-fill-rpms  into  cg-final-mmds

file modified
+2

@@ -30,9 +30,11 @@ 

          python-sqlalchemy \

          python-virtualenv \

          python-futures \

+         python2-pungi \

          python3 \

          python3-devel \

          python3-docutils \

+         python3-pungi \

          redhat-rpm-config \

          redhat-rpm-config \

          rpm-build \

file modified
+1

@@ -35,6 +35,7 @@ 

      python-six \

      python-solv \

      python-sqlalchemy \

+     python2-pungi \

      # Test-only dependencies

      python-flake8 \

      python-mock \

@@ -38,6 +38,7 @@ 

  

  from six import text_type

  import koji

+ import pungi.arch

  

  from module_build_service import log, build_logs, Modulemd

  

@@ -49,6 +50,16 @@ 

      return KojiModuleBuilder.get_session(config, owner)

  

  

+ def koji_retrying_multicall_map(*args, **kwargs):

+     """

+     Wrapper around KojiModuleBuilder.koji_retrying_multicall_map, because

+     we cannot import that method normally because of import loop.

+     """

+     from module_build_service.builder.KojiModuleBuilder import \

+         koji_retrying_multicall_map as multicall

+     return multicall(*args, **kwargs)

+ 

+ 

  class KojiContentGenerator(object):

      """ Class for handling content generator imports of module builds into Koji """

  

@@ -204,12 +215,36 @@ 

              # If the tag doesn't exist.. then there are no rpms in that tag.

              return []

  

-         # Extract some srpm-level info from the build attach it to each rpm

+         # Get the exclusivearch and excludearch lists for each RPM.

+         # The exclusivearch and excludearch lists are set in source RPM from which the RPM

+         # was built.

+         # Create temporary dict with source RPMs in rpm_id:build_id format.

+         src_rpms_ids = {rpm["id"]: rpm["build_id"] for rpm in rpms if rpm["arch"] == "src"}

+         # Prepare the arguments for Koji multicall.

+         # We will call session.getRPMHeaders(...) for each SRC RPM to get exclusivearch and

+         # excludearch headers.

+         multicall_kwargs = [{"rpmID": rpm_id, "headers": ["exclusivearch", "excludearch"]}

+                             for rpm_id in src_rpms_ids.keys()]

+         src_rpms_headers = koji_retrying_multicall_map(

+             session, session.getRPMHeaders, list_of_kwargs=multicall_kwargs)

+ 

+         # Temporary dict with build_id as a key to find builds easily.

          builds = {build['build_id']: build for build in builds}

+ 

+         # Handle the multicall result. For each build associated with the source RPM,

+         # store the exclusivearch and excludearch lists.

+         for build_id, headers in zip(src_rpms_ids.values(), src_rpms_headers):

+             builds[build_id]["exclusivearch"] = headers["exclusivearch"]

+             builds[build_id]["excludearch"] = headers["excludearch"]

+ 

+         # Check each RPM and fill-in additional data from its build to get them

+         # easily in other methods.

          for rpm in rpms:

              idx = rpm['build_id']

              rpm['srpm_name'] = builds[idx]['name']

              rpm['srpm_nevra'] = builds[idx]['nvr']

+             rpm['exclusivearch'] = builds[idx]['exclusivearch']

+             rpm['excludearch'] = builds[idx]['excludearch']

  

          return rpms

  

@@ -333,7 +368,7 @@ 

                      self._koji_rpm_to_component_record(rpm))

          else:

              # Check the RPM artifacts built for this architecture in modulemd file,

-             # find the matchign RPM in the `rpms_dict` comming from Koji and use it

+             # find the matching RPM in the `rpms_dict` coming from Koji and use it

              # to generate list of components for this architecture.

              # We cannot simply use the data from MMD here without `rpms_dict`, because

              # RPM sigmd5 signature is not stored in MMD.

@@ -384,10 +419,109 @@ 

  

          return ret

  

+     def _fill_in_rpms_list(self, mmd, arch):

+         """

+         Fills in the list of built RPMs in architecture specific `mmd` for `arch`

+         using the data from `self.rpms_dict`.

+ 

+         :param Modulemd.Module mmd: MMD to add built RPMs to.

+         :param str arch: Architecture for which to add RPMs.

+         :rtype: Modulemd.Module

+         :return: MMD with built RPMs filled in.

+         """

+         # List of all architectures compatible with input architecture including

+         # the multilib architectures.

+         # Example input/output:

+         #   "x86_64" -> ['x86_64', 'athlon', 'i686', 'i586', 'i486', 'i386', 'noarch']

+         #   "i686" -> ['i686', 'i586', 'i486', 'i386', 'noarch']

+         compatible_arches = pungi.arch.get_compatible_arches(arch, multilib=True)

+         # List of only multilib architectures.

+         # For example:

+         #   "x86_64" -> ['athlon', 'i386', 'i586', 'i486', 'i686']

+         #   "i686" -> []

+         multilib_arches = set(compatible_arches) - set(

+             pungi.arch.get_compatible_arches(arch))

+ 

+         # Modulemd.SimpleSet into which we will add the RPMs.

+         rpm_artifacts = Modulemd.SimpleSet()

+ 

+         # Check each RPM in `self.rpms_dict` to find out if it can be included in mmd

+         # for this architecture.

+         for nevra, rpm in self.rpms_dict.items():

+             srpm = rpm["srpm_name"]

+ 

+             # Skip the RPM if it is excluded on this arch or exclusive

+             # for different arch.

+             if rpm["excludearch"] and set(rpm["excludearch"]) & set(compatible_arches):

+                 continue

+             if rpm["exclusivearch"] and not set(rpm["exclusivearch"]) & set(compatible_arches):

+                 continue

+ 

+             # Check the "whitelist" buildopts section of MMD.

+             # When "whitelist" is defined, it overrides component names in

+             # `mmd.get_rpm_components()`. The whitelist is used when module needs to build

+             # package with different SRPM name than the package name. This is case for example

+             # for software collections where SRPM name can be "httpd24-httpd", but package name

+             # is still "httpd". In this case, get_rpm_components() would contain "httpd", but the

+             # rpm["srpm_name"] would be "httpd24-httpd".

+             whitelist = None

+             buildopts = mmd.get_buildopts()

+             if buildopts:

+                 whitelist = buildopts.get_rpm_whitelist()

+                 if whitelist:

+                     if srpm not in whitelist:

+                         # Package is not in the whitelist, skip it.

+                         continue

+ 

+             # If there is no whitelist, just check that the SRPM name we have here

+             # exists in the list of components.

+             # In theory, there should never be situation where modular tag contains

+             # some RPM built from SRPM not included in get_rpm_components() or in whitelist,

+             # but the original Pungi code checked for this case.

+             if not whitelist and srpm not in mmd.get_rpm_components().keys():

+                 continue

+ 

+             # Do not include this RPM if it is filtered.

+             if rpm["name"] in mmd.get_rpm_filter().get():

+                 continue

+ 

+             # Skip the rpm if it's built for multilib arch, but

+             # multilib is not enabled for this srpm in MMD.

+             try:

+                 mmd_component = mmd.get_rpm_components()[srpm]

+                 multilib = mmd_component.get_multilib()

+                 multilib = multilib.get() if multilib else set()

+                 # The `multilib` set defines the list of architectures for which

+                 # the multilib is enabled.

+                 #

+                 # Filter out RPMs from multilib architectures if multilib is not

+                 # enabled for current arch. Keep the RPMs from non-multilib compatible

+                 # architectures.

+                 if arch not in multilib and rpm["arch"] in multilib_arches:

+                     continue

+             except KeyError:

+                 # TODO: This exception is raised only when "whitelist" is used.

+                 # Since components in whitelist have different names than ones in

+                 # components list, we won't find them there.

+                 # We would need to track the RPMs srpm_name from whitelist back to

+                 # original package name used in MMD's components list. This is possible

+                 # but original Pungi code is not doing that. This is TODO for future

+                 # improvements.

+ 

+                 # No such component, disable any multilib

+                 if rpm["arch"] not in ("noarch", arch):

+                     continue

+ 

+             # Add RPM to packages.

+             rpm_artifacts.add(nevra)

+ 

+         mmd.set_rpm_artifacts(rpm_artifacts)

+         return mmd

+ 

      def _finalize_mmd(self, arch):

          """

          Finalizes the modulemd:

-             - TODO: Fills in the list of built RPMs respecting filters, whitelist and multilib.

+             - Fills in the list of built RPMs respecting filters, whitelist and multilib.

              - TODO: Fills in the list of licences.

  

          :param str arch: Name of arch to generate the final modulemd for.

@@ -395,7 +529,9 @@ 

          :return: Finalized modulemd string.

          """

          mmd = self.module.mmd()

-         # TODO: Fill in the list of built RPMs.

+         # Fill in the list of built RPMs.

+         mmd = self._fill_in_rpms_list(mmd, arch)

+ 

          # TODO: Fill in the licences.

          return unicode(mmd.dumps())

  

file modified
+219 -1

@@ -27,9 +27,10 @@ 

  

  import module_build_service.messaging

  import module_build_service.scheduler.handlers.repos # noqa

- from module_build_service import models, conf, build_logs

+ from module_build_service import models, conf, build_logs, Modulemd

  

  from mock import patch, Mock, MagicMock, call, mock_open

+ import kobo.rpmlib

  

  from tests import init_data

  

@@ -301,3 +302,220 @@ 

              'filesize': 315,

              'type': 'file'

          }

+ 

+     @patch("module_build_service.builder.KojiContentGenerator.get_session")

+     def test_koji_rpms_in_tag(self, get_session):

+         koji_session = MagicMock()

+         koji_session.getUser.return_value = GET_USER_RV

+         koji_session.getTag.return_value = {"arches": "x86_64"}

+ 

+         rpms = [

+             {

+                 'id': 1,

+                 'arch': 'src',

+                 'build_id': 875991,

+                 'name': 'module-build-macros',

+                 'release': '1.module_92011fe6',

+                 'version': '0.1'

+             },

+             {

+                 'id': 2,

+                 'arch': 'noarch',

+                 'build_id': 875991,

+                 'name': 'module-build-macros',

+                 'release': '1.module_92011fe6',

+                 'version': '0.1'

+             },

+             {

+                 'id': 3,

+                 'arch': 'src',

+                 'build_id': 875636,

+                 'name': 'ed',

+                 'release': '2.module_bd6e0eb1',

+                 'version': '1.14.1'

+             },

+             {

+                 'id': 4,

+                 'arch': 'x86_64',

+                 'build_id': 875636,

+                 'name': 'ed',

+                 'release': '2.module_bd6e0eb1',

+                 'version': '1.14.1'

+             },

+         ]

+ 

+         builds = [

+             {

+                 'build_id': 875636,

+                 'name': 'ed',

+                 'release': '2.module_bd6e0eb1',

+                 'version': '1.14.1',

+                 'nvr': 'ed-2.module_bd6e0eb1-1.14.1',

+             },

+             {

+                 'build_id': 875991,

+                 'name': 'module-build-macros',

+                 'release': '1.module_92011fe6',

+                 'version': '0.1',

+                 'nvr': 'module-build-macros-0.1-1.module_92011fe6',

+             }

+         ]

+ 

+         koji_session.listTaggedRPMS.return_value = (rpms, builds)

+         koji_session.multiCall.side_effect = [

+             # getRPMHeaders response

+             [[{'excludearch': ["x86_64"], 'exclusivearch': []}],

+              [{'excludearch': [], 'exclusivearch': ["x86_64"]}]]

+         ]

+         get_session.return_value = koji_session

+ 

+         rpms = self.cg._koji_rpms_in_tag("tag")

+         for rpm in rpms:

+             # We want to mainly check the excludearch and exclusivearch code.

+             if rpm["name"] == "module-build-macros":

+                 assert rpm["excludearch"] == ["x86_64"]

+             else:

+                 assert rpm["exclusivearch"] == ["x86_64"]

+ 

+     def _add_test_rpm(self, nevra, srpm_name=None, multilib=None,

+                       koji_srpm_name=None, excludearch=None, exclusivearch=None):

+         """

+         Helper method to add test RPM to ModuleBuild used by KojiContentGenerator

+         and also to Koji tag used to generate the Content Generator build.

+ 

+         :param str nevra: NEVRA of the RPM to add.

+         :param str srpm_name: Name of SRPM the added RPM is built from.

+         :param list multilib: List of architecture for which the multilib should be turned on.

+         :param str koji_srpm_name: If set, overrides the `srpm_name` in Koji tag. This is

+             needed to test the case when the built "src" package has different name than

+             the package in Koji. This is for example case of software collections where

+             `srpm_name` is "httpd" but `koji_srpm_name` would be "httpd24-httpd".

+         :param list excludearch: List of architectures this package is excluded from.

+         :param list exclusivearch: List of architectures this package is exclusive for.

+         """

+         parsed_nevra = kobo.rpmlib.parse_nvra(nevra)

+         parsed_nevra["payloadhash"] = "hash"

+         if koji_srpm_name:

+             parsed_nevra["srpm_name"] = koji_srpm_name

+         else:

+             parsed_nevra["srpm_name"] = srpm_name

+         parsed_nevra["excludearch"] = excludearch or []

+         parsed_nevra["exclusivearch"] = exclusivearch or []

+         self.cg.rpms.append(parsed_nevra)

+         self.cg.rpms_dict[nevra] = parsed_nevra

+ 

+         mmd = self.cg.module.mmd()

+         if srpm_name not in mmd.get_rpm_components().keys():

+             component = Modulemd.ComponentRpm()

+             component.set_name(srpm_name)

+             component.set_rationale("foo")

+ 

+             if multilib:

+                 multilib_set = Modulemd.SimpleSet()

+                 for arch in multilib:

+                     multilib_set.add(arch)

+                 component.set_multilib(multilib_set)

+ 

+             mmd.add_rpm_component(component)

+             self.cg.module.modulemd = mmd.dumps()

+             self.cg.modulemd = mmd.dumps()

+ 

+     def test_fill_in_rpms_list(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")

+ 

+         mmd = self.cg.module.mmd()

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only x86_64 packages should be filled in, because we requested x86_64 arch.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

+             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+ 

+     def test_fill_in_rpms_exclusivearch(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",

+                            exclusivearch=["x86_64"])

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch", "perl-Tangerine",

+                            exclusivearch=["ppc64le"])

+ 

+         mmd = self.cg.module.mmd()

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only dhcp-libs should be filled in, because perl-Tangerine has different

+         # exclusivearch.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.noarch"])

+ 

+     def test_fill_in_rpms_excludearch(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.noarch", "dhcp",

+                            excludearch=["x86_64"])

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch", "perl-Tangerine",

+                            excludearch=["ppc64le"])

+ 

+         mmd = self.cg.module.mmd()

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only perl-Tangerine should be filled in, because dhcp-libs is excluded from x86_64.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "perl-Tangerine-12:4.3.5-5.module_2118aef6.noarch"])

+ 

+     def test_fill_in_rpms_rpm_whitelist(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",

+                            koji_srpm_name="python27-dhcp")

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",

+                            koji_srpm_name="python27-dhcp")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine",

+                            koji_srpm_name="foo-perl-Tangerine")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",

+                            koji_srpm_name="foo-perl-Tangerine")

+ 

+         mmd = self.cg.module.mmd()

+         opts = mmd.get_buildopts()

+         opts.set_rpm_whitelist(["python27-dhcp"])

+         mmd.set_buildopts(opts)

+ 

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only x86_64 dhcp-libs should be filled in, because only python27-dhcp is whitelisted

+         # srpm name.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64"])

+ 

+     def test_fill_in_rpms_list_filters(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp")

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine")

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine")

+ 

+         mmd = self.cg.module.mmd()

+         filter_list = Modulemd.SimpleSet()

+         filter_list.add("dhcp-libs")

+         mmd.set_rpm_filter(filter_list)

+ 

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only x86_64 perl-Tangerine should be filled in, because dhcp-libs is filtered out.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

+ 

+     def test_fill_in_rpms_list_multilib(self):

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64", "dhcp",

+                            multilib=["x86_64"])

+         self._add_test_rpm("dhcp-libs-12:4.3.5-5.module_2118aef6.i686", "dhcp",

+                            multilib=["x86_64"])

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64", "perl-Tangerine",

+                            multilib=["ppc64le"])

+         self._add_test_rpm("perl-Tangerine-12:4.3.5-5.module_2118aef6.i686", "perl-Tangerine",

+                            multilib=["ppc64le"])

+ 

+         mmd = self.cg.module.mmd()

+         mmd = self.cg._fill_in_rpms_list(mmd, "x86_64")

+ 

+         # Only i686 package for dhcp-libs should be added, because perl-Tangerine does not have

+         # multilib set.

+         assert set(mmd.get_rpm_artifacts().get()) == set([

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64",

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.i686",

+             "perl-Tangerine-12:4.3.5-5.module_2118aef6.x86_64"])

This PR is against cg-final-mmds branch and it is just part of the end goal which is generating the final modulemd files in the content generator build. It is therefore known to not be complete, but I decided to code this bigger feature in smaller chunks to make reviews easier.

This PR ports the Pungi code to fill in the list of RPMs into the architecture specific MMD files. It respects exclusivearch/excludearch RPM headers, multilib, whitelists and filters. It works the very same way as the current Pungi code.

It adds new dependency on pungi.arch. The pungi.arch is not on PyPi and is expected to be in host site_packages. I'm not sure how good this is - we can always bundle the arch.py and arch_utils.py from Pungi in MBS, but I think it's better to have it on single place.

In comparison to Pungi code, it also has different way how to get exclusivearch and excludearch headers. Pungi can just read them from /mnt/redhat, but in MBS, we need to ask Koji to read them for us and return it back. This is done using getRPMHeaders Koji API call using multicall.

Would the python-pungi package work? We already don't use PyPi for libmodulemd, so adding this as an RPM dep in Vagrant and the Dockerfile would likely work.

Optional: Moving this definition down to right before it is used (line 46 of the diff) would make this easier to read.

Nice! +1 once the Pungi dependency is figured out in Vagrant and the tests

@mprahl, hm, there's an issue with "pungi" on RHEL7. It is not in EPEL7 in up to date version - there is previous major version only. We might try to get it into epel7-infra - updating in epel7 is not possible because the new version is not compatible with the old one.

@mprahl, I've asked Patrick and Lubomir Sedlar if it's possible to include pungi-4.x in epel7-infra and they don't see any reason why not, so we are OK. Lubomir is currently working on epel7 build and I will tag it there once it is done.

rebased onto dd07c6f

7 months ago

Pull-Request has been merged by jkaluza

7 months ago