#1006 Attach architecture specific mmd files to content generator build, for now without arch-specific data.
Merged 7 months ago by jkaluza. Opened 7 months ago by jkaluza.
jkaluza/fm-orchestrator cg-mmd-per-arch  into  cg-final-mmds

@@ -34,11 +34,12 @@ 

  import tempfile

  import time

  from io import open

+ import kobo.rpmlib

  

  from six import text_type

  import koji

  

- from module_build_service import log, build_logs

+ from module_build_service import log, build_logs, Modulemd

  

  logging.basicConfig(level=logging.DEBUG)

  

@@ -62,6 +63,12 @@ 

          self.module_name = module.name

          self.mmd = module.modulemd

          self.config = config

+         # List of architectures the module is built for.

+         self.arches = []

+         # List of RPMs tagged in module.koji_tag as returned by Koji.

+         self.rpms = []

+         # Dict constructed from `self.rpms` with NEVRA as a key.

+         self.rpms_dict = {}

  

      def __repr__(self):

          return "<KojiContentGenerator module: %s>" % (self.module_name)

@@ -259,40 +266,91 @@ 

          }

          return ret

  

-     def _get_output(self, output_path):

-         ret = []

-         rpms = self._koji_rpms_in_tag(self.module.koji_tag)

-         components = []

-         for rpm in rpms:

-             components.append(

-                 {

-                     u"name": rpm["name"],

-                     u"version": rpm["version"],

-                     u"release": rpm["release"],

-                     u"arch": rpm["arch"],

-                     u"epoch": rpm["epoch"],

-                     u"sigmd5": rpm["payloadhash"],

-                     u"type": u"rpm"

+     def _koji_rpm_to_component_record(self, rpm):

+         """

+         Helper method returning CG "output" for RPM from the `rpm` dict.

+ 

+         :param dict rpm: RPM dict as returned by Koji.

+         :rtype: dict

+         :return: CG "output" dict.

+         """

+         return {

+             u"name": rpm["name"],

+             u"version": rpm["version"],

+             u"release": rpm["release"],

+             u"arch": rpm["arch"],

+             u"epoch": rpm["epoch"],

+             u"sigmd5": rpm["payloadhash"],

+             u"type": u"rpm"

+         }

+ 

+     def _get_arch_mmd_output(self, output_path, arch):

+         """

+         Returns the CG "output" dict for architecture specific modulemd file.

+ 

+         :param str output_path: Path where the modulemd files are stored.

+         :param str arch: Architecture for which to generate the "output" dict.

+         :param dict rpms_dict: Dictionary with all RPMs built in this module.

+             The key is NEVRA string, value is RPM dict as obtained from Koji.

+             This dict is used to generate architecture specific "components"

+             section in the "output" record.

+         :rtype: dict

+         :return: Dictionary with record in "output" list.

+         """

+         ret = {

+             'buildroot_id': 1,

+             'arch': arch,

+             'type': 'file',

+             'extra': {

+                 'typeinfo': {

+                     'module': {}

                  }

-             )

+             },

+             'checksum_type': 'md5',

+         }

  

-         ret.append(

-             {

-                 u'buildroot_id': 1,

-                 u'arch': u'noarch',

-                 u'type': u'file',

-                 u'extra': {

-                     u'typeinfo': {

-                         u'module': {}

-                     }

-                 },

-                 u'filesize': len(self.mmd),

-                 u'checksum_type': u'md5',

-                 u'checksum': text_type(hashlib.md5(self.mmd.encode('utf-8')).hexdigest()),

-                 u'filename': u'modulemd.txt',

-                 u'components': components

-             }

-         )

+         # Noarch architecture represents "generic" modulemd.txt.

+         if arch == "noarch":

+             mmd_filename = "modulemd.txt"

+         else:

+             mmd_filename = "modulemd.%s.txt" % arch

+ 

+         # Read the modulemd file to get the filesize/checksum and also

+         # parse it to get the Modulemd instance.

+         mmd_path = os.path.join(output_path, mmd_filename)

+         with open(mmd_path) as mmd_f:

+             data = mmd_f.read()

+             mmd = Modulemd.Module().new_from_string(data)

+             ret['filename'] = mmd_filename

+             ret['filesize'] = len(data)

+             ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest()

+ 

+         components = []

+         if arch == "noarch":

+             # For generic noarch modulemd, include all the RPMs.

+             for rpm in self.rpms:

+                 components.append(

+                     self._koji_rpm_to_component_record(rpm))

+         else:

+             # Check the RPM artifacts built for this architecture in modulemd file,

+             # find the matchign RPM in the `rpms_dict` comming from Koji and use it

"matchign" => "matching" :)

"comming" => "coming"

+             # to generate list of components for this architecture.

+             # We cannot simply use the data from MMD here without `rpms_dict`, because

+             # RPM sigmd5 signature is not stored in MMD.

+             for rpm in mmd.get_rpm_artifacts().get():

+                 if rpm not in self.rpms_dict:

+                     raise RuntimeError("RPM %s found in the final modulemd but not "

+                                        "in Koji tag." % rpm)

+                 tag_rpm = self.rpms_dict[rpm]

+                 components.append(

+                     self._koji_rpm_to_component_record(tag_rpm))

+         ret["components"] = components

+         return ret

+ 

+     def _get_output(self, output_path):

+         ret = []

+         for arch in self.arches + ["noarch"]:

+             ret.append(self._get_arch_mmd_output(output_path, arch))

  

          try:

              log_path = os.path.join(output_path, "build.log")

@@ -326,6 +384,21 @@ 

  

          return ret

  

+     def _finalize_mmd(self, arch):

+         """

+         Finalizes the modulemd:

+             - TODO: Fills in the list of built RPMs respecting filters, whitelist and multilib.

+             - TODO: Fills in the list of licences.

should this be here?

Yes, as I said in the description, it's part of bigger feature developed in separate branch and not everything is done so far. So these TODOs here are valid.

+ 

+         :param str arch: Name of arch to generate the final modulemd for.

+         :rtype: str

+         :return: Finalized modulemd string.

+         """

+         mmd = self.module.mmd()

+         # TODO: Fill in the list of built RPMs.

+         # TODO: Fill in the licences.

same

+         return unicode(mmd.dumps())

+ 

      def _prepare_file_directory(self):

          """ Creates a temporary directory that will contain all the files

          mentioned in the outputs section

@@ -334,10 +407,17 @@ 

          """

          prepdir = tempfile.mkdtemp(prefix="koji-cg-import")

          mmd_path = os.path.join(prepdir, "modulemd.txt")

-         log.info("Writing modulemd.yaml to %r" % mmd_path)

+         log.info("Writing generic modulemd.yaml to %r" % mmd_path)

          with open(mmd_path, "w") as mmd_f:

              mmd_f.write(self.mmd)

  

+         for arch in self.arches:

+             mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch)

+             log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path))

+             mmd = self._finalize_mmd(arch)

+             with open(mmd_path, "w") as mmd_f:

+                 mmd_f.write(mmd)

+ 

          log_path = os.path.join(prepdir, "build.log")

          try:

              source = build_logs.path(self.module)

@@ -408,12 +488,19 @@ 

                   "Koji", nvr, tag)

          session.tagBuild(tag_info["id"], nvr)

  

+     def _load_koji_tag(self, koji_session):

+         tag = koji_session.getTag(self.module.koji_tag)

+         self.arches = tag["arches"].split(" ") if tag["arches"] else []

+         self.rpms = self._koji_rpms_in_tag(self.module.koji_tag)

+         self.rpms_dict = {kobo.rpmlib.make_nvra(rpm, force_epoch=True): rpm for rpm in self.rpms}

+ 

      def koji_import(self):

          """This method imports given module into the configured koji instance as

          a content generator based build

  

          Raises an exception when error is encountered during import"""

          session = get_session(self.config, self.owner)

+         self._load_koji_tag(session)

  

          file_dir = self._prepare_file_directory()

          metadata = self._get_content_generator_metadata(file_dir)

@@ -29,7 +29,7 @@ 

  import module_build_service.scheduler.handlers.repos # noqa

  from module_build_service import models, conf, build_logs

  

- from mock import patch, Mock, MagicMock, call

+ from mock import patch, Mock, MagicMock, call, mock_open

  

  from tests import init_data

  

@@ -85,6 +85,7 @@ 

          """ Test generation of content generator json """

          koji_session = MagicMock()

          koji_session.getUser.return_value = GET_USER_RV

+         koji_session.getTag.return_value = {"arches": ""}

          get_session.return_value = koji_session

          distro.return_value = ("Fedora", "25", "Twenty Five")

          machine.return_value = "i686"

@@ -113,6 +114,7 @@ 

          build_logs.start(self.cg.module)

          build_logs.stop(self.cg.module)

  

+         self.cg._load_koji_tag(koji_session)

          file_dir = self.cg._prepare_file_directory()

          ret = self.cg._get_content_generator_metadata(file_dir)

          rpms_in_tag.assert_called_once()

@@ -131,6 +133,7 @@ 

          """ Test generation of content generator json """

          koji_session = MagicMock()

          koji_session.getUser.return_value = GET_USER_RV

+         koji_session.getTag.return_value = {"arches": ""}

          get_session.return_value = koji_session

          distro.return_value = ("Fedora", "25", "Twenty Five")

          machine.return_value = "i686"

@@ -154,6 +157,7 @@ 

                                           "test_get_generator_json_expected_output.json")

          with open(expected_output_path) as expected_output_file:

              expected_output = json.load(expected_output_file)

+         self.cg._load_koji_tag(koji_session)

          file_dir = self.cg._prepare_file_directory()

          ret = self.cg._get_content_generator_metadata(file_dir)

          rpms_in_tag.assert_called_once()

@@ -165,6 +169,19 @@ 

          with open(path.join(dir_path, "modulemd.txt")) as mmd:

              assert len(mmd.read()) == 1134

  

+     def test_prepare_file_directory_per_arch_mmds(self):

+         """ Test preparation of directory with output files """

+         self.cg.arches = ["x86_64", "i686"]

+         dir_path = self.cg._prepare_file_directory()

+         with open(path.join(dir_path, "modulemd.txt")) as mmd:

+             assert len(mmd.read()) == 1134

+ 

+         with open(path.join(dir_path, "modulemd.x86_64.txt")) as mmd:

+             assert len(mmd.read()) == 242

+ 

+         with open(path.join(dir_path, "modulemd.i686.txt")) as mmd:

+             assert len(mmd.read()) == 242

+ 

      @patch("module_build_service.builder.KojiContentGenerator.get_session")

      def test_tag_cg_build(self, get_session):

          """ Test that the CG build is tagged. """

@@ -217,3 +234,70 @@ 

          self.cg._tag_cg_build()

  

          koji_session.tagBuild.assert_not_called()

+ 

+     @patch("module_build_service.builder.KojiContentGenerator.open", create=True)

+     def test_get_arch_mmd_output(self, patched_open):

+         patched_open.return_value = mock_open(

+             read_data=self.cg.mmd).return_value

+         ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64")

+         assert ret == {

+             'arch': 'x86_64',

+             'buildroot_id': 1,

+             'checksum': 'bf1615b15f6a0fee485abe94af6b56b6',

+             'checksum_type': 'md5',

+             'components': [],

+             'extra': {'typeinfo': {'module': {}}},

+             'filename': 'modulemd.x86_64.txt',

+             'filesize': 1134,

+             'type': 'file'

+         }

+ 

+     @patch("module_build_service.builder.KojiContentGenerator.open", create=True)

+     def test_get_arch_mmd_output_components(self, patched_open):

+         mmd = self.cg.module.mmd()

+         rpm_artifacts = mmd.get_rpm_artifacts()

+         rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")

+         mmd.set_rpm_artifacts(rpm_artifacts)

+         mmd_data = mmd.dumps()

+ 

+         patched_open.return_value = mock_open(

+             read_data=mmd_data).return_value

+ 

+         self.cg.rpms = [{

+             "name": "dhcp",

+             "version": "4.3.5",

+             "release": "5.module_2118aef6",

+             "arch": "x86_64",

+             "epoch": "12",

+             "payloadhash": "hash",

+         }]

+ 

+         self.cg.rpms_dict = {

+             "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64": {

+                 "name": "dhcp",

+                 "version": "4.3.5",

+                 "release": "5.module_2118aef6",

+                 "arch": "x86_64",

+                 "epoch": "12",

+                 "payloadhash": "hash",

+             }

+         }

+ 

+         ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64")

+         assert ret == {

+             'arch': 'x86_64',

+             'buildroot_id': 1,

+             'checksum': '1bcc38b6f19285b3656b84a0443f46d2',

+             'checksum_type': 'md5',

+             'components': [{u'arch': 'x86_64',

+                             u'epoch': '12',

+                             u'name': 'dhcp',

+                             u'release': '5.module_2118aef6',

+                             u'sigmd5': 'hash',

+                             u'type': u'rpm',

+                             u'version': '4.3.5'}],

+             'extra': {'typeinfo': {'module': {}}},

+             'filename': 'modulemd.x86_64.txt',

+             'filesize': 315,

+             'type': 'file'

+         }

This PR is against cg-final-mmds branch and it is just part of the end goal which is generating the final modulemd files in the content generator build. It is therefore known to not be complete, but I decided to code this bigger feature in smaller chunks to make reviews easier.

This PR changes the KojiContentGenerator to upload architecture specific modulemd files. Current old code only uploads the modulemd.txt file which is noarch and contains all the components from the Koji tag. With this PR, the modulemd.txt is still uploaded as before, but there are also architecture specific modulemd.$arch.txt files uploaded which will in the future contain architecture specific data (see TODO in the code).

The components sections in metadata section for these files contains only RPM packages associated with this particular architecture of the uploaded modulemd file.

Yes, as I said in the description, it's part of bigger feature developed in separate branch and not everything is done so far. So these TODOs here are valid.

Pull-Request has been merged by jkaluza

7 months ago