From 20cf79f336c22440d46e0d103abac4fd16bbf15d Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Sep 04 2018 07:47:03 +0000 Subject: Attach architecture specific mmd files to content generator build, for now without arch-specific data. --- diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 1a24fbc..383f744 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -34,11 +34,12 @@ import subprocess import tempfile import time from io import open +import kobo.rpmlib from six import text_type import koji -from module_build_service import log, build_logs +from module_build_service import log, build_logs, Modulemd logging.basicConfig(level=logging.DEBUG) @@ -62,6 +63,12 @@ class KojiContentGenerator(object): self.module_name = module.name self.mmd = module.modulemd self.config = config + # List of architectures the module is built for. + self.arches = [] + # List of RPMs tagged in module.koji_tag as returned by Koji. + self.rpms = [] + # Dict constructed from `self.rpms` with NEVRA as a key. + self.rpms_dict = {} def __repr__(self): return "" % (self.module_name) @@ -259,40 +266,91 @@ class KojiContentGenerator(object): } return ret - def _get_output(self, output_path): - ret = [] - rpms = self._koji_rpms_in_tag(self.module.koji_tag) - components = [] - for rpm in rpms: - components.append( - { - u"name": rpm["name"], - u"version": rpm["version"], - u"release": rpm["release"], - u"arch": rpm["arch"], - u"epoch": rpm["epoch"], - u"sigmd5": rpm["payloadhash"], - u"type": u"rpm" + def _koji_rpm_to_component_record(self, rpm): + """ + Helper method returning CG "output" for RPM from the `rpm` dict. + + :param dict rpm: RPM dict as returned by Koji. + :rtype: dict + :return: CG "output" dict. + """ + return { + u"name": rpm["name"], + u"version": rpm["version"], + u"release": rpm["release"], + u"arch": rpm["arch"], + u"epoch": rpm["epoch"], + u"sigmd5": rpm["payloadhash"], + u"type": u"rpm" + } + + def _get_arch_mmd_output(self, output_path, arch): + """ + Returns the CG "output" dict for architecture specific modulemd file. + + :param str output_path: Path where the modulemd files are stored. + :param str arch: Architecture for which to generate the "output" dict. + :param dict rpms_dict: Dictionary with all RPMs built in this module. + The key is NEVRA string, value is RPM dict as obtained from Koji. + This dict is used to generate architecture specific "components" + section in the "output" record. + :rtype: dict + :return: Dictionary with record in "output" list. + """ + ret = { + 'buildroot_id': 1, + 'arch': arch, + 'type': 'file', + 'extra': { + 'typeinfo': { + 'module': {} } - ) + }, + 'checksum_type': 'md5', + } - ret.append( - { - u'buildroot_id': 1, - u'arch': u'noarch', - u'type': u'file', - u'extra': { - u'typeinfo': { - u'module': {} - } - }, - u'filesize': len(self.mmd), - u'checksum_type': u'md5', - u'checksum': text_type(hashlib.md5(self.mmd.encode('utf-8')).hexdigest()), - u'filename': u'modulemd.txt', - u'components': components - } - ) + # Noarch architecture represents "generic" modulemd.txt. + if arch == "noarch": + mmd_filename = "modulemd.txt" + else: + mmd_filename = "modulemd.%s.txt" % arch + + # Read the modulemd file to get the filesize/checksum and also + # parse it to get the Modulemd instance. + mmd_path = os.path.join(output_path, mmd_filename) + with open(mmd_path) as mmd_f: + data = mmd_f.read() + mmd = Modulemd.Module().new_from_string(data) + ret['filename'] = mmd_filename + ret['filesize'] = len(data) + ret['checksum'] = hashlib.md5(data.encode('utf-8')).hexdigest() + + components = [] + if arch == "noarch": + # For generic noarch modulemd, include all the RPMs. + for rpm in self.rpms: + components.append( + self._koji_rpm_to_component_record(rpm)) + else: + # Check the RPM artifacts built for this architecture in modulemd file, + # find the matchign RPM in the `rpms_dict` comming from Koji and use it + # to generate list of components for this architecture. + # We cannot simply use the data from MMD here without `rpms_dict`, because + # RPM sigmd5 signature is not stored in MMD. + for rpm in mmd.get_rpm_artifacts().get(): + if rpm not in self.rpms_dict: + raise RuntimeError("RPM %s found in the final modulemd but not " + "in Koji tag." % rpm) + tag_rpm = self.rpms_dict[rpm] + components.append( + self._koji_rpm_to_component_record(tag_rpm)) + ret["components"] = components + return ret + + def _get_output(self, output_path): + ret = [] + for arch in self.arches + ["noarch"]: + ret.append(self._get_arch_mmd_output(output_path, arch)) try: log_path = os.path.join(output_path, "build.log") @@ -326,6 +384,21 @@ class KojiContentGenerator(object): return ret + def _finalize_mmd(self, arch): + """ + Finalizes the modulemd: + - TODO: Fills in the list of built RPMs respecting filters, whitelist and multilib. + - TODO: Fills in the list of licences. + + :param str arch: Name of arch to generate the final modulemd for. + :rtype: str + :return: Finalized modulemd string. + """ + mmd = self.module.mmd() + # TODO: Fill in the list of built RPMs. + # TODO: Fill in the licences. + return unicode(mmd.dumps()) + def _prepare_file_directory(self): """ Creates a temporary directory that will contain all the files mentioned in the outputs section @@ -334,10 +407,17 @@ class KojiContentGenerator(object): """ prepdir = tempfile.mkdtemp(prefix="koji-cg-import") mmd_path = os.path.join(prepdir, "modulemd.txt") - log.info("Writing modulemd.yaml to %r" % mmd_path) + log.info("Writing generic modulemd.yaml to %r" % mmd_path) with open(mmd_path, "w") as mmd_f: mmd_f.write(self.mmd) + for arch in self.arches: + mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch) + log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path)) + mmd = self._finalize_mmd(arch) + with open(mmd_path, "w") as mmd_f: + mmd_f.write(mmd) + log_path = os.path.join(prepdir, "build.log") try: source = build_logs.path(self.module) @@ -408,12 +488,19 @@ class KojiContentGenerator(object): "Koji", nvr, tag) session.tagBuild(tag_info["id"], nvr) + def _load_koji_tag(self, koji_session): + tag = koji_session.getTag(self.module.koji_tag) + self.arches = tag["arches"].split(" ") if tag["arches"] else [] + self.rpms = self._koji_rpms_in_tag(self.module.koji_tag) + self.rpms_dict = {kobo.rpmlib.make_nvra(rpm, force_epoch=True): rpm for rpm in self.rpms} + def koji_import(self): """This method imports given module into the configured koji instance as a content generator based build Raises an exception when error is encountered during import""" session = get_session(self.config, self.owner) + self._load_koji_tag(session) file_dir = self._prepare_file_directory() metadata = self._get_content_generator_metadata(file_dir) diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 3953b46..c8e2bee 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -29,7 +29,7 @@ import module_build_service.messaging import module_build_service.scheduler.handlers.repos # noqa from module_build_service import models, conf, build_logs -from mock import patch, Mock, MagicMock, call +from mock import patch, Mock, MagicMock, call, mock_open from tests import init_data @@ -85,6 +85,7 @@ class TestBuild: """ Test generation of content generator json """ koji_session = MagicMock() koji_session.getUser.return_value = GET_USER_RV + koji_session.getTag.return_value = {"arches": ""} get_session.return_value = koji_session distro.return_value = ("Fedora", "25", "Twenty Five") machine.return_value = "i686" @@ -113,6 +114,7 @@ class TestBuild: build_logs.start(self.cg.module) build_logs.stop(self.cg.module) + self.cg._load_koji_tag(koji_session) file_dir = self.cg._prepare_file_directory() ret = self.cg._get_content_generator_metadata(file_dir) rpms_in_tag.assert_called_once() @@ -131,6 +133,7 @@ class TestBuild: """ Test generation of content generator json """ koji_session = MagicMock() koji_session.getUser.return_value = GET_USER_RV + koji_session.getTag.return_value = {"arches": ""} get_session.return_value = koji_session distro.return_value = ("Fedora", "25", "Twenty Five") machine.return_value = "i686" @@ -154,6 +157,7 @@ class TestBuild: "test_get_generator_json_expected_output.json") with open(expected_output_path) as expected_output_file: expected_output = json.load(expected_output_file) + self.cg._load_koji_tag(koji_session) file_dir = self.cg._prepare_file_directory() ret = self.cg._get_content_generator_metadata(file_dir) rpms_in_tag.assert_called_once() @@ -165,6 +169,19 @@ class TestBuild: with open(path.join(dir_path, "modulemd.txt")) as mmd: assert len(mmd.read()) == 1134 + def test_prepare_file_directory_per_arch_mmds(self): + """ Test preparation of directory with output files """ + self.cg.arches = ["x86_64", "i686"] + dir_path = self.cg._prepare_file_directory() + with open(path.join(dir_path, "modulemd.txt")) as mmd: + assert len(mmd.read()) == 1134 + + with open(path.join(dir_path, "modulemd.x86_64.txt")) as mmd: + assert len(mmd.read()) == 242 + + with open(path.join(dir_path, "modulemd.i686.txt")) as mmd: + assert len(mmd.read()) == 242 + @patch("module_build_service.builder.KojiContentGenerator.get_session") def test_tag_cg_build(self, get_session): """ Test that the CG build is tagged. """ @@ -217,3 +234,70 @@ class TestBuild: self.cg._tag_cg_build() koji_session.tagBuild.assert_not_called() + + @patch("module_build_service.builder.KojiContentGenerator.open", create=True) + def test_get_arch_mmd_output(self, patched_open): + patched_open.return_value = mock_open( + read_data=self.cg.mmd).return_value + ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64") + assert ret == { + 'arch': 'x86_64', + 'buildroot_id': 1, + 'checksum': 'bf1615b15f6a0fee485abe94af6b56b6', + 'checksum_type': 'md5', + 'components': [], + 'extra': {'typeinfo': {'module': {}}}, + 'filename': 'modulemd.x86_64.txt', + 'filesize': 1134, + 'type': 'file' + } + + @patch("module_build_service.builder.KojiContentGenerator.open", create=True) + def test_get_arch_mmd_output_components(self, patched_open): + mmd = self.cg.module.mmd() + rpm_artifacts = mmd.get_rpm_artifacts() + rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64") + mmd.set_rpm_artifacts(rpm_artifacts) + mmd_data = mmd.dumps() + + patched_open.return_value = mock_open( + read_data=mmd_data).return_value + + self.cg.rpms = [{ + "name": "dhcp", + "version": "4.3.5", + "release": "5.module_2118aef6", + "arch": "x86_64", + "epoch": "12", + "payloadhash": "hash", + }] + + self.cg.rpms_dict = { + "dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64": { + "name": "dhcp", + "version": "4.3.5", + "release": "5.module_2118aef6", + "arch": "x86_64", + "epoch": "12", + "payloadhash": "hash", + } + } + + ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64") + assert ret == { + 'arch': 'x86_64', + 'buildroot_id': 1, + 'checksum': '1bcc38b6f19285b3656b84a0443f46d2', + 'checksum_type': 'md5', + 'components': [{u'arch': 'x86_64', + u'epoch': '12', + u'name': 'dhcp', + u'release': '5.module_2118aef6', + u'sigmd5': 'hash', + u'type': u'rpm', + u'version': '4.3.5'}], + 'extra': {'typeinfo': {'module': {}}}, + 'filename': 'modulemd.x86_64.txt', + 'filesize': 315, + 'type': 'file' + }