From 8ac8167570da6d4aa7cf2ec347332f0c53c78a3e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 26 2017 14:45:51 +0000 Subject: PR#506: Track artifacts coming from koji itself Merges #506 https://pagure.io/koji/pull-request/506 --- diff --git a/docs/source/content_generator_metadata.rst b/docs/source/content_generator_metadata.rst index e878ef6..bf57324 100644 --- a/docs/source/content_generator_metadata.rst +++ b/docs/source/content_generator_metadata.rst @@ -108,6 +108,14 @@ Each map in the buildroots list contains the following entries: - checksum: The checksum of the file. - checksum\_type: The checksum type used. +- For maps where **type = kojifile**, the following fields will be present: + - filename: The name of the file. + - filesize: The size of the file. + - checksum: The checksum of the file. + - checksum\_type: The checksum type used. + - nvr: Build nvr from which this file origins. + - archive\_id: ID of archive from specified build. + - The format may be extended with other types in the future. - extra: A map containing information specific to the Content Generator that produced the files to import. For OSBS, the extra map should diff --git a/hub/kojihub.py b/hub/kojihub.py index 63f929e..c73a13f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5360,6 +5360,10 @@ class CG_Importer(object): match = self.match_file(comp) if match: files.append(match) + elif comp['type'] == 'kojifile': + match = self.match_kojifile(comp) + if match: + files.append(match) else: raise koji.GenericError("Unknown component type: %(type)s" % comp) return rpms, files @@ -5411,6 +5415,29 @@ class CG_Importer(object): return None #raise koji.GenericError("No match: %(filename)s (size %(filesize)s, sum %(checksum)s" % comp) + def match_kojifile(self, comp): + """Look up the file by archive id and sanity check the other data""" + assert(comp['type'] == 'kojifile') + archive = get_archive(comp['archive_id'], strict=True) + build = get_build(archive['build_id'], strict=True) + + for key in ['nvr', 'filename']: + if key not in comp: + raise koji.GenericError('%s field missing for component, ' + 'archive_id=%s' % (key, archive['id'])) + expected = { + 'nvr': build['nvr'], + 'filename': archive['filename'], + 'filesize': int(archive['size']), + 'checksum': archive['checksum'], + 'checksum_type': koji.CHECKSUM_TYPES[archive['checksum_type']], + } + for key in expected: + if key in comp and expected[key] != comp[key]: + raise koji.GenericError('Component field %s does not match for ' + 'archive_id=%s: %s != %s' % (key, archive['id'], + expected[key], comp[key])) + return archive def prep_outputs(self): metadata = self.metadata diff --git a/tests/test_hub/cg_importer_json/default.json b/tests/test_hub/cg_importer_json/default.json index e8a2094..03eb402 100644 --- a/tests/test_hub/cg_importer_json/default.json +++ b/tests/test_hub/cg_importer_json/default.json @@ -52,7 +52,15 @@ "filename": "jboss-eap-6.3.3-full-build.zip", "filesize": 12345678, "checksum": "5ec2f29c4e1c2e2aa6552836e236a158", - "checksum_type": "md5"}], + "checksum_type": "md5"}, + {"type": "kojifile", + "filename": "jboss-eap-6.4.10-1.win6.src.zip", + "filesize": 21670719, + "checksum": "215cf04db6bdbefb9644f001995bd550", + "checksum_type": "md5", + "archive_id": 1, + "nvr": "jboss-eap-6.4.10-1.win6"} + ], "extra": {"osbs": {"build_id": 12345, "builder_image_id": 67890}} }], diff --git a/tests/test_hub/test_cg_importer.py b/tests/test_hub/test_cg_importer.py index f77ce67..5207f6f 100644 --- a/tests/test_hub/test_cg_importer.py +++ b/tests/test_hub/test_cg_importer.py @@ -135,3 +135,62 @@ class TestCGImporter(unittest.TestCase): x = kojihub.CG_Importer() x.get_metadata('default.json', 'cg_importer_json') x.import_metadata() + + +class TestMatchKojiFile(unittest.TestCase): + + def setUp(self): + self.importer = kojihub.CG_Importer() + self.archive1 = { + 'id': 99, + 'build_id': 42, + 'checksum': 'e1f95555eae04b8e1ebdc5555c5555f0', + 'checksum_type': 0, + 'filename': 'foo-bar-3.0.jar', + 'size': 42710, + } + self.build1 = { + 'id': 79218, + 'nvr': 'foo-3.0-1', + } + self.comp1 = { + 'type': 'kojifile', + 'archive_id': self.archive1['id'], + 'nvr': self.build1['nvr'], + 'filename': self.archive1['filename'], + } + self.get_archive = mock.patch('kojihub.get_archive').start() + self.get_build = mock.patch('kojihub.get_build').start() + + def tearDown(self): + mock.patch.stopall() + + def test_match_kojifile_basic(self): + comp = self.comp1 + self.get_archive.return_value = self.archive1 + self.get_build.return_value = self.build1 + match = self.importer.match_kojifile(comp) + self.assertEqual(match, self.archive1) + self.get_build.assert_called_once() + + def test_match_kojifile_missing_fields(self): + self.get_archive.return_value = self.archive1 + self.get_build.return_value = self.build1 + # nvr missing + comp = self.comp1.copy() + del comp['nvr'] + with self.assertRaises(koji.GenericError): + self.importer.match_kojifile(comp) + # filename missing + comp = self.comp1.copy() + del comp['filename'] + with self.assertRaises(koji.GenericError): + self.importer.match_kojifile(comp) + + def test_match_kojifile_mismatch(self): + comp = self.comp1 + comp['filesize'] = 1 + self.get_archive.return_value = self.archive1 + self.get_build.return_value = self.build1 + with self.assertRaises(koji.GenericError): + self.importer.match_kojifile(comp)