From 35b3e51fcc85f9c7c55a95fddfe083612c0cbd71 Mon Sep 17 00:00:00 2001 From: Jana Cupova Date: Feb 06 2023 12:17:33 +0000 Subject: Use strict for rpm without signed copies or checksums + small review fixes --- diff --git a/kojihub/kojihub.py b/kojihub/kojihub.py index 6e9b3f9..c6e9b25 100644 --- a/kojihub/kojihub.py +++ b/kojihub/kojihub.py @@ -8030,8 +8030,8 @@ def write_signed_rpm(an_rpm, sigkey, force=False, checksum_types=None): if os.path.exists(signedpath): if not force: # already present - chsum_list = calculate_chsum(signedpath, checksum_types) - create_rpm_checksum(rpm_id=rpm_id, sigkey=sigkey, chsum_list=chsum_list) + chsum_dict = calculate_chsum(signedpath, checksum_types) + create_rpm_checksum(rpm_id, sigkey, chsum_dict) return else: os.unlink(signedpath) @@ -8041,7 +8041,7 @@ def write_signed_rpm(an_rpm, sigkey, force=False, checksum_types=None): koji.ensuredir(os.path.dirname(signedpath)) msum = MultiSum(checksum_types) koji.splice_rpm_sighdr(sighdr, rpm_path, dst=signedpath, callback=msum.update) - create_rpm_checksum(rpm_id=rpm_id, sigkey=sigkey, chsum_list=msum.checksums) + create_rpm_checksum(rpm_id, sigkey, msum.checksums) def query_history(tables=None, **kwargs): @@ -12315,6 +12315,71 @@ class RootExports(object): queryRPMSigs = staticmethod(query_rpm_sigs) + def getRPMChecksums(self, rpm_id, checksum_types=None, cacheonly=False, strict=False): + """Returns RPM checksums for specific rpm. + + :param int rpm_id: RPM id + :param list checksum_type: List of checksum types. Default sha256 checksum type + :param bool cacheonly: when False, checksum is created for missing checksum type + when True, checksum is returned as None when checsum is missing + for specific checksum type + :returns: A dict of specific checksum types and checksums + """ + if not isinstance(rpm_id, int): + raise koji.GenericError('rpm_id must be an integer') + rpm_info = get_rpm(rpm_id, strict=True) + if not checksum_types: + checksum_types = context.opts.get('RPMDefaultChecksums').split() + if not isinstance(checksum_types, list): + raise koji.GenericError('checksum_type must be a list') + + for ch_type in checksum_types: + if ch_type not in koji.CHECKSUM_TYPES: + raise koji.GenericError(f"Checksum_type {ch_type} isn't supported") + + query = QueryProcessor(tables=['rpmsigs'], columns=['sigkey'], + clauses=['rpm_id=%(rpm_id)i'], values={'rpm_id': rpm_id}) + sigkeys = [r['sigkey'] for r in query.execute()] + list_checksums_sigkeys = {s: set(checksum_types) for s in sigkeys} + + checksum_type_int = [koji.CHECKSUM_TYPES[chsum] for chsum in checksum_types] + query_checksum = QueryProcessor(tables=['rpm_checksum'], + columns=['checksum', 'checksum_type', 'sigkey'], + clauses={'rpm_id=%(rpm_id)i', + 'checksum_type IN %(checksum_type)s'}, + values={'rpm_id': rpm_id, + 'checksum_type': checksum_type_int}) + query_result = query_checksum.execute() + if not query_result or not sigkeys: + if strict: + nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info + raise koji.GenericError( + f"Rpm {nvra} doesn't have cached checksums or signed copies.") + else: + return {} + query_result = query_checksum.execute() + if len(query_result) == (len(checksum_type_int) * len(sigkeys)) or cacheonly: + return create_rpm_checksums_output(query_result, list_checksums_sigkeys) + else: + missing_chsum_sigkeys = list_checksums_sigkeys.copy() + for r in query_result: + if r['checksum_type'] in checksum_type_int and r['sigkey'] in sigkeys: + missing_chsum_sigkeys[r['sigkey']].remove( + koji.CHECKSUM_TYPES[r['checksum_type']]) + + for sigkey, chsums in missing_chsum_sigkeys.items(): + builddir = koji.pathinfo.build(rpm_info) + rpm_path = os.path.join(builddir, koji.pathinfo.rpm(rpm_info)) + sig_path = os.path.join(builddir, koji.pathinfo.sighdr(rpm_info, sigkey)) + with open(sig_path, 'rb') as fo: + sighdr = fo.read() + msum = MultiSum(checksum_types) + for buf in koji.spliced_sig_reader(rpm_path, sighdr): + msum.update(buf) + create_rpm_checksum(rpm_id, sigkey, msum.checksums) + query_result = query_checksum.execute() + return create_rpm_checksums_output(query_result, list_checksums_sigkeys) + def writeSignedRPM(self, an_rpm, sigkey, force=False): """Write a signed copy of the rpm""" context.session.assertPerm('sign') @@ -13694,59 +13759,6 @@ class RootExports(object): values=locals(), opts=queryOpts) return query.iterate() - def getRPMChecksums(self, rpm_id, checksum_types=None, cacheonly=False, strict=False): - """Returns RPM checksums for specific rpm. - - :param int rpm_id: RPM id - :param list checksum_type: List of checksum types. Default sha256 checksum type - :param bool cacheonly: when False, checksum is created for missing checksum type - when True, checksum is returned as None when checsum is missing - for specific checksum type - :returns: A dict of specific checksum types and checksums - """ - if not isinstance(rpm_id, int): - raise koji.GenericError('rpm_id must be an integer') - if not checksum_types: - checksum_types = context.opts.get('RPMDefaultChecksums').split() - if not isinstance(checksum_types, list): - raise koji.GenericError('checksum_type must be a list') - - for ch_type in checksum_types: - if ch_type not in koji.CHECKSUM_TYPES: - raise koji.GenericError(f"Checksum_type {ch_type} isn't supported") - - query = QueryProcessor(tables=['rpmsigs'], columns=['sigkey'], - clauses=['rpm_id=%(rpm_id)i'], values={'rpm_id': rpm_id}) - sigkeys = [r['sigkey'] for r in query.execute()] - if not sigkeys: - if strict: - raise koji.GenericError(f'No cached signature for rpm ID {rpm_id}.') - else: - return {} - list_checksums_sigkeys = {s: set(checksum_types) for s in sigkeys} - - checksum_type_int = [koji.CHECKSUM_TYPES[chsum] for chsum in checksum_types] - query_checksum = QueryProcessor(tables=['rpm_checksum'], - columns=['checksum', 'checksum_type', 'sigkey'], - clauses={'rpm_id=%(rpm_id)i', - 'checksum_type IN %(checksum_type)s'}, - values={'rpm_id': rpm_id, - 'checksum_type': checksum_type_int}) - query_result = query_checksum.execute() - if len(query_result) == (len(checksum_type_int) * len(sigkeys)) or cacheonly: - return create_rpm_checksums_output(query_result, list_checksums_sigkeys) - else: - missing_chsum_sigkeys = list_checksums_sigkeys.copy() - for r in query_result: - if r['checksum_type'] in checksum_type_int and r['sigkey'] in sigkeys: - missing_chsum_sigkeys[r['sigkey']].remove( - koji.CHECKSUM_TYPES[r['checksum_type']]) - - for sigkey, chsums in missing_chsum_sigkeys.items(): - write_signed_rpm(rpm_id, sigkey, checksum_types=list(chsums)) - query_result = query_checksum.execute() - return create_rpm_checksums_output(query_result, list_checksums_sigkeys) - class BuildRoot(object): @@ -15549,18 +15561,16 @@ def create_rpm_checksums_output(query_result, list_chsum_sigkeys): return result -def create_rpm_checksum(**kwargs): +def create_rpm_checksum(rpm_id, sigkey, chsum_dict): """Creates RPM checksum. :param int rpm_id: RPM id :param string sigkey: Sigkey for specific RPM :param list checksum_type: List of checksum types. """ - chsum_list = kwargs.get('chsum_list') - sigkey = kwargs.get('sigkey') - rpm_id = kwargs.get('rpm_id') + chsum_dict = chsum_dict.copy() - checksum_type_int = [koji.CHECKSUM_TYPES[func] for func, _ in chsum_list.items()] + checksum_type_int = [koji.CHECKSUM_TYPES[func] for func, _ in chsum_dict.items()] query = QueryProcessor(tables=['rpm_checksum'], columns=['checksum_type'], clauses=["checksum_type IN %(checksum_types)s", 'sigkey=%(sigkey)s'], values={'checksum_types': checksum_type_int, 'sigkey': sigkey}) @@ -15570,10 +15580,10 @@ def create_rpm_checksum(**kwargs): else: for r in rows: if r['checksum_type'] in checksum_type_int: - del chsum_list[koji.CHECKSUM_TYPES[r['checksum_type']]] - if chsum_list: + del chsum_dict[koji.CHECKSUM_TYPES[r['checksum_type']]] + if chsum_dict: insert = BulkInsertProcessor(table='rpm_checksum') - for func, chsum in chsum_list.items(): + for func, chsum in chsum_dict.items(): insert.add_record(rpm_id=rpm_id, sigkey=sigkey, checksum=chsum.hexdigest(), checksum_type=koji.CHECKSUM_TYPES[func]) insert.execute() diff --git a/tests/test_hub/test_create_rpm_checksum.py b/tests/test_hub/test_create_rpm_checksum.py index 1af67a5..3f4bef4 100644 --- a/tests/test_hub/test_create_rpm_checksum.py +++ b/tests/test_hub/test_create_rpm_checksum.py @@ -33,13 +33,11 @@ class TestCreateRPMChecksum(unittest.TestCase): return query def test_checksum_exists(self): - src = 'test-src' - dst = 'test-dst' rpm_id = 123 - chsum_list = {'md5': 'chsum-1', 'sha256': 'chsum-2'} + chsum_dict = {'md5': 'chsum-1', 'sha256': 'chsum-2'} sigkey = 'test-sigkey' self.query_execute.return_value = [{'checksum_type': 'md5'}, {'checksum_type': 'sha256'}] - result = kojihub.create_rpm_checksum(src, dst, rpm_id, sigkey, chsum_list) + result = kojihub.create_rpm_checksum(rpm_id, sigkey, chsum_dict) self.assertIsNone(result) self.assertEqual(len(self.queries), 1) diff --git a/tests/test_hub/test_get_rpm_checksums.py b/tests/test_hub/test_get_rpm_checksums.py index 8024c8b..b32e23f 100644 --- a/tests/test_hub/test_get_rpm_checksums.py +++ b/tests/test_hub/test_get_rpm_checksums.py @@ -14,10 +14,14 @@ class TestGetRpmChecksums(unittest.TestCase): self.create_rpm_checksum_output = mock.patch( 'kojihub.kojihub.create_rpm_checksums_output').start() self.write_signed_rpm = mock.patch('kojihub.kojihub.write_signed_rpm').start() + self.get_rpm = mock.patch('kojihub.kojihub.get_rpm').start() self.QueryProcessor = mock.patch('kojihub.kojihub.QueryProcessor', side_effect=self.getQuery).start() self.queries = [] self.query_execute = mock.MagicMock() + self.rpm_info = {'id': 123, 'name': 'test-name', 'version': '1.1', 'release': '123', + 'arch': 'arch'} + self.nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % self.rpm_info def tearDown(self): mock.patch.stopall() @@ -36,6 +40,7 @@ class TestGetRpmChecksums(unittest.TestCase): def test_checksum_types_not_list(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = 'type' with self.assertRaises(koji.GenericError) as ex: self.exports.getRPMChecksums(rpm_id, checksum_types=checksum_types) @@ -50,6 +55,7 @@ class TestGetRpmChecksums(unittest.TestCase): def test_all_checksum_exists(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = ['md5', 'sha256'] expected_result = {'sigkey1': {'md5': 'checksum-md5', 'sha256': 'checksum-sha256'}} self.query_execute.side_effect = [ @@ -73,41 +79,59 @@ class TestGetRpmChecksums(unittest.TestCase): def test_missing_checksum_not_sigkey_without_strict(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = ['md5'] self.query_execute.side_effect = [[], []] result = self.exports.getRPMChecksums(rpm_id, checksum_types=checksum_types) self.assertEqual({}, result) - self.assertEqual(len(self.queries), 1) + self.assertEqual(len(self.queries), 2) query = self.queries[0] self.assertEqual(query.tables, ['rpmsigs']) self.assertEqual(query.joins, None) self.assertEqual(query.clauses, ['rpm_id=%(rpm_id)i']) + query = self.queries[1] + self.assertEqual(query.tables, ['rpm_checksum']) + self.assertEqual(query.joins, None) + self.assertEqual(query.clauses, + ['checksum_type IN %(checksum_type)s', 'rpm_id=%(rpm_id)i']) + self.write_signed_rpm.assert_not_called() + self.create_rpm_checksum_output.assert_not_called() + def test_missing_checksum_not_sigkey_with_strict(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = ['md5'] self.query_execute.side_effect = [[], []] with self.assertRaises(koji.GenericError) as ex: self.exports.getRPMChecksums(rpm_id, checksum_types=checksum_types, strict=True) - self.assertEqual(f'No cached signature for rpm ID {rpm_id}.', str(ex.exception)) + self.assertEqual(f"Rpm {self.nvra} doesn't have cached checksums or signed copies.", + str(ex.exception)) - self.assertEqual(len(self.queries), 1) + self.assertEqual(len(self.queries), 2) query = self.queries[0] self.assertEqual(query.tables, ['rpmsigs']) self.assertEqual(query.joins, None) self.assertEqual(query.clauses, ['rpm_id=%(rpm_id)i']) + query = self.queries[1] + self.assertEqual(query.tables, ['rpm_checksum']) + self.assertEqual(query.joins, None) + self.assertEqual(query.clauses, + ['checksum_type IN %(checksum_type)s', 'rpm_id=%(rpm_id)i']) + self.write_signed_rpm.assert_not_called() + self.create_rpm_checksum_output.assert_not_called() + def test_missing_valid_checksum_generated(self): rpm_id = 123 checksum_types = ['md5'] - expected_result = {'sigkey1': {'md5': 'checksum-md5'}} + self.get_rpm.return_value = self.rpm_info + expected_result = {} self.query_execute.side_effect = [ [{'sigkey': 'sigkey-1'}], [], [{'checksum': 'checksum-md5', 'checksum_type': 0}]] - self.write_signed_rpm.return_value = None - self.create_rpm_checksum_output.return_value = expected_result result = self.exports.getRPMChecksums(rpm_id, checksum_types=checksum_types) self.assertEqual(expected_result, result) @@ -122,9 +146,40 @@ class TestGetRpmChecksums(unittest.TestCase): self.assertEqual(query.joins, None) self.assertEqual(query.clauses, ['checksum_type IN %(checksum_type)s', 'rpm_id=%(rpm_id)i']) + self.write_signed_rpm.assert_not_called() + self.create_rpm_checksum_output.assert_not_called() + + def test_missing_valid_checksum_generated_with_strict(self): + rpm_id = 123 + checksum_types = ['md5'] + self.get_rpm.return_value = self.rpm_info + self.query_execute.side_effect = [ + [{'sigkey': 'sigkey-1'}], + [], + [{'checksum': 'checksum-md5', 'checksum_type': 0}]] + with self.assertRaises(koji.GenericError) as ex: + self.exports.getRPMChecksums(rpm_id, checksum_types=checksum_types, strict=True) + self.assertEqual(f"Rpm {self.nvra} doesn't have cached checksums or signed copies.", + str(ex.exception)) + + self.assertEqual(len(self.queries), 2) + query = self.queries[0] + self.assertEqual(query.tables, ['rpmsigs']) + self.assertEqual(query.joins, None) + self.assertEqual(query.clauses, ['rpm_id=%(rpm_id)i']) + + query = self.queries[1] + self.assertEqual(query.tables, ['rpm_checksum']) + self.assertEqual(query.joins, None) + self.assertEqual(query.clauses, + ['checksum_type IN %(checksum_type)s', 'rpm_id=%(rpm_id)i']) + + self.write_signed_rpm.assert_not_called() + self.create_rpm_checksum_output.assert_not_called() def test_missing_valid_more_checksum_generated_and_exists(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = ['md5', 'sha256'] expected_result = {'sigkey1': {'md5': 'checksum-md5', 'sha256': 'checksum-sha256'}} self.query_execute.side_effect = [ @@ -151,6 +206,7 @@ class TestGetRpmChecksums(unittest.TestCase): def test_missing_valid_more_checksum_generated_and_exists_more_sigkeys(self): rpm_id = 123 + self.get_rpm.return_value = self.rpm_info checksum_types = ['md5', 'sha256'] expected_result = {'sigkey1': {'md5': 'checksum-md5', 'sha256': 'checksum-sha256'}, 'sigkey2': {'md5': 'checksum-md5', 'sha256': 'checksum-sha256'}} diff --git a/tests/test_hub/test_reset_build.py b/tests/test_hub/test_reset_build.py index a51c945..7ea681c 100644 --- a/tests/test_hub/test_reset_build.py +++ b/tests/test_hub/test_reset_build.py @@ -187,4 +187,4 @@ class TestResetBuild(unittest.TestCase): self.assertEqual(rv, None) self.assertEqual(len(self.queries), 0) self.assertEqual(len(self.deletes), 0) - self.assertEqual(len(self.updates), 0) \ No newline at end of file + self.assertEqual(len(self.updates), 0)