From a8a8913bbc1abf823db7b77edd32bc135d4b1948 Mon Sep 17 00:00:00 2001 From: mprahl Date: Mar 19 2019 20:45:00 +0000 Subject: Add the ability to override a buildrequired module stream based on a module's branch This moves the functionality that was in rpkg to MBS. See the following PRs for more context: https://pagure.io/rpkg/pull-request/390 https://pagure.io/rpkg/pull-request/420 --- diff --git a/module_build_service/config.py b/module_build_service/config.py index 361d5fd..e003669 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -499,7 +499,23 @@ class Config(object): 'allowed_users': { 'type': set, 'default': set(), - 'desc': 'The users/service accounts that don\'t require to be part of a group'} + 'desc': 'The users/service accounts that don\'t require to be part of a group'}, + 'br_stream_override_module': { + 'type': str, + 'default': 'platform', + 'desc': ('The module name to override in the buildrequires based on the branch name. ' + '"br_stream_override_regexes" must also be set for this to take ' + 'effect.') + }, + 'br_stream_override_regexes': { + 'type': list, + 'default': [], + 'desc': ('The list of regexes used to parse the stream override from the branch name. ' + '"br_stream_override_module" must also be set for this to take ' + 'effect. The regexes can contain multiple capture groups that will be ' + 'concatenated. Any null capture groups will be ignored. The first regex that ' + 'matches the branch will be used.') + }, } def __init__(self, conf_section_obj): diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 05fcfff..37b970c 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -30,6 +30,7 @@ import tempfile import os from multiprocessing.dummy import Pool as ThreadPool from datetime import datetime +import copy from module_build_service.utils import to_text_type import kobo.rpmlib @@ -516,10 +517,46 @@ def _apply_dep_overrides(mmd, params): :raises ValidationError: if one of the overrides doesn't apply """ dep_overrides = { - 'buildrequires': params.get('buildrequire_overrides', {}), - 'requires': params.get('require_overrides', {}) + 'buildrequires': copy.copy(params.get('buildrequire_overrides', {})), + 'requires': copy.copy(params.get('require_overrides', {})) } + # Parse the module's branch to determine if it should override the stream of the buildrequired + # module defined in conf.br_stream_override_module + branch_search = None + if params.get('branch') and conf.br_stream_override_module and conf.br_stream_override_regexes: + # Only parse the branch for a buildrequire override if the user didn't manually specify an + # override for the module specified in conf.br_stream_override_module + if not dep_overrides['buildrequires'].get(conf.br_stream_override_module): + branch_search = None + for regex in conf.br_stream_override_regexes: + branch_search = re.search(regex, params['branch']) + if branch_search: + log.debug( + 'The stream override regex `%s` matched the branch %s', + regex, params['branch']) + break + else: + log.debug('No stream override regexes matched the branch "%s"', params['branch']) + + # If a stream was parsed from the branch, then add it as a stream override for the module + # specified in conf.br_stream_override_module + if branch_search: + # Concatenate all the groups that are not None together to get the desired stream. + # This approach is taken in case there are sections to ignore. + # For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`. + parsed_stream = ''.join(group for group in branch_search.groups() if group) + if parsed_stream: + dep_overrides['buildrequires'][conf.br_stream_override_module] = [parsed_stream] + log.info( + 'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"', + conf.br_stream_override_module, parsed_stream, params['branch']) + else: + log.warning( + ('The regex `%s` only matched empty capture groups on the branch "%s". The regex ' + 'is invalid and should be rewritten.'), + regex, params['branch']) + unused_dep_overrides = { 'buildrequires': set(dep_overrides['buildrequires'].keys()), 'requires': set(dep_overrides['requires'].keys()) @@ -541,6 +578,10 @@ def _apply_dep_overrides(mmd, params): getattr(dep, 'set_' + dep_type)(reqs) for dep_type in unused_dep_overrides.keys(): + # If a stream override was applied from parsing the branch and it wasn't applicable, + # just ignore it + if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]: + unused_dep_overrides[dep_type].remove(conf.br_stream_override_module) if unused_dep_overrides[dep_type]: raise ValidationError( 'The {} overrides for the following modules aren\'t applicable: {}' diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 7ed653d..db5b973 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -44,6 +44,7 @@ from module_build_service import db, version, Modulemd import module_build_service.config as mbs_config import module_build_service.scheduler.handlers.modules from module_build_service.utils import import_mmd, load_mmd +from module_build_service.glib import dict_values, from_variant_dict user = ('Homer J. Simpson', set(['packager'])) @@ -1673,7 +1674,6 @@ class TestViews: def test_buildrequires_is_included_in_json_output(self): # Inject xmd/mbs/buildrequires into an existing module build for # assertion later. - from module_build_service.glib import dict_values, from_variant_dict from module_build_service.models import make_session from module_build_service import conf br_modulea = dict(stream='6', version='1', context='1234') @@ -1903,3 +1903,96 @@ class TestViews: # this test is the same as the previous except YAML_SUBMIT_ALLOWED is False, # but it should still succeed since yaml is always allowed for scratch builds assert rv.status_code == 201 + + @pytest.mark.parametrize('branch, platform_override', ( + ('10', None), + ('10-rhel-8.0.0', 'el8.0.0'), + ('10-LP-product1.2', 'product1.2'), + )) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + @patch.object(module_build_service.config.Config, 'br_stream_override_regexes', + new_callable=PropertyMock) + def test_submit_build_dep_override_from_branch( + self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override): + """ + Test that MBS will parse the SCM branch to determine the platform stream to buildrequire. + """ + mocked_regexes.return_value = [ + r'(?:rh)(el)(?:\-)(\d+\.\d+\.\d+)$', + r'(?:\-LP\-)(.+)$' + ] + init_data(data_size=1, multiple_stream_versions=True) + # Create a platform for whatever the override is so the build submission succeeds + if platform_override: + platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True) + platform_mmd.set_stream(platform_override) + if platform_override == 'el8.0.0': + xmd = from_variant_dict(platform_mmd.get_xmd()) + xmd['mbs']['virtual_streams'] = 'el8' + platform_mmd.set_xmd(dict_values(xmd)) + import_mmd(db.session, platform_mmd) + + FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + + post_url = '/module-build-service/2/module-builds/' + scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe' + 'efbd35246a81b6cb8d49') + + rv = self.client.post(post_url, data=json.dumps({'branch': branch, 'scmurl': scm_url})) + data = json.loads(rv.data) + assert rv.status_code == 201 + + mmd = Modulemd.Module().new_from_string(data[0]['modulemd']) + assert len(mmd.get_dependencies()) == 1 + dep = mmd.get_dependencies()[0] + if platform_override: + expected_br = {platform_override} + else: + expected_br = {'f28'} + assert set(dep.get_buildrequires()['platform'].get()) == expected_br + # The requires should not change + assert set(dep.get_requires()['platform'].get()) == {'f28'} + + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + @patch.object(module_build_service.config.Config, 'br_stream_override_regexes', + new_callable=PropertyMock) + def test_submit_build_dep_override_from_branch_br_override( + self, mocked_regexes, mocked_scm, mocked_get_user): + """ + Test that when the branch includes a stream override for the platform module, that the + provided "buildrequire_override" for the platform module takes precedence. + """ + mocked_regexes.return_value = [r'(?:\-LP\-)(.+)$'] + init_data(data_size=1, multiple_stream_versions=True) + # Create a platform for the override so the build submission succeeds + platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True) + platform_mmd.set_stream('product1.3') + import_mmd(db.session, platform_mmd) + + FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + + post_url = '/module-build-service/2/module-builds/' + scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe' + 'efbd35246a81b6cb8d49') + json_input = { + 'branch': '10-LP-product1.2', + 'scmurl': scm_url, + 'buildrequire_overrides': {'platform': ['product1.3']} + } + + rv = self.client.post(post_url, data=json.dumps(json_input)) + data = json.loads(rv.data) + assert rv.status_code == 201 + + mmd = Modulemd.Module().new_from_string(data[0]['modulemd']) + assert len(mmd.get_dependencies()) == 1 + dep = mmd.get_dependencies()[0] + # The buildrequire_override value should take precedence over the stream override from + # parsing the branch + assert set(dep.get_buildrequires()['platform'].get()) == {'product1.3'} + # The requires should not change + assert set(dep.get_requires()['platform'].get()) == {'f28'}