PR#1176: Add the ability to override a buildrequired module stream based on a module's branch - fm-orchestrator

fm-orchestrator

#1176 Add the ability to override a buildrequired module stream based on a module's branch

Merged 5 years ago by jkaluza. Opened 5 years ago by mprahl.

stream-overrides into master

Add the ability to override a buildrequired module stream based on a module's branch

mprahl • 5 years ago

a8a8913

module_build_service/config.py

file modified

+17 -1

		`@@ -499,7 +499,23 @@`
		`'allowed_users': {`
		`'type': set,`
		`'default': set(),`
		`- 'desc': 'The users/service accounts that don\'t require to be part of a group'}`
		`+ 'desc': 'The users/service accounts that don\'t require to be part of a group'},`
		`+ 'br_stream_override_module': {`
		`+ 'type': str,`
		`+ 'default': 'platform',`
		`+ 'desc': ('The module name to override in the buildrequires based on the branch name. '`
		`+ '"br_stream_override_regexes" must also be set for this to take '`
		`+ 'effect.')`
		`+ },`
		`+ 'br_stream_override_regexes': {`
		`+ 'type': list,`
		`+ 'default': [],`
		`+ 'desc': ('The list of regexes used to parse the stream override from the branch name. '`
		`+ '"br_stream_override_module" must also be set for this to take '`
		`+ 'effect. The regexes can contain multiple capture groups that will be '`
		`+ 'concatenated. Any null capture groups will be ignored. The first regex that '`
		`+ 'matches the branch will be used.')`
		`+ },`
		`}`

		`def __init__(self, conf_section_obj):`

module_build_service/utils/submit.py

file modified

+43 -2

		`@@ -30,6 +30,7 @@`
		`import os`
		`from multiprocessing.dummy import Pool as ThreadPool`
		`from datetime import datetime`
		`+ import copy`
		`from module_build_service.utils import to_text_type`

		`import kobo.rpmlib`
		`@@ -516,10 +517,46 @@`
		`:raises ValidationError: if one of the overrides doesn't apply`
		`"""`
		`dep_overrides = {`
		`- 'buildrequires': params.get('buildrequire_overrides', {}),`
		`- 'requires': params.get('require_overrides', {})`
		`+ 'buildrequires': copy.copy(params.get('buildrequire_overrides', {})),`
		`+ 'requires': copy.copy(params.get('require_overrides', {}))`
		`}`

		`+ # Parse the module's branch to determine if it should override the stream of the buildrequired`
		`+ # module defined in conf.br_stream_override_module`
		`+ branch_search = None`
		`+ if params.get('branch') and conf.br_stream_override_module and conf.br_stream_override_regexes:`
		`+ # Only parse the branch for a buildrequire override if the user didn't manually specify an`
		`+ # override for the module specified in conf.br_stream_override_module`
		`+ if not dep_overrides['buildrequires'].get(conf.br_stream_override_module):`
		`+ branch_search = None`
		`+ for regex in conf.br_stream_override_regexes:`
		`+ branch_search = re.search(regex, params['branch'])`
		`+ if branch_search:`
		`+ log.debug(`
		+ 'The stream override regex `%s` matched the branch %s',
		`+ regex, params['branch'])`
		`+ break`
		`+ else:`
		`+ log.debug('No stream override regexes matched the branch "%s"', params['branch'])`
		`+`
		`+ # If a stream was parsed from the branch, then add it as a stream override for the module`
		`+ # specified in conf.br_stream_override_module`
		`+ if branch_search:`
		`+ # Concatenate all the groups that are not None together to get the desired stream.`
		`+ # This approach is taken in case there are sections to ignore.`
		+ # For instance, if we need to parse `el8.0.0` from `rhel-8.0.0`.
		`+ parsed_stream = ''.join(group for group in branch_search.groups() if group)`
		`+ if parsed_stream:`
		`+ dep_overrides['buildrequires'][conf.br_stream_override_module] = [parsed_stream]`
		`+ log.info(`
		`+ 'The buildrequired stream of "%s" was overriden with "%s" based on the branch "%s"',`
		`+ conf.br_stream_override_module, parsed_stream, params['branch'])`
		`+ else:`
		`+ log.warning(`
		+ ('The regex `%s` only matched empty capture groups on the branch "%s". The regex '
		`+ 'is invalid and should be rewritten.'),`
		`+ regex, params['branch'])`
		`+`
		`unused_dep_overrides = {`
		`'buildrequires': set(dep_overrides['buildrequires'].keys()),`
		`'requires': set(dep_overrides['requires'].keys())`
		`@@ -541,6 +578,10 @@`
		`getattr(dep, 'set_' + dep_type)(reqs)`

		`for dep_type in unused_dep_overrides.keys():`
		`+ # If a stream override was applied from parsing the branch and it wasn't applicable,`
		`+ # just ignore it`
		`+ if branch_search and conf.br_stream_override_module in unused_dep_overrides[dep_type]:`
		`+ unused_dep_overrides[dep_type].remove(conf.br_stream_override_module)`
		`if unused_dep_overrides[dep_type]:`
		`raise ValidationError(`
		`'The {} overrides for the following modules aren\'t applicable: {}'`

tests/test_views/test_views.py

file modified

+94 -1

		`@@ -44,6 +44,7 @@`
		`import module_build_service.config as mbs_config`
		`import module_build_service.scheduler.handlers.modules`
		`from module_build_service.utils import import_mmd, load_mmd`
		`+ from module_build_service.glib import dict_values, from_variant_dict`


		`user = ('Homer J. Simpson', set(['packager']))`
		`@@ -1673,7 +1674,6 @@`
		`def test_buildrequires_is_included_in_json_output(self):`
		`# Inject xmd/mbs/buildrequires into an existing module build for`
		`# assertion later.`
		`- from module_build_service.glib import dict_values, from_variant_dict`
		`from module_build_service.models import make_session`
		`from module_build_service import conf`
		`br_modulea = dict(stream='6', version='1', context='1234')`
		`@@ -1903,3 +1903,96 @@`
		`# this test is the same as the previous except YAML_SUBMIT_ALLOWED is False,`
		`# but it should still succeed since yaml is always allowed for scratch builds`
		`assert rv.status_code == 201`
		`+`
		`+ @pytest.mark.parametrize('branch, platform_override', (`
		`+ ('10', None),`
		`+ ('10-rhel-8.0.0', 'el8.0.0'),`
		`+ ('10-LP-product1.2', 'product1.2'),`
		`+ ))`
		`+ @patch('module_build_service.auth.get_user', return_value=user)`
		`+ @patch('module_build_service.scm.SCM')`
		`+ @patch.object(module_build_service.config.Config, 'br_stream_override_regexes',`
		`+ new_callable=PropertyMock)`
		`+ def test_submit_build_dep_override_from_branch(`
		`+ self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override):`
		`+ """`
		`+ Test that MBS will parse the SCM branch to determine the platform stream to buildrequire.`
		`+ """`
		`+ mocked_regexes.return_value = [`
		`+ r'(?:rh)(el)(?:\-)(\d+\.\d+\.\d+)$',`
		`+ r'(?:\-LP\-)(.+)$'`
		`+ ]`
		`+ init_data(data_size=1, multiple_stream_versions=True)`
		`+ # Create a platform for whatever the override is so the build submission succeeds`
		`+ if platform_override:`
		`+ platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True)`
		`+ platform_mmd.set_stream(platform_override)`
		`+ if platform_override == 'el8.0.0':`
		`+ xmd = from_variant_dict(platform_mmd.get_xmd())`
		`+ xmd['mbs']['virtual_streams'] = 'el8'`
		`+ platform_mmd.set_xmd(dict_values(xmd))`
		`+ import_mmd(db.session, platform_mmd)`
		`+`
		`+ FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',`
		`+ '620ec77321b2ea7b0d67d82992dda3e1d67055b4')`
		`+`
		`+ post_url = '/module-build-service/2/module-builds/'`
		`+ scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe'`
		`+ 'efbd35246a81b6cb8d49')`
		`+`
		`+ rv = self.client.post(post_url, data=json.dumps({'branch': branch, 'scmurl': scm_url}))`
		`+ data = json.loads(rv.data)`
		`+ assert rv.status_code == 201`
		`+`
		`+ mmd = Modulemd.Module().new_from_string(data[0]['modulemd'])`
		`+ assert len(mmd.get_dependencies()) == 1`
		`+ dep = mmd.get_dependencies()[0]`
		`+ if platform_override:`
		`+ expected_br = {platform_override}`
		`+ else:`
		`+ expected_br = {'f28'}`
		`+ assert set(dep.get_buildrequires()['platform'].get()) == expected_br`
		`+ # The requires should not change`
		`+ assert set(dep.get_requires()['platform'].get()) == {'f28'}`
		`+`
		`+ @patch('module_build_service.auth.get_user', return_value=user)`
		`+ @patch('module_build_service.scm.SCM')`
		`+ @patch.object(module_build_service.config.Config, 'br_stream_override_regexes',`
		`+ new_callable=PropertyMock)`
		`+ def test_submit_build_dep_override_from_branch_br_override(`
		`+ self, mocked_regexes, mocked_scm, mocked_get_user):`
		`+ """`
		`+ Test that when the branch includes a stream override for the platform module, that the`
		`+ provided "buildrequire_override" for the platform module takes precedence.`
		`+ """`
		`+ mocked_regexes.return_value = [r'(?:\-LP\-)(.+)$']`
		`+ init_data(data_size=1, multiple_stream_versions=True)`
		`+ # Create a platform for the override so the build submission succeeds`
		`+ platform_mmd = load_mmd(path.join(base_dir, 'staged_data', 'platform.yaml'), True)`
		`+ platform_mmd.set_stream('product1.3')`
		`+ import_mmd(db.session, platform_mmd)`
		`+`
		`+ FakeSCM(mocked_scm, 'testmodule', 'testmodule.yaml',`
		`+ '620ec77321b2ea7b0d67d82992dda3e1d67055b4')`
		`+`
		`+ post_url = '/module-build-service/2/module-builds/'`
		`+ scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe'`
		`+ 'efbd35246a81b6cb8d49')`
		`+ json_input = {`
		`+ 'branch': '10-LP-product1.2',`
		`+ 'scmurl': scm_url,`
		`+ 'buildrequire_overrides': {'platform': ['product1.3']}`
		`+ }`
		`+`
		`+ rv = self.client.post(post_url, data=json.dumps(json_input))`
		`+ data = json.loads(rv.data)`
		`+ assert rv.status_code == 201`
		`+`
		`+ mmd = Modulemd.Module().new_from_string(data[0]['modulemd'])`
		`+ assert len(mmd.get_dependencies()) == 1`
		`+ dep = mmd.get_dependencies()[0]`
		`+ # The buildrequire_override value should take precedence over the stream override from`
		`+ # parsing the branch`
		`+ assert set(dep.get_buildrequires()['platform'].get()) == {'product1.3'}`
		`+ # The requires should not change`
		`+ assert set(dep.get_requires()['platform'].get()) == {'f28'}`