From 0aff640ef3ac0768ae0cd61421f69d19d0aba8af Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Aug 31 2018 19:23:04 +0000 Subject: [PATCH 1/2] Optionally, block building modules with EOL streams. Requested by @mboddu in: - https://pagure.io/fm-orchestrator/issue/960 - https://pagure.io/modularity/issue/102 --- diff --git a/conf/config.py b/conf/config.py index 0ab30c0..0b4b4f8 100644 --- a/conf/config.py +++ b/conf/config.py @@ -32,6 +32,7 @@ class BaseConfiguration(object): KOJI_REPOSITORY_URL = 'https://kojipkgs.fedoraproject.org/repos' KOJI_TAG_PREFIXES = ['module'] KOJI_ENABLE_CONTENT_GENERATOR = True + PDC_CHECK_FOR_EOL = False PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' PDC_INSECURE = False PDC_DEVELOP = True diff --git a/module_build_service/config.py b/module_build_service/config.py index de3c6cf..63165b7 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -159,6 +159,14 @@ class Config(object): 'type': str, 'default': 'https://mbs.fedoraproject.org/module-build-service/1/module-builds/', 'desc': 'MBS instance url for MBSResolver'}, + 'pdc_check_for_eol': { + 'type': bool, + 'default': False, + 'desc': 'Flag to determine whether or not MBS should block EOL modules from building.'}, + 'pdc_url': { + 'type': str, + 'default': 'https://pdc.fedoraproject.org/rest_api/v1', + 'desc': 'PDC URL, used for checking stream EOL.'}, 'koji_config': { 'type': str, 'default': None, diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index d4aaa1f..9b0bfbd 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -29,7 +29,9 @@ import tempfile import os from multiprocessing.dummy import Pool as ThreadPool from datetime import datetime + import kobo.rpmlib +import requests from module_build_service import conf, db, log, models, Modulemd from module_build_service.errors import ( @@ -440,6 +442,26 @@ def submit_module_build(username, url, mmd, scm, optional_params=None): return modules +def _is_eol_in_pdc(name, stream): + """ Check PDC if the module name:stream is no longer active. """ + + params = {'type': 'module', 'global_component': name, 'name': stream} + url = conf.pdc_url + '/component-branches/' + + response = requests.get(url, params=params) + if not response: + raise ValidationError("Failed to talk to PDC {}{}".format(response, response.text)) + + data = response.json() + results = data['results'] + if not results: + raise ValidationError("No such module {}:{} found at {}".format( + name, stream, response.request.url)) + + # If the module is active, then it is not EOL and vice versa. + return not results[0]['active'] + + def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. @@ -467,6 +489,11 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): "Failed to remove temporary directory {!r}: {}".format( td, str(e))) + if conf.pdc_check_for_eol: + if _is_eol_in_pdc(scm.name, scm.branch): + raise ValidationError( + 'Module {}:{} is marked as EOL in PDC.'.format(scm.name, scm.branch)) + # If the name was set in the modulemd, make sure it matches what the scmurl # says it should be if mmd.get_name() and mmd.get_name() != scm.name: diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 1504586..4122afb 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -414,6 +414,25 @@ class TestBuild: # Make sure the build is done assert module_build.state == models.BUILD_STATES['ready'] + @patch('module_build_service.config.Config.pdc_check_for_eol', + new_callable=PropertyMock, return_value=True) + @patch('module_build_service.utils.submit._is_eol_in_pdc', return_value=True) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_eol_module(self, mocked_scm, mocked_get_user, is_eol, check, + conf_system, dbg): + """ Tests the build of a module with an eol stream. """ + FakeSCM(mocked_scm, 'python3', 'python3-no-components.yaml', + '620ec77321b2ea7b0d67d82992dda3e1d67055b4') + rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( + {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'})) + + assert rv.status_code == 400 + data = json.loads(rv.data) + assert data['status'] == 400 + assert data['message'] == u'Module python3:master is marked as EOL in PDC.' + @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') def test_submit_build_from_yaml_not_allowed( diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 1c494b2..21ce874 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -551,6 +551,34 @@ class TestUtils: expected_tag = 'module-1cf457d452e54dda' assert tag == expected_tag + @patch("module_build_service.utils.submit.requests") + def test_pdc_eol_check(self, requests): + """ Push mock pdc responses through the eol check function. """ + + response = mock.Mock() + response.json.return_value = {"results": [{ + "id": 347907, + "global_component": "mariadb", + "name": "10.1", + "slas": [{ + "id": 694207, + "sla": "security_fixes", + "eol": "2019-12-01", + }], + "type": "module", + "active": True, + "critical_path": False, + }]} + requests.get.return_value = response + + is_eol = module_build_service.utils.submit._is_eol_in_pdc('mariadb', '10.1') + assert not is_eol + + response.json.return_value["results"][0]["active"] = False + + is_eol = module_build_service.utils.submit._is_eol_in_pdc('mariadb', '10.1') + assert is_eol + class DummyModuleBuilder(GenericBuilder): """ From e7089d7be0a6a56e9cbd793ec7b0c2fc596bbcc3 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sep 04 2018 13:51:56 +0000 Subject: [PATCH 2/2] Rename this to be PDC-agnostic. --- diff --git a/conf/config.py b/conf/config.py index 0b4b4f8..117d29e 100644 --- a/conf/config.py +++ b/conf/config.py @@ -32,7 +32,7 @@ class BaseConfiguration(object): KOJI_REPOSITORY_URL = 'https://kojipkgs.fedoraproject.org/repos' KOJI_TAG_PREFIXES = ['module'] KOJI_ENABLE_CONTENT_GENERATOR = True - PDC_CHECK_FOR_EOL = False + CHECK_FOR_EOL = False PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' PDC_INSECURE = False PDC_DEVELOP = True diff --git a/module_build_service/config.py b/module_build_service/config.py index 63165b7..494ab0b 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -159,7 +159,7 @@ class Config(object): 'type': str, 'default': 'https://mbs.fedoraproject.org/module-build-service/1/module-builds/', 'desc': 'MBS instance url for MBSResolver'}, - 'pdc_check_for_eol': { + 'check_for_eol': { 'type': bool, 'default': False, 'desc': 'Flag to determine whether or not MBS should block EOL modules from building.'}, diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 9b0bfbd..d2bcec9 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -489,7 +489,7 @@ def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): "Failed to remove temporary directory {!r}: {}".format( td, str(e))) - if conf.pdc_check_for_eol: + if conf.check_for_eol: if _is_eol_in_pdc(scm.name, scm.branch): raise ValidationError( 'Module {}:{} is marked as EOL in PDC.'.format(scm.name, scm.branch)) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 4122afb..0603d6e 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -414,7 +414,7 @@ class TestBuild: # Make sure the build is done assert module_build.state == models.BUILD_STATES['ready'] - @patch('module_build_service.config.Config.pdc_check_for_eol', + @patch('module_build_service.config.Config.check_for_eol', new_callable=PropertyMock, return_value=True) @patch('module_build_service.utils.submit._is_eol_in_pdc', return_value=True) @patch('module_build_service.auth.get_user', return_value=user)