From dd5667665d321bbfbcc18e7bbfffc95b58953626 Mon Sep 17 00:00:00 2001 From: mprahl Date: May 21 2019 12:55:52 +0000 Subject: Query the Red Hat Product Pages to see if this is a Z stream build when configured In certain use-cases, a module's buildrequires may remain the same in the modulemd, but a different support stream of the buildrequired base module should be used. For example, since RHEL 8.0.0 is GA, any modules that buildrequire platform:el8.0.0 should buildrequire platform:el8.0.0z instead. --- diff --git a/module_build_service/config.py b/module_build_service/config.py index d5ded11..1262b27 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -30,6 +30,8 @@ import pkg_resources import re import sys +from six import string_types + from module_build_service import logger @@ -623,6 +625,25 @@ class Config(object): "type": bool, "default": False, "desc": "Allow module scratch builds", + }, + "product_pages_url": { + "type": str, + "default": "", + "desc": "The URL to the Product Pages. This is queried to determine if a base module " + "stream has been released. If it has, the stream may be modified automatically " + "to use a different support stream.", + }, + "product_pages_module_streams": { + "type": dict, + "default": {}, + "desc": "The keys are regexes of base module streams that should be checked in the Red " + "Hat Product Pages. The values are tuples. The first value is a string that " + "should be appended to the stream if there is a match and the release the " + "stream represents has been released. The second value is a template string " + "that represents the release in Product Pages and can accept format kwargs of " + "x, y, and z (represents the version). The third value is an optional template " + "string that represent the Product Pages release for major releases " + "(e.g. 8.0.0). After the first match, the rest will be ignored." } } @@ -838,3 +859,42 @@ class Config(object): s, ", ".join(SUPPORTED_RESOLVERS.keys())) ) self._resolver = s + + def _setifok_product_pages_module_streams(self, d): + if not isinstance(d, dict): + raise ValueError("PRODUCT_PAGES_MODULE_STREAMS must be a dict") + + for regex, values in d.items(): + try: + re.compile(regex) + except (TypeError, re.error): + raise ValueError( + 'The regex `%r` in the configuration "PRODUCT_PAGES_MODULE_STREAMS" is invalid' + % regex + ) + + if not isinstance(values, list) and not isinstance(values, tuple): + raise ValueError( + 'The values in the configured dictionary for "PRODUCT_PAGES_MODULE_STREAMS" ' + "must be a list or tuple" + ) + + if len(values) != 3: + raise ValueError( + "There must be three entries in each value in the dictionary configured for " + '"PRODUCT_PAGES_MODULE_STREAMS"' + ) + + for i, value in enumerate(values): + if not isinstance(value, string_types): + # The last value is optional + if value is None and i == 2: + continue + + raise ValueError( + 'The value in the %i index of the values in "PRODUCT_PAGES_MODULE_STREAMS" ' + "must be a string" + % i + ) + + self._product_pages_module_streams = d diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index 3c6497b..bd68436 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -753,6 +753,125 @@ def resolve_base_module_virtual_streams(name, streams): return new_streams +def _process_support_streams(mmd, params): + """ + Check if any buildrequired base modules require a support stream suffix. + + This checks the Red Hat Product Pages to see if the buildrequired base module stream has been + released, if yes, then add the appropriate stream suffix. + + :param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on + :param dict params: the API parameters passed in by the user + """ + config_msg = ( + 'Skipping the release date checks for adding a stream suffix since "%s" ' + "is not configured" + ) + if not conf.product_pages_url: + log.debug(config_msg, "product_pages_url") + return + elif not conf.product_pages_module_streams: + log.debug(config_msg, "product_pages_module_streams") + return + + buildrequire_overrides = params.get("buildrequire_overrides", {}) + + def new_streams_func(name, streams): + if name not in conf.base_module_names: + log.debug("The module %s is not a base module. Skipping the release date check.", name) + return streams + elif name in buildrequire_overrides: + log.debug( + "The module %s is a buildrequire override. Skipping the release date check.", name) + return streams + + new_streams = copy.deepcopy(streams) + for i, stream in enumerate(streams): + for regex, values in conf.product_pages_module_streams.items(): + if re.match(regex, stream): + log.debug( + 'The regex `%s` from the configuration "product_pages_module_streams" ' + "matched the stream %s", + regex, stream, + ) + stream_suffix, pp_release_template, pp_major_release_template = values + break + else: + log.debug( + 'No regexes in the configuration "product_pages_module_streams" matched the ' + "stream %s. Skipping the release date check for this stream.", + stream, + ) + continue + + if stream.endswith(stream_suffix): + log.debug( + 'The stream %s already contains the stream suffix of "%s". Skipping the ' + "release date check.", + stream, stream_suffix + ) + continue + + stream_version = models.ModuleBuild.get_stream_version(stream) + if not stream_version: + log.debug("A stream version couldn't be parsed from %s", stream) + continue + + # Convert the stream_version float to an int to make the math below deal with only + # integers + stream_version_int = int(stream_version) + # For example 80000 => 8 + x = stream_version_int // 10000 + # For example 80100 => 1 + y = (stream_version_int - x * 10000) // 100 + # For example 80104 => 4 + z = stream_version_int - x * 10000 - y * 100 + # Check if the stream version is x.0.0 + if stream_version_int % 10000 == 0 and pp_major_release_template: + # For example, el8.0.0 => rhel-8-0 + pp_release = pp_major_release_template.format(x=x, y=y, z=z) + else: + # For example el8.0.1 => rhel-8-0.1 + pp_release = pp_release_template.format(x=x, y=y, z=z) + + url = "{}/api/v7/releases/{}/?fields=ga_date".format( + conf.product_pages_url.rstrip("/"), pp_release) + + try: + pp_rv = requests.get(url, timeout=15) + pp_json = pp_rv.json() + # Catch requests failures and JSON parsing errors + except (requests.exceptions.RequestException, ValueError): + log.exception( + "The query to the Product Pages at %s failed. Assuming it is not yet released.", + url, + ) + continue + + ga_date = pp_json.get("ga_date") + if not ga_date: + log.debug("A release date for the release %s could not be determined", pp_release) + continue + + if datetime.strptime(ga_date, '%Y-%m-%d') > datetime.utcnow(): + log.debug( + "The release %s hasn't been released yet. Not adding a stream suffix.", + ga_date + ) + continue + + new_stream = stream + stream_suffix + log.info( + 'Replacing the buildrequire "%s:%s" with "%s:%s", since the stream is released', + name, stream, name, new_stream + ) + new_streams[i] = new_stream + + return new_streams + + _modify_buildtime_streams(mmd, new_streams_func) + + def submit_module_build(username, mmd, params): """ Submits new module build. @@ -786,6 +905,7 @@ def submit_module_build(username, mmd, params): default_streams = params["default_streams"] _apply_dep_overrides(mmd, params) _modify_buildtime_streams(mmd, resolve_base_module_virtual_streams) + _process_support_streams(mmd, params) mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams) if not mmds: diff --git a/tests/staged_data/testmodule_el821.yaml b/tests/staged_data/testmodule_el821.yaml new file mode 100644 index 0000000..a57154c --- /dev/null +++ b/tests/staged_data/testmodule_el821.yaml @@ -0,0 +1,37 @@ +document: modulemd +version: 1 +data: + summary: A test module in all its beautiful beauty + description: >- + This module demonstrates how to write simple modulemd files And + can be used for testing the build and release pipeline. ’ + license: + module: [ MIT ] + dependencies: + buildrequires: + platform: el8.2.1 + requires: + platform: el8.0.0 + references: + community: https://docs.pagure.org/modularity/ + documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules + profiles: + default: + rpms: + - tangerine + api: + rpms: + - perl-Tangerine + - tangerine + components: + rpms: + perl-List-Compare: + rationale: A dependency of tangerine. + ref: master + perl-Tangerine: + rationale: Provides API for this module and is a dependency of tangerine. + ref: master + tangerine: + rationale: Provides API for this module. + buildorder: 10 + ref: master diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 7674836..f1a094d 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -2436,3 +2436,151 @@ class TestViews: dep = mmd.get_dependencies()[0] assert dep.get_buildtime_streams("platform") == ["el8.25.0"] assert dep.get_runtime_streams("platform") == ["el8"] + + @pytest.mark.parametrize( + "pp_url, pp_streams, get_rv, br_stream, br_override, expected_stream", + ( + # Test a stream of a major release + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {}, + "el8.0.0z", + ), + # Test when the releases GA date is far in the future + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"ga_date": "2099-10-30"}, + "el8.0.0", + {}, + "el8.0.0", + ), + # Test when product_pages_url isn't set + ( + "", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {}, + "el8.0.0", + ), + # Test when the release isn't found in Product Pages + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"detail": "Not found."}, + "el8.0.0", + {}, + "el8.0.0", + ), + # Test when a non-major release stream + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"ga_date": "2019-05-07"}, + "el8.2.1", + {}, + "el8.2.1z", + ), + # Test that when buildrequire overrides is set for platform, nothing changes + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {"platform": ["el8.0.0"]}, + "el8.0.0", + ), + # Test when product_pages_module_streams is not set + ( + "https://pp.domain.local/pp/", + {}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {}, + "el8.0.0", + ), + # Test when there is no stream that matches the configured regexes + ( + "https://pp.domain.local/pp/", + {r"js.+": ("z", "js-{x}-{y}", "js-{x}-{y}")}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {}, + "el8.0.0", + ), + # Test when there is no configured special Product Pages template for major releases + ( + "https://pp.domain.local/pp/", + {r"el.+": ("z", "rhel-{x}-{y}", None)}, + {"ga_date": "2019-05-07"}, + "el8.0.0", + {}, + "el8.0.0z", + ), + ), + ) + @patch( + "module_build_service.config.Config.product_pages_url", + new_callable=PropertyMock, + ) + @patch( + "module_build_service.config.Config.product_pages_module_streams", + new_callable=PropertyMock, + ) + @patch("requests.get") + @patch("module_build_service.auth.get_user", return_value=user) + @patch("module_build_service.scm.SCM") + def test_submit_build_automatic_z_stream_detection( + self, mocked_scm, mocked_get_user, mock_get, mock_pp_streams, mock_pp_url, pp_url, + pp_streams, get_rv, br_stream, br_override, expected_stream, + ): + # Configure the Product Pages URL + mock_pp_url.return_value = pp_url + mock_pp_streams.return_value = pp_streams + # Mock the Product Pages query + mock_get.return_value.json.return_value = get_rv + mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml")) + # Create the required platforms + for stream in ("el8.0.0", "el8.0.0z", "el8.2.1", "el8.2.1z"): + mmd = mmd.copy(mmd.get_module_name(), stream) + import_mmd(db.session, mmd) + + # Use a testmodule that buildrequires platform:el8.0.0 or platform:el8.2.1 + FakeSCM( + mocked_scm, + "testmodule", + "testmodule_{}.yaml".format(br_stream.replace(".", "")), + "620ec77321b2ea7b0d67d82992dda3e1d67055b4", + ) + + post_url = "/module-build-service/2/module-builds/" + scm_url = ( + "https://src.stg.fedoraproject.org/modules/testmodule.git?#" + "68931c90de214d9d13feefbd35246a81b6cb8d49" + ) + payload = {"branch": "master", "scmurl": scm_url} + if br_override: + payload["buildrequire_overrides"] = br_override + rv = self.client.post(post_url, json=payload) + data = json.loads(rv.data) + + mmd = load_mmd(data[0]["modulemd"]) + assert len(mmd.get_dependencies()) == 1 + dep = mmd.get_dependencies()[0] + assert dep.get_buildtime_streams("platform") == [expected_stream] + # The runtime stream suffix should remain unchanged + assert dep.get_runtime_streams("platform") == ["el8.0.0"] + + if pp_url and not br_override and pp_streams.get(r"el.+"): + if br_stream == "el8.0.0": + pp_release = "rhel-8-0" + else: + pp_release = "rhel-8-2.1" + expected_url = "{}api/v7/releases/{}/?fields=ga_date".format(pp_url, pp_release) + mock_get.assert_called_once_with(expected_url, timeout=15) + else: + mock_get.assert_not_called()