From 95bacc4e157907b2629b6b96e3c732a3be58005e Mon Sep 17 00:00:00 2001 From: Valerij Maljulin Date: May 17 2019 14:10:06 +0000 Subject: Add greenwave query to done handler Signed-off-by: Valerij Maljulin --- diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index abc5602..9d20d24 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -37,6 +37,7 @@ from module_build_service.utils import ( ) from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError from module_build_service.utils.ursine import handle_stream_collision_modules +from module_build_service.utils.greenwave import greenwave from requests.exceptions import ConnectionError from module_build_service.utils import mmd_to_str @@ -131,10 +132,11 @@ def done(config, session, msg): # This is ok.. it's a race condition we can ignore. pass - # Scratch builds stay in 'done' state, otherwise move to 'ready' + # Scratch builds stay in 'done' state if not build.scratch: - build.transition(config, state="ready") - session.commit() + if greenwave is None or greenwave.check_gating(build): + build.transition(config, state="ready") + session.commit() build_logs.stop(build) module_build_service.builder.GenericBuilder.clear_cache(build) diff --git a/module_build_service/utils/greenwave.py b/module_build_service/utils/greenwave.py index 282c49f..e5dd290 100644 --- a/module_build_service/utils/greenwave.py +++ b/module_build_service/utils/greenwave.py @@ -24,6 +24,7 @@ import requests import json +from functools import reduce from module_build_service import log, conf from module_build_service.errors import GreenwaveError @@ -34,50 +35,51 @@ class Greenwave(object): Initialize greenwave instance with config """ self.url = conf.greenwave_url - if not self.url: - raise GreenwaveError("No Greenwave URL set") self._decision_context = conf.greenwave_decision_context if not self.decision_context: raise GreenwaveError("No Greenwave decision context set") self._subj_type = conf.greenwave_subject_type self._gw_timeout = conf.greenwave_timeout - def query_decision(self, build, prod_version): + def _greenwave_query(self, query_type, payload=None): """ - Query decision to greenwave - :param build: build object - :type build: module_build_service.models.ModuleBuild - :param prod_version: The product version string used for querying WaiverDB - :type prod_version: str + Make a query to greenwave + :param query_type: will be part of url + :type query_type: str + :param payload: request payload used in 'decision' query + :type payload: str :return: response :rtype: dict """ - payload = { - "decision_context": self.decision_context, - "product_version": prod_version, - "subject_type": self.subject_type, - "subject_identifier": build.nvr_string - } - url = "{0}/decision".format(self.url) - headers = {"Content-Type": "application/json"} + query_func = requests.post if payload else requests.get + kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout} + + if payload: + kwargs["headers"] = {"Content-Type": "application/json"} + kwargs["data"] = payload + try: - response = requests.post( - url=url, headers=headers, data=json.dumps(payload), timeout=self.timeout) + response = query_func(**kwargs) except requests.exceptions.Timeout: raise GreenwaveError("Greenwave request timed out") except Exception as exc: - log.exception(str(exc)) - raise GreenwaveError("Greenwave request error") + error_message = "Unspecified greenwave request error" \ + '(original exception was: "{0}")'.format(str(exc)) + log.exception(error_message) + raise GreenwaveError(error_message) try: resp_json = response.json() except ValueError: log.debug("Greenwave response content (status {0}): {1}".format( - response.status_code, response.text)) + response.status_code, response.text + )) raise GreenwaveError("Greenwave returned invalid JSON.") - log.debug('Query to Greenwave result: status=%d, content="%s"', - (response.status_code, resp_json)) + log.debug( + 'Query to Greenwave (%s) result: status=%d, content="%s"', + (kwargs["url"], response.status_code, resp_json) + ) if response.status_code == 200: return resp_json @@ -87,7 +89,88 @@ class Greenwave(object): except KeyError: err_msg = response.text raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format( - response.status_code, err_msg)) + response.status_code, err_msg + )) + + def query_decision(self, build, prod_version): + """ + Query decision to greenwave + :param build: build object + :type build: module_build_service.models.ModuleBuild + :param prod_version: The product version string used for querying WaiverDB + :type prod_version: str + :return: response + :rtype: dict + """ + payload = { + "decision_context": self.decision_context, + "product_version": prod_version, + "subject_type": self.subject_type, + "subject_identifier": build.nvr_string + } + return self._greenwave_query('decision', json.dumps(payload)) + + def query_policies(self, return_all=False): + """ + Query policies to greenwave + :param return_all: Return all policies, if False select by subject_type and decision_context + :type return_all: bool + :return: response + :rtype: dict + """ + response = self._greenwave_query('policies') + + if return_all: + return response + + try: + selective_resp = { + "policies": [ + pol for pol in response["policies"] + if pol["decision_context"] == self.decision_context + and pol["subject_type"] == self.subject_type + ] + } + except KeyError: + log.exception("Incorrect greenwave response (Mandatory key is missing)") + raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)") + return selective_resp + + def get_product_versions(self): + """ + Return a set of product versions according to decision_context and subject_type + :return: product versions + :rtype: set + """ + return reduce( + lambda old, new: old.union(new), + [pol["product_versions"] for pol in self.query_policies()["policies"]], + set() + ) + + def check_gating(self, build): + """ + Query decision to greenwave + :param build: build object + :type build: module_build_service.models.ModuleBuild + :return: True if at least one GW response contains policies_satisfied set to true + :rtype: bool + """ + try: + versions = self.get_product_versions() + except GreenwaveError: + log.warning('An error occured while getting a product versions') + return False + + for ver in versions: + try: + if self.query_decision(build, ver)["policies_satisfied"]: + # at least one positive result is enough + return True + except (KeyError, GreenwaveError) as exc: + log.warning('Incorrect greenwave result "%s", ignoring', str(exc)) + + return False @property def url(self): @@ -96,8 +179,9 @@ class Greenwave(object): @url.setter def url(self, value): value = value.rstrip("/") - if value: - self._url = value + if not value: + raise GreenwaveError("No Greenwave URL set") + self._url = value @property def decision_context(self): @@ -114,3 +198,10 @@ class Greenwave(object): @timeout.setter def timeout(self, value): self._gw_timeout = value + + +try: + greenwave = Greenwave() +except GreenwaveError: + log.warning('Greenwave is not configured or configured improperly') + greenwave = None diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 6191695..96942a7 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -406,9 +406,12 @@ class TestBuild: pass @pytest.mark.parametrize("mmd_version", [1, 2]) + @patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True) @patch("module_build_service.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") - def test_submit_build(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc, mmd_version): + def test_submit_build( + self, mocked_scm, mocked_get_user, mocked_greenwave, conf_system, dbg, hmsc, mmd_version + ): """ Tests the build of testmodule.yaml using FakeModuleBuilder which succeeds everytime. @@ -481,12 +484,17 @@ class TestBuild: assert module_build.module_builds_trace[4].state == models.BUILD_STATES["ready"] assert len(module_build.module_builds_trace) == 5 + @pytest.mark.parametrize("gating_result", (True, False)) + @patch("module_build_service.utils.greenwave.Greenwave.check_gating") @patch("module_build_service.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") - def test_submit_build_no_components(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc): + def test_submit_build_no_components( + self, mocked_scm, mocked_get_user, mocked_greenwave, conf_system, dbg, hmsc, gating_result + ): """ Tests the build of a module with no components """ + mocked_greenwave.return_value = gating_result FakeSCM( mocked_scm, "python3", @@ -512,7 +520,10 @@ class TestBuild: # Make sure no component builds were registered assert len(module_build.component_builds) == 0 # Make sure the build is done - assert module_build.state == models.BUILD_STATES["ready"] + if gating_result: + assert module_build.state == models.BUILD_STATES["ready"] + else: + assert module_build.state == models.BUILD_STATES["done"] @patch( "module_build_service.config.Config.check_for_eol", @@ -1412,10 +1423,11 @@ class TestBuild: models.BUILD_STATES["ready"], ] + @patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True) @patch("module_build_service.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") def test_submit_build_resume_init_fail( - self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc + self, mocked_scm, mocked_get_user, mock_greenwave, conf_system, dbg, hmsc ): """ Tests that resuming the build fails when the build is in init state @@ -1654,10 +1666,11 @@ class TestBuild: module = db.session.query(models.ModuleBuild).get(module_build_id) assert module.state == models.BUILD_STATES["build"] + @patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True) @patch("module_build_service.auth.get_user", return_value=user) @patch("module_build_service.scm.SCM") def test_submit_br_metadata_only_module( - self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc + self, mocked_scm, mocked_get_user, mock_greenwave, conf_system, dbg, hmsc ): """ Test that when a build is submitted with a buildrequire without a Koji tag, diff --git a/tests/test_utils/test_greenwave.py b/tests/test_utils/test_greenwave.py index f41a65c..92165a5 100644 --- a/tests/test_utils/test_greenwave.py +++ b/tests/test_utils/test_greenwave.py @@ -23,13 +23,14 @@ import json from mock import patch, Mock -import module_build_service.utils.greenwave +import pytest +from module_build_service.utils.greenwave import greenwave from tests import make_module class TestGreenwaveQuery(): @patch("module_build_service.utils.greenwave.requests") - def test_greenwave_decision(self, mock_requests): + def test_greenwave_query_decision(self, mock_requests): resp_status = 200 resp_content = { "applicable_policies": ["osci_compose_modules"], @@ -56,8 +57,7 @@ class TestGreenwaveQuery(): fake_build = make_module("pkg:0.1:1:c1", requires_list={"platform": "el8"}) - gw = module_build_service.utils.greenwave.Greenwave() - got_response = gw.query_decision(fake_build, prod_version="xxxx-8") + got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8") assert got_response == resp_content assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == { @@ -68,3 +68,112 @@ class TestGreenwaveQuery(): "Content-Type": "application/json"} assert mock_requests.post.call_args_list[0][1]["url"] == \ "https://greenwave.example.local/api/v1.0/decision" + + @pytest.mark.parametrize("return_all", (False, True)) + @patch("module_build_service.utils.greenwave.requests") + def test_greenwave_query_policies(self, mock_requests, return_all): + resp_status = 200 + resp_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver2"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "decision_context_2", + "product_versions": ["ver4"], + "rules": [], + "subject_type": "subject_type_2" + } + ] + } + selected_policies = {"policies": resp_content["policies"][:-1]} + + response = Mock() + response.json.return_value = resp_content + response.status_code = resp_status + mock_requests.get.return_value = response + + got_response = greenwave.query_policies(return_all) + + if return_all: + assert got_response == resp_content + else: + assert got_response == selected_policies + assert mock_requests.get.call_args_list[0][1]["url"] == \ + "https://greenwave.example.local/api/v1.0/policies" + + @patch("module_build_service.utils.greenwave.requests") + def test_greenwave_get_product_versions(self, mock_requests): + resp_status = 200 + resp_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver2"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "decision_context_2", + "product_versions": ["ver4"], + "rules": [], + "subject_type": "subject_type_2" + } + ] + } + expected_versions = {"ver1", "ver2", "ver3"} + + response = Mock() + response.json.return_value = resp_content + response.status_code = resp_status + mock_requests.get.return_value = response + + versions_set = greenwave.get_product_versions() + + assert versions_set == expected_versions + assert mock_requests.get.call_args_list[0][1]["url"] == \ + "https://greenwave.example.local/api/v1.0/policies" + + @pytest.mark.parametrize("policies_satisfied", (True, False)) + @patch("module_build_service.utils.greenwave.requests") + def test_greenwave_check_gating(self, mock_requests, policies_satisfied): + resp_status = 200 + policies_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + } + ] + } + + responses = [Mock() for i in range(3)] + for r in responses: + r.status_code = resp_status + responses[0].json.return_value = policies_content + responses[1].json.return_value = {"policies_satisfied": False} + responses[2].json.return_value = {"policies_satisfied": policies_satisfied} + mock_requests.get.return_value = responses[0] + mock_requests.post.side_effect = responses[1:] + + fake_build = make_module("pkg:0.1:1:c1", requires_list={"platform": "el8"}) + result = greenwave.check_gating(fake_build) + + assert result == policies_satisfied