From a8b90c083b1292a69338db8087fee52526fc02b9 Mon Sep 17 00:00:00 2001 From: mprahl Date: Jan 16 2020 19:39:45 +0000 Subject: Move utils/greenwave.py to scheduler/greenwave.py --- diff --git a/module_build_service/scheduler/batches.py b/module_build_service/scheduler/batches.py index 43140ff..59179c2 100644 --- a/module_build_service/scheduler/batches.py +++ b/module_build_service/scheduler/batches.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT -import concurrent.futures import threading +import concurrent.futures from module_build_service import conf, log, models from module_build_service.db_session import db_session diff --git a/module_build_service/scheduler/greenwave.py b/module_build_service/scheduler/greenwave.py new file mode 100644 index 0000000..1d2f52f --- /dev/null +++ b/module_build_service/scheduler/greenwave.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +from functools import reduce +import json + +import requests + +from module_build_service import log, conf +from module_build_service.errors import GreenwaveError + + +class Greenwave(object): + def __init__(self): + """ + Initialize greenwave instance with config + """ + self.url = conf.greenwave_url + self._decision_context = conf.greenwave_decision_context + if not self.decision_context: + raise RuntimeError("No Greenwave decision context set") + self._subj_type = conf.greenwave_subject_type + self._gw_timeout = conf.greenwave_timeout + self.error_occurred = False + + def _greenwave_query(self, query_type, payload=None): + """ + Make a query to greenwave + :param query_type: will be part of url + :type query_type: str + :param payload: request payload used in 'decision' query + :type payload: str + :return: response + :rtype: dict + """ + query_func = requests.post if payload else requests.get + kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout} + + if payload: + kwargs["headers"] = {"Content-Type": "application/json"} + kwargs["data"] = payload + + try: + response = query_func(**kwargs) + except requests.exceptions.Timeout: + raise GreenwaveError("Greenwave request timed out") + except Exception as exc: + error_message = "Unspecified greenwave request error " \ + '(original exception was: "{0}")'.format(str(exc)) + log.exception(error_message) + raise GreenwaveError(error_message) + + try: + resp_json = response.json() + except ValueError: + log.debug("Greenwave response content (status {0}): {1}".format( + response.status_code, response.text + )) + raise GreenwaveError("Greenwave returned invalid JSON.") + + log.debug( + 'Query to Greenwave (%s) result: status=%d, content="%s"', + kwargs["url"], response.status_code, resp_json + ) + + if response.status_code == 200: + return resp_json + + try: + err_msg = resp_json["message"] + except KeyError: + err_msg = response.text + raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format( + response.status_code, err_msg + )) + + def query_decision(self, build, prod_version): + """ + Query decision to greenwave + :param build: build object + :type build: module_build_service.models.ModuleBuild + :param prod_version: The product version string used for querying WaiverDB + :type prod_version: str + :return: response + :rtype: dict + """ + payload = { + "decision_context": self.decision_context, + "product_version": prod_version, + "subject_type": self.subject_type, + "subject_identifier": build.nvr_string + } + return self._greenwave_query('decision', json.dumps(payload)) + + def query_policies(self, return_all=False): + """ + Query policies to greenwave + :param return_all: Return all policies, if False select by subject_type and decision_context + :type return_all: bool + :return: response + :rtype: dict + """ + response = self._greenwave_query('policies') + + if return_all: + return response + + try: + selective_resp = { + "policies": [ + pol for pol in response["policies"] + if pol["decision_context"] == self.decision_context + and pol["subject_type"] == self.subject_type + ] + } + except KeyError: + log.exception("Incorrect greenwave response (Mandatory key is missing)") + raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)") + return selective_resp + + def get_product_versions(self): + """ + Return a set of product versions according to decision_context and subject_type + :return: product versions + :rtype: set + """ + return reduce( + lambda old, new: old.union(new), + [pol["product_versions"] for pol in self.query_policies()["policies"]], + set() + ) + + def check_gating(self, build): + """ + Query decision to greenwave + :param build: build object + :type build: module_build_service.models.ModuleBuild + :return: True if at least one GW response contains policies_satisfied set to true + :rtype: bool + """ + self.error_occurred = False + try: + versions = self.get_product_versions() + except GreenwaveError: + log.warning('An error occured while getting a product versions') + self.error_occurred = True + return False + + for ver in versions: + try: + if self.query_decision(build, ver)["policies_satisfied"]: + # at least one positive result is enough + return True + except (KeyError, GreenwaveError) as exc: + self.error_occurred = True + log.warning('Incorrect greenwave result "%s", ignoring', str(exc)) + + return False + + @property + def url(self): + return self._url + + @url.setter + def url(self, value): + value = value.rstrip("/") + if not value: + raise RuntimeError("No Greenwave URL set") + self._url = value + + @property + def decision_context(self): + return self._decision_context + + @property + def subject_type(self): + return self._subj_type + + @property + def timeout(self): + return self._gw_timeout + + @timeout.setter + def timeout(self, value): + self._gw_timeout = value + + +try: + greenwave = Greenwave() +except RuntimeError: + log.warning('Greenwave is not configured or configured improperly') + greenwave = None diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 510b5b4..4c94bb6 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -17,9 +17,9 @@ from module_build_service.utils import ( from module_build_service.db_session import db_session from module_build_service.builder import GenericBuilder from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError -from module_build_service.utils.greenwave import greenwave from module_build_service.scheduler.default_modules import ( add_default_modules, handle_collisions_with_base_module_rpms) +from module_build_service.scheduler.greenwave import greenwave from module_build_service.utils.submit import format_mmd from module_build_service.scheduler import events from module_build_service.utils.ursine import handle_stream_collision_modules diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index 6ac3a7c..33d2a67 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -12,13 +12,13 @@ import module_build_service.scheduler.consumer from module_build_service import celery_app, conf, models, log from module_build_service.builder import GenericBuilder from module_build_service.common.koji import get_session -from module_build_service.utils.greenwave import greenwave from module_build_service.db_session import db_session from module_build_service.scheduler.batches import ( at_concurrent_component_threshold, start_next_batch_build, ) from module_build_service.scheduler.consumer import ON_MODULE_CHANGE_HANDLERS +from module_build_service.scheduler.greenwave import greenwave from module_build_service.scheduler.handlers.components import build_task_finalize from module_build_service.scheduler.handlers.tags import tagged diff --git a/module_build_service/utils/greenwave.py b/module_build_service/utils/greenwave.py deleted file mode 100644 index 66544cd..0000000 --- a/module_build_service/utils/greenwave.py +++ /dev/null @@ -1,189 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import requests -import json -from functools import reduce -from module_build_service import log, conf -from module_build_service.errors import GreenwaveError - - -class Greenwave(object): - def __init__(self): - """ - Initialize greenwave instance with config - """ - self.url = conf.greenwave_url - self._decision_context = conf.greenwave_decision_context - if not self.decision_context: - raise RuntimeError("No Greenwave decision context set") - self._subj_type = conf.greenwave_subject_type - self._gw_timeout = conf.greenwave_timeout - self.error_occurred = False - - def _greenwave_query(self, query_type, payload=None): - """ - Make a query to greenwave - :param query_type: will be part of url - :type query_type: str - :param payload: request payload used in 'decision' query - :type payload: str - :return: response - :rtype: dict - """ - query_func = requests.post if payload else requests.get - kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout} - - if payload: - kwargs["headers"] = {"Content-Type": "application/json"} - kwargs["data"] = payload - - try: - response = query_func(**kwargs) - except requests.exceptions.Timeout: - raise GreenwaveError("Greenwave request timed out") - except Exception as exc: - error_message = "Unspecified greenwave request error " \ - '(original exception was: "{0}")'.format(str(exc)) - log.exception(error_message) - raise GreenwaveError(error_message) - - try: - resp_json = response.json() - except ValueError: - log.debug("Greenwave response content (status {0}): {1}".format( - response.status_code, response.text - )) - raise GreenwaveError("Greenwave returned invalid JSON.") - - log.debug( - 'Query to Greenwave (%s) result: status=%d, content="%s"', - kwargs["url"], response.status_code, resp_json - ) - - if response.status_code == 200: - return resp_json - - try: - err_msg = resp_json["message"] - except KeyError: - err_msg = response.text - raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format( - response.status_code, err_msg - )) - - def query_decision(self, build, prod_version): - """ - Query decision to greenwave - :param build: build object - :type build: module_build_service.models.ModuleBuild - :param prod_version: The product version string used for querying WaiverDB - :type prod_version: str - :return: response - :rtype: dict - """ - payload = { - "decision_context": self.decision_context, - "product_version": prod_version, - "subject_type": self.subject_type, - "subject_identifier": build.nvr_string - } - return self._greenwave_query('decision', json.dumps(payload)) - - def query_policies(self, return_all=False): - """ - Query policies to greenwave - :param return_all: Return all policies, if False select by subject_type and decision_context - :type return_all: bool - :return: response - :rtype: dict - """ - response = self._greenwave_query('policies') - - if return_all: - return response - - try: - selective_resp = { - "policies": [ - pol for pol in response["policies"] - if pol["decision_context"] == self.decision_context - and pol["subject_type"] == self.subject_type - ] - } - except KeyError: - log.exception("Incorrect greenwave response (Mandatory key is missing)") - raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)") - return selective_resp - - def get_product_versions(self): - """ - Return a set of product versions according to decision_context and subject_type - :return: product versions - :rtype: set - """ - return reduce( - lambda old, new: old.union(new), - [pol["product_versions"] for pol in self.query_policies()["policies"]], - set() - ) - - def check_gating(self, build): - """ - Query decision to greenwave - :param build: build object - :type build: module_build_service.models.ModuleBuild - :return: True if at least one GW response contains policies_satisfied set to true - :rtype: bool - """ - self.error_occurred = False - try: - versions = self.get_product_versions() - except GreenwaveError: - log.warning('An error occured while getting a product versions') - self.error_occurred = True - return False - - for ver in versions: - try: - if self.query_decision(build, ver)["policies_satisfied"]: - # at least one positive result is enough - return True - except (KeyError, GreenwaveError) as exc: - self.error_occurred = True - log.warning('Incorrect greenwave result "%s", ignoring', str(exc)) - - return False - - @property - def url(self): - return self._url - - @url.setter - def url(self, value): - value = value.rstrip("/") - if not value: - raise RuntimeError("No Greenwave URL set") - self._url = value - - @property - def decision_context(self): - return self._decision_context - - @property - def subject_type(self): - return self._subj_type - - @property - def timeout(self): - return self._gw_timeout - - @timeout.setter - def timeout(self, value): - self._gw_timeout = value - - -try: - greenwave = Greenwave() -except RuntimeError: - log.warning('Greenwave is not configured or configured improperly') - greenwave = None diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 4342a11..dac994a 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -452,7 +452,7 @@ class TestBuild(BaseTestBuild): FakeModuleBuilder.on_get_task_info_cb = on_get_task_info_cb self.p_check_gating = patch( - "module_build_service.utils.greenwave.Greenwave.check_gating", + "module_build_service.scheduler.greenwave.Greenwave.check_gating", return_value=True) self.mock_check_gating = self.p_check_gating.start() diff --git a/tests/test_scheduler/test_greenwave.py b/tests/test_scheduler/test_greenwave.py new file mode 100644 index 0000000..6bd0d68 --- /dev/null +++ b/tests/test_scheduler/test_greenwave.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import json + +from mock import patch, Mock +import pytest + +from module_build_service.scheduler.greenwave import greenwave +from tests import clean_database, make_module_in_db + + +class TestGreenwaveQuery(): + + def setup_method(self, method): + clean_database() + + @patch("module_build_service.scheduler.greenwave.requests") + def test_greenwave_query_decision(self, mock_requests): + resp_status = 200 + resp_content = { + "applicable_policies": ["osci_compose_modules"], + "policies_satisfied": True, + "satisfied_requirements": [ + { + "result_id": 7336633, + "testcase": "test-ci.test-module.tier1", + "type": "test-result-passed" + }, + { + "result_id": 7336650, + "testcase": "test-ci.test-module.tier2", + "type": "test-result-passed" + } + ], + "summary": "All required tests passed", + "unsatisfied_requirements": [] + } + response = Mock() + response.json.return_value = resp_content + response.status_code = resp_status + mock_requests.post.return_value = response + + fake_build = make_module_in_db( + "pkg:0.1:1:c1", [{ + "requires": {"platform": ["el8"]}, + "buildrequires": {"platform": ["el8"]}, + }], + ) + got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8") + + assert got_response == resp_content + assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == { + "decision_context": "test_dec_context", + "product_version": "xxxx-8", "subject_type": "some-module", + "subject_identifier": "pkg-0.1-1.c1"} + assert mock_requests.post.call_args_list[0][1]["headers"] == { + "Content-Type": "application/json"} + assert mock_requests.post.call_args_list[0][1]["url"] == \ + "https://greenwave.example.local/api/v1.0/decision" + + @pytest.mark.parametrize("return_all", (False, True)) + @patch("module_build_service.scheduler.greenwave.requests") + def test_greenwave_query_policies(self, mock_requests, return_all): + resp_status = 200 + resp_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver2"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "decision_context_2", + "product_versions": ["ver4"], + "rules": [], + "subject_type": "subject_type_2" + } + ] + } + selected_policies = {"policies": resp_content["policies"][:-1]} + + response = Mock() + response.json.return_value = resp_content + response.status_code = resp_status + mock_requests.get.return_value = response + + got_response = greenwave.query_policies(return_all) + + if return_all: + assert got_response == resp_content + else: + assert got_response == selected_policies + assert mock_requests.get.call_args_list[0][1]["url"] == \ + "https://greenwave.example.local/api/v1.0/policies" + + @patch("module_build_service.scheduler.greenwave.requests") + def test_greenwave_get_product_versions(self, mock_requests): + resp_status = 200 + resp_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver2"], + "rules": [], + "subject_type": "some-module" + }, + { + "decision_context": "decision_context_2", + "product_versions": ["ver4"], + "rules": [], + "subject_type": "subject_type_2" + } + ] + } + expected_versions = {"ver1", "ver2", "ver3"} + + response = Mock() + response.json.return_value = resp_content + response.status_code = resp_status + mock_requests.get.return_value = response + + versions_set = greenwave.get_product_versions() + + assert versions_set == expected_versions + assert mock_requests.get.call_args_list[0][1]["url"] == \ + "https://greenwave.example.local/api/v1.0/policies" + + @pytest.mark.parametrize("policies_satisfied", (True, False)) + @patch("module_build_service.scheduler.greenwave.requests") + def test_greenwave_check_gating(self, mock_requests, policies_satisfied): + resp_status = 200 + policies_content = { + "policies": [ + { + "decision_context": "test_dec_context", + "product_versions": ["ver1", "ver3"], + "rules": [], + "subject_type": "some-module" + } + ] + } + + responses = [Mock() for i in range(3)] + for r in responses: + r.status_code = resp_status + responses[0].json.return_value = policies_content + responses[1].json.return_value = {"policies_satisfied": False} + responses[2].json.return_value = {"policies_satisfied": policies_satisfied} + mock_requests.get.return_value = responses[0] + mock_requests.post.side_effect = responses[1:] + + fake_build = make_module_in_db( + "pkg:0.1:1:c1", [{ + "requires": {"platform": ["el8"]}, + "buildrequires": {"platform": ["el8"]}, + }], + ) + result = greenwave.check_gating(fake_build) + + assert result == policies_satisfied diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index e9851ee..92a657c 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -556,7 +556,7 @@ class TestPoller: expected_tagged_calls, any_order=True) @pytest.mark.parametrize("greenwave_result", [True, False]) - @patch("module_build_service.utils.greenwave.Greenwave.check_gating") + @patch("module_build_service.scheduler.greenwave.Greenwave.check_gating") def test_poll_greenwave(self, mock_gw, create_builder, dbg, greenwave_result): module_build1 = models.ModuleBuild.get_by_id(db_session, 1) diff --git a/tests/test_utils/test_greenwave.py b/tests/test_utils/test_greenwave.py deleted file mode 100644 index f1a3af1..0000000 --- a/tests/test_utils/test_greenwave.py +++ /dev/null @@ -1,171 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -import json -from mock import patch, Mock -import pytest -from module_build_service.utils.greenwave import greenwave -from tests import clean_database, make_module_in_db - - -class TestGreenwaveQuery(): - - def setup_method(self, method): - clean_database() - - @patch("module_build_service.utils.greenwave.requests") - def test_greenwave_query_decision(self, mock_requests): - resp_status = 200 - resp_content = { - "applicable_policies": ["osci_compose_modules"], - "policies_satisfied": True, - "satisfied_requirements": [ - { - "result_id": 7336633, - "testcase": "test-ci.test-module.tier1", - "type": "test-result-passed" - }, - { - "result_id": 7336650, - "testcase": "test-ci.test-module.tier2", - "type": "test-result-passed" - } - ], - "summary": "All required tests passed", - "unsatisfied_requirements": [] - } - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.post.return_value = response - - fake_build = make_module_in_db( - "pkg:0.1:1:c1", [{ - "requires": {"platform": ["el8"]}, - "buildrequires": {"platform": ["el8"]}, - }], - ) - got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8") - - assert got_response == resp_content - assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == { - "decision_context": "test_dec_context", - "product_version": "xxxx-8", "subject_type": "some-module", - "subject_identifier": "pkg-0.1-1.c1"} - assert mock_requests.post.call_args_list[0][1]["headers"] == { - "Content-Type": "application/json"} - assert mock_requests.post.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/decision" - - @pytest.mark.parametrize("return_all", (False, True)) - @patch("module_build_service.utils.greenwave.requests") - def test_greenwave_query_policies(self, mock_requests, return_all): - resp_status = 200 - resp_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver2"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "decision_context_2", - "product_versions": ["ver4"], - "rules": [], - "subject_type": "subject_type_2" - } - ] - } - selected_policies = {"policies": resp_content["policies"][:-1]} - - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.get.return_value = response - - got_response = greenwave.query_policies(return_all) - - if return_all: - assert got_response == resp_content - else: - assert got_response == selected_policies - assert mock_requests.get.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/policies" - - @patch("module_build_service.utils.greenwave.requests") - def test_greenwave_get_product_versions(self, mock_requests): - resp_status = 200 - resp_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver2"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "decision_context_2", - "product_versions": ["ver4"], - "rules": [], - "subject_type": "subject_type_2" - } - ] - } - expected_versions = {"ver1", "ver2", "ver3"} - - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.get.return_value = response - - versions_set = greenwave.get_product_versions() - - assert versions_set == expected_versions - assert mock_requests.get.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/policies" - - @pytest.mark.parametrize("policies_satisfied", (True, False)) - @patch("module_build_service.utils.greenwave.requests") - def test_greenwave_check_gating(self, mock_requests, policies_satisfied): - resp_status = 200 - policies_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - } - ] - } - - responses = [Mock() for i in range(3)] - for r in responses: - r.status_code = resp_status - responses[0].json.return_value = policies_content - responses[1].json.return_value = {"policies_satisfied": False} - responses[2].json.return_value = {"policies_satisfied": policies_satisfied} - mock_requests.get.return_value = responses[0] - mock_requests.post.side_effect = responses[1:] - - fake_build = make_module_in_db( - "pkg:0.1:1:c1", [{ - "requires": {"platform": ["el8"]}, - "buildrequires": {"platform": ["el8"]}, - }], - ) - result = greenwave.check_gating(fake_build) - - assert result == policies_satisfied