From 6d27fb74b39339a49324e33c9e3335c0da2efe6c Mon Sep 17 00:00:00 2001 From: Lukas Holecek Date: Jan 24 2023 08:41:43 +0000 Subject: Allow listing custom source for RemoteRule (#122) Allows defining custom remote rule URLs in policies, for example: --- id: example_policy product_versions: [fedora-37] decision_context: bodhi_update_push_stable subject_type: koji_build rules: - !RemoteRule sources: - http://gating.example.com/gating1.yml - http://gating.example.com/gating2.yml --- diff --git a/docs/package-specific-policies.rst b/docs/package-specific-policies.rst index b7409dc..5b7a7a6 100644 --- a/docs/package-specific-policies.rst +++ b/docs/package-specific-policies.rst @@ -226,7 +226,31 @@ $NVR``. Then it parses URL in "source" field to get namespace ("rpms" or repository name). For HTTP method, the remote rule URL is constructed based on the URL template -specified in Greenwave configuration. The URL template is something like:: +specified in Greenwave configuration (``REMOTE_RULE_POLICIES`` option). The URL +template is for example:: http://example.com/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml +The URL templates in the configuration can be also overridden in policies using +``sources`` property of ``RemoteRule``. + +.. code-block:: yaml + :linenos: + + --- !Policy + product_versions: + - fedora-* + decision_context: bodhi_update_push_testing + subject_type: koji_build + rules: + - !RemoteRule + sources: + - http://gating.example.com/gating1.yml + - http://gating.example.com/gating2.yml + +Greenwave goes through list of URLs in the specified order. If a resource is +not found (returns 404 HTTP status), processing continues with the following +one. If the HTTP status is 200 it picks the resource and does not process any +following URLs. If the status is anything else or parsing the remote policy +file fails, decision will end up with "failed-fetch-gating-yaml" unsatisfied +requirement. diff --git a/docs/policies.rst b/docs/policies.rst index d6f90ed..060d71b 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -333,10 +333,10 @@ Below is an example configuration of remote rule URLs: .. code-block:: console REMOTE_RULE_POLICIES = { - 'brew-build-group': ( - 'https://git.example.com/devops/greenwave-policies/side-tags/raw/ - 'master/{subject_id}.yaml' - ), + 'brew-build-group': [ + 'https://greenwave.example.com/policies/{subject_id}.yaml', + 'https://greenwave.example.com/policies/{pkg_name}.yaml', + ], '*': ( 'https://src.fedoraproject.org/{pkg_namespace}' '{pkg_name}/raw/{rev}/f/gating.yaml' @@ -352,3 +352,6 @@ must be set. Parameter ``{subject_id}`` can also be used in URL template. If the subject identifier contains a hash starting with the ``sha256:`` prefix, this prefix would be removed. + +For details about fetching the remote policy files, see +:ref:`fetching-gating-yaml`. diff --git a/greenwave/policies.py b/greenwave/policies.py index 0b03bcb..dc9162e 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -38,10 +38,7 @@ def load_policies(policies_dir): return policies -def _remote_urls(subject): - """ - Returns generator with possible remote rule URLs. - """ +def _remote_url_templates(subject): rr_policies_conf = current_app.config.get('REMOTE_RULE_POLICIES', {}) cur_subject_urls = ( rr_policies_conf.get(subject.type) or @@ -56,7 +53,14 @@ def _remote_urls(subject): if not isinstance(cur_subject_urls, list): cur_subject_urls = [cur_subject_urls] - for current_url in cur_subject_urls: + return cur_subject_urls + + +def _remote_urls(subject, url_templates): + """ + Returns generator with possible remote rule URLs. + """ + for current_url in url_templates: url_params = {} if '{pkg_name}' in current_url or '{pkg_namespace}' in current_url or \ '{rev}' in current_url: @@ -567,6 +571,7 @@ class Rule(SafeYAMLObject): class RemoteRule(Rule): yaml_tag = '!RemoteRule' safe_yaml_attributes = { + 'sources': SafeYAMLList(str, optional=True), 'required': SafeYAMLBool(optional=True, default=False), } @@ -586,7 +591,8 @@ class RemoteRule(Rule): answers = [] try: - for remote_policies_url in _remote_urls(subject): + url_templates = self.sources or _remote_url_templates(subject) + for remote_policies_url in _remote_urls(subject, url_templates): remote_policies_urls.append(remote_policies_url) response = greenwave.resources.retrieve_yaml_remote_rule(remote_policies_url) if response is not None: diff --git a/greenwave/tests/test_rules.py b/greenwave/tests/test_rules.py index bd7b6d6..c7c94e2 100644 --- a/greenwave/tests/test_rules.py +++ b/greenwave/tests/test_rules.py @@ -94,7 +94,10 @@ def test_invalid_nvr_iden(mock_retrieve_scm_from_koji, mock_retrieve_yaml_remote subject = create_subject('koji_build', nvr) policies = Policy.safe_load_all(policy_yaml) policy = policies[0] - assert 'Koji XMLRPC fault' in str(RemoteRule._get_sub_policies(None, policy, subject)[1][0]) + rule = policy.rules[0] + sub_policies, answers = rule._get_sub_policies(policy, subject) + assert len(answers) == 1 + assert 'Koji XMLRPC fault' in answers[0].error @mock.patch('greenwave.resources.retrieve_yaml_remote_rule') @@ -338,3 +341,49 @@ def test_passing_test_case_rule_replace_using_valid_times( decision.check(subject, policies, results_retriever=results_retriever) assert [x.to_json()['type'] for x in decision.answers] == ['test-result-missing'] assert decision.answers[0].test_case_name == test_case_name + + +@mock.patch('greenwave.resources.retrieve_yaml_remote_rule') +def test_remote_rule_custom_sources(mock_retrieve_yaml_remote_rule, app): + url1 = "http://gating.example.com/gating1.yml" + url2 = "http://gating.example.com/gating2.yml" + policy_yaml = dedent(f""" + --- + id: "some_policy" + product_versions: [rhel-9000] + decision_context: bodhi_update_push_stable + subject_type: koji_build + rules: + - !RemoteRule + sources: + - {url1} + - {url2} + """) + gating_yaml = dedent(""" + --- + decision_context: bodhi_update_push_stable + rules: + - !PassingTestCaseRule {test_case_name: some_test_case} + """) + mock_retrieve_yaml_remote_rule.side_effect = lambda url: gating_yaml if url == url2 else None + + policies = Policy.safe_load_all(policy_yaml) + assert len(policies) == 1 + + nvr = 'nethack-1.2.3-1.el9000' + subject = create_subject('koji_build', nvr) + + decision = Decision('bodhi_update_push_stable', 'rhel-9000') + results_retriever = mock.MagicMock() + results_retriever.retrieve.return_value = [] + decision.check(subject, policies, results_retriever=results_retriever) + + answer1 = decision.answers[0].to_json() + assert answer1["type"] == "fetched-gating-yaml" + assert answer1["source"] == url2 + + answer2 = decision.answers[1].to_json() + assert answer2["type"] == "test-result-missing" + assert answer2["testcase"] == "some_test_case" + + assert len(decision.answers) == 2