From f82313413792ed45408aabc1b4e22fdd5704ad17 Mon Sep 17 00:00:00 2001 From: Valerij Maljulin Date: Jan 30 2020 15:42:49 +0000 Subject: Allow usage of the side tags repo JIRA: FACTORY-5691 This also reverts commit be53a785d30527719db1f2bdb6859337eb1fb510. Signed-off-by: Valerij Maljulin --- diff --git a/Dockerfile b/Dockerfile index cbd4406..89157d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ ARG cacert_url=undefined WORKDIR /src RUN dnf -y install \ + git-core \ python3-dogpile-cache \ python3-fedmsg \ python3-flask \ diff --git a/Vagrantfile b/Vagrantfile index a8fc358..044524a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -5,6 +5,7 @@ $script = <<-'SCRIPT' set -e dnf -y install \ + git-core \ postgresql-server \ postgresql-contrib \ python3-gunicorn \ diff --git a/docs/package-specific-policies.rst b/docs/package-specific-policies.rst index 73e083e..825fdc8 100644 --- a/docs/package-specific-policies.rst +++ b/docs/package-specific-policies.rst @@ -52,7 +52,7 @@ Refer to :doc:`policies` for details about each of the keys in the YAML file. .. _tolerate-invalid-gating-yaml: -Tolerate an invalid gating.yaml file +Tolerate an invalid remote rule file ------------------------------------ A gating.yaml file is considered invalid if it has an invalid syntax (yaml @@ -67,18 +67,17 @@ To skip this problem, it is possible to submit a waiver with the tool ``testcase`` equal to ``invalid-gating-yaml``. It is not necessary to have a result in Resultsdb for this testcase. -The side effect is that all the policies defined in the gating.yaml +The side effect is that all the policies defined in the remote rule file will be completely ignored by Greenwave. .. _missing-gating-yaml: -Missing gating.yaml file +Missing remote rule file ------------------------ -Missing gating.yaml file (i.e. not present in dist-git repo of the tested -package in the required revision) is just skipped and not treated as -unsatisfied requirement by default. To change this, ``required`` boolean +Missing remote rule file (i.e. not present in the configured repo) is just skipped +and not treated as unsatisfied requirement by default. To change this, ``required`` boolean attribute of ``RemoteRule`` must be set to ``true``. .. code-block:: yaml @@ -91,8 +90,7 @@ attribute of ``RemoteRule`` must be set to ``true``. rules: - !RemoteRule {required: true} -For such policy, if gating.yaml is missing, could result in the following -decision. +For such policy, missing remote rule file could result in the following decision. .. code-block:: json @@ -118,7 +116,7 @@ Tutorial - How to configure the RemoteRule If you want to add some additional policies, you can follow this tutorial. -We need to write the gating.yaml file. The one for this example will +We need to write a remote rule file. The one for this example will be this one: :: @@ -131,7 +129,7 @@ be this one: rules: - !PassingTestCaseRule {test_case_name: dist.depcheck} -*NB*. It is not possible to insert a RemoteRule inside a gating.yaml file. +*NB*. It is not possible to insert a RemoteRule inside a remote rule file. This will provoke an error. You need now to push the new file (or the changes) in your dist-git @@ -145,8 +143,8 @@ source code of your project): Now you can find in the link of the build in Koji the nvr of the build. Example: ``python-ansi2html-1.1.1-114.fc28`` -In case of a misconfigured gating.yaml you would need to repeate the -build. To avoid this it is possible to validate the gating.yaml file +In case of a misconfigured remote rule you would need to repeate the +build. To avoid this it is possible to validate the remote rule file before starting the build. To do that you can use this command (in this example we are using the Fedora Greenwave instance in production): @@ -195,28 +193,26 @@ Once you create a result in ResultsDB for that testcase (with decision will change and all the requirements will be satisfied (if everything was configured in the correct way). -If your gating.yaml file will be misconfigured, Greenwave will reply -that the gating.yaml file is wrong. If you just want to skip this check +If your remote rule file is misconfigured, Greenwave will reply +that the remote rule file is wrong. If you just want to skip this check without build again, just look at the previous section in this page. .. _fetching-gating-yaml: -How is gating.yaml file retrieved? ----------------------------------- +How is the remote rule file being retrieved? +-------------------------------------------- -The "gating.yaml" file is downloaded from a dist-git repository based on the -source URL of a specific build in Koji. - -The file is fetched from specific git commit (the revision is part of the -build's source URL). +The remote rule file (usually called ``gating.yaml``) is downloaded +from a repository based on the source URL of a specific build in Koji. +Different URLs can be set for different subject types. More specifically, Greenwave first gets the build data ``koji call getBuild $NVR``. Then it parses URL in "source" field to get namespace ("rpms" or "containers" etc.), the git commit and package name (or rather the git repository name). -The "gating.yaml" URL is constructed based on ``DIST_GIT_URL_TEMPLATE`` +For HTTP method, the remote rule URL is constructed based on ``HTTP_URL_TEMPLATE`` specified in Greenwave configuration. The URL template is something like:: - {DIST_GIT_BASE_URL}/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml + http://example.com/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml diff --git a/docs/policies.rst b/docs/policies.rst index 6db6fa8..f41c0f8 100644 --- a/docs/policies.rst +++ b/docs/policies.rst @@ -249,19 +249,39 @@ Here's an example of a RemoteRule: Once the code is pushed, Greenwave will start to check if there is a -gating.yaml file in your dist-git repo. If you didn't configure any -gating.yaml file nothing will change. +remote rule file in your repo. If you didn't configure any remote rule file +nothing will change. -Greenwave will check if a gating.yaml exists, if it does, it pulls it +Greenwave will check if a remote rule file exists, if it does, it pulls it down, loads it, and uses it to additionally evaluate the subject of the decision. -Greenwave requires these configuration parameters ``KOJI_BASE_URL``, -``DIST_GIT_BASE_URL`` and ``DIST_GIT_URL_TEMPLATE``. Here's the default -for the Fedora instance: +Greenwave requires these configuration parameters ``KOJI_BASE_URL`` and +``REMOTE_RULE_POLICIES``. + +``REMOTE_RULE_POLICIES`` is a map, where the key is the subject type. There could be +a default pattern "*" used when no subject type matched. Old parameter ``DIST_GIT_URL_TEMPLATE`` +is used if there is no default subject type, but please note that it is obsolete +and should not be used in new configurations. Each subject should contain a map of parameters +depending on a retrieval mechanism. + +Greenwave has two mechanisms to retrieve the remote rule file: ``git archive`` is +using for side tags rules and using a git front-end. ``GIT_URL`` and ``GIT_PATH_TEMPLATE`` +should be set for ``git archive`` mechanism, ``HTTP_URL_TEMPLATE`` +should be set if you are going to use a git front-end. + +Below is an example configuration where ``git archive`` is being used for "brew-build-group" +subject type and HTTP is being used for other: .. code-block:: console - DIST_GIT_BASE_URL = 'https://src.fedoraproject.org/' - DIST_GIT_URL_TEMPLATE = '{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' - KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' + REMOTE_RULE_POLICIES = { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + }, + '*': { + 'HTTP_URL_TEMPLATE': 'https://src.fedoraproject.org/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' + } + } + KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index 8bbe8b5..fff5b51 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ import logging + from flask import Flask from greenwave.api_v1 import api from greenwave.monitor import monitor_api @@ -16,12 +17,20 @@ log = logging.getLogger(__name__) def _can_use_remote_rule(config): - # Ensure that the required config settings are set - return (bool( - config.get('DIST_GIT_BASE_URL') and - config.get('DIST_GIT_URL_TEMPLATE') and - config.get('KOJI_BASE_URL')) - ) + if not config.get('KOJI_BASE_URL'): + return False + + if config.get('DIST_GIT_URL_TEMPLATE'): + return True + + if config.get('REMOTE_RULE_POLICIES'): + return all( + (conf_item.get('GIT_URL') and conf_item.get('GIT_PATH_TEMPLATE')) or + conf_item.get('HTTP_URL_TEMPLATE') + for conf_item in config.get('REMOTE_RULE_POLICIES').values() + ) + + return False def _has_remote_rule(policies): @@ -50,10 +59,10 @@ def create_app(config_obj=None): if not _can_use_remote_rule(app.config) and _has_remote_rule(app.config['policies']): raise RuntimeError( - "If you want to apply a RemoteRule" - " you need to configure 'DIST_GIT_BASE_URL', " - "'DIST_GIT_URL_TEMPLATE' and KOJI_BASE_URL in " - "your configuration." + 'If you want to apply a RemoteRule, you must have "KOJI_BASE_URL" and ' + '"DIST_GIT_URL_TEMPLATE" or "REMOTE_RULE_POLICIES" or both set in your configuration. ' + 'Each field in "REMOTE_RULE_POLICIES" map have to contain either ' + '"GIT_URL"/"GIT_PATH_TEMPLATE" or "HTTP_URL_TEMPLATE".' ) # register error handlers @@ -82,4 +91,4 @@ def healthcheck(): Returns a 200 response if the application is alive and able to serve requests. """ - return ('Health check OK', 200, [('Content-Type', 'text/plain')]) + return 'Health check OK', 200, [('Content-Type', 'text/plain')] diff --git a/greenwave/config.py b/greenwave/config.py index aa372df..70242fb 100644 --- a/greenwave/config.py +++ b/greenwave/config.py @@ -23,8 +23,23 @@ class Config(object): WAIVERDB_API_URL = 'https://waiverdb.fedoraproject.org/api/v1.0' # Remote rule configuration - DIST_GIT_BASE_URL = 'https://src.fedoraproject.org/' - DIST_GIT_URL_TEMPLATE = '{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml' + # NOTE: DIST_GIT_URL_TEMPLATE is obsolete and used here only for + # backward compatibility. They maybe removed in future versions. Use REMOTE_RULE_POLICIES['*'] + # instead + DIST_GIT_URL_TEMPLATE = \ + 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml' + REMOTE_RULE_POLICIES = { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + }, + '*': { + 'HTTP_URL_TEMPLATE': + 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml' + } + } + REMOTE_RULE_GIT_TIMEOUT = 30 + REMOTE_RULE_GIT_MAX_RETRY = 3 KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub' # Options for outbound HTTP requests made by python-requests REQUESTS_TIMEOUT = (6.1, 15) diff --git a/greenwave/policies.py b/greenwave/policies.py index 8165d56..efac2ba 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -213,7 +213,7 @@ class TestResultErrored(RuleNotSatisfied): return TestResultPassed(self.test_case_name, self.result_id) -class InvalidGatingYaml(RuleNotSatisfied): +class InvalidRemoteRuleYaml(RuleNotSatisfied): """ Remote policy parsing failed. """ @@ -236,7 +236,7 @@ class InvalidGatingYaml(RuleNotSatisfied): return None -class MissingGatingYaml(RuleNotSatisfied): +class MissingRemoteRuleYaml(RuleNotSatisfied): """ Remote policy not found in remote repository. """ @@ -433,6 +433,14 @@ class RemoteRule(Rule): 'required': SafeYAMLBool(optional=True, default=False), } + def _get_config_urls(self, rr_config, subject): + if subject in rr_config: + return rr_config[subject] + if '*' in rr_config: + return rr_config['*'] + raise RuntimeError(f'Cannot use a remote rule for {subject} subject ' + f'as it has not been configured') + def _get_sub_policies(self, policy, subject): if not subject.supports_remote_rule: return [] @@ -447,10 +455,18 @@ class RemoteRule(Rule): # if the element is actually a container and not a pkg there will be a "-container" # string at the end of the "pkg_name" and it will not match with the one in the - # gating.yaml URL + # remote rule file URL if pkg_namespace == 'containers': pkg_name = re.sub('-container$', '', pkg_name) - response = greenwave.resources.retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace) + rr_policies_conf = current_app.config.get('REMOTE_RULE_POLICIES') + if not rr_policies_conf or '*' not in rr_policies_conf: + rr_policies_conf['*'] = { + 'HTTP_URL_TEMPLATE': current_app.config['DIST_GIT_URL_TEMPLATE'] + } + cur_subject_config = self._get_config_urls(rr_policies_conf, policy.subject_type) + response = greenwave.resources.retrieve_yaml_remote_rule( + rev, pkg_name, pkg_namespace, cur_subject_config + ) if response is None: # greenwave extension file not found @@ -479,12 +495,12 @@ class RemoteRule(Rule): policies = self._get_sub_policies(policy, subject) except SafeYAMLError as e: return [ - InvalidGatingYaml(subject, 'invalid-gating-yaml', str(e)) + InvalidRemoteRuleYaml(subject, 'invalid-gating-yaml', str(e)) ] if policies is None: if self.required: - return [MissingGatingYaml(subject)] + return [MissingRemoteRuleYaml(subject)] return [] answers = [] diff --git a/greenwave/resources.py b/greenwave/resources.py index 8242d13..67bb862 100644 --- a/greenwave/resources.py +++ b/greenwave/resources.py @@ -8,6 +8,9 @@ waiverdb, etc..). import logging import re +from io import BytesIO +import tarfile +import subprocess import socket from urllib.parse import urlparse @@ -47,6 +50,7 @@ class ResultsRetriever(BaseRetriever): """ Retrieves results from cache or ResultsDB. """ + def __init__(self, **args): super().__init__(**args) self.cache = {} @@ -92,6 +96,7 @@ class WaiversRetriever(BaseRetriever): """ Retrieves waivers from WaiverDB. """ + def _retrieve_all(self, filters): if self.since: for filter_ in filters: @@ -138,8 +143,8 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): if not source: raise NoSourceException( 'Failed to retrieve SCM URL from Koji build "{}" at "{}" ' - '(expected SCM URL in "source" attribute)' - .format(nvr, koji_url)) + '(expected SCM URL in "source" attribute)'.format(nvr, koji_url) + ) url = urlparse(source) @@ -153,8 +158,8 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): if not rev: raise BadGateway( 'Failed to parse SCM URL "{}" from Koji build "{}" at "{}" ' - '(missing URL fragment with SCM revision information)' - .format(source, nvr, koji_url)) + '(missing URL fragment with SCM revision information)'.format(source, nvr, koji_url) + ) pkg_name = url.path.split('/')[-1] pkg_name = re.sub(r'\.git$', '', pkg_name) @@ -162,29 +167,77 @@ def retrieve_scm_from_koji_build(nvr, build, koji_url): @cached -def retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace): - """ Retrieve cached gating.yaml content for a given rev from the dist-git web UI. """ +def retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace, rr_config): + """ Retrieve a remote rule file content from the given repo""" + if rr_config.get('GIT_URL') and rr_config.get('GIT_PATH_TEMPLATE'): + return _retrieve_yaml_remote_rule_git_archive( + pkg_name, pkg_namespace, rr_config['GIT_URL'], rr_config['GIT_PATH_TEMPLATE'] + ) + else: + return _retrieve_yaml_remote_rule_web( + rev, pkg_name, pkg_namespace, rr_config['HTTP_URL_TEMPLATE'] + ) + + +_retrieve_remote_rule_error = 'Error occurred while retrieving a remote rule file from the repo.' + + +def _retrieve_yaml_remote_rule_web(rev, pkg_name, pkg_namespace, url_template): + """ Retrieve a remote rule file content from the git web UI. """ data = { - "DIST_GIT_BASE_URL": (current_app.config['DIST_GIT_BASE_URL'].rstrip('/') + - ('/' if pkg_namespace else '')), - "pkg_namespace": pkg_namespace, + "pkg_namespace": pkg_namespace + ('/' if pkg_namespace else ''), "pkg_name": pkg_name, "rev": rev } - url = current_app.config['DIST_GIT_URL_TEMPLATE'].format(**data) + url = url_template.format(**data) response = requests_session.request('HEAD', url) if response.status_code == 404: return None if response.status_code != 200: - raise BadGateway('Error occurred looking for gating.yaml file in the dist-git repo.') + raise BadGateway(_retrieve_remote_rule_error) - # gating.yaml found... + # remote rule file found... response = requests_session.request('GET', url) response.raise_for_status() return response.content +def _retrieve_yaml_remote_rule_git_archive(pkg_name, pkg_namespace, git_url, path_template): + """ Retrieve a remote rule file content from a git repo using git archive. """ + git_path = path_template.format(pkg_name=pkg_name, pkg_namespace=pkg_namespace) + cmd = ['git', 'archive', f'--remote={git_url}', 'master', git_path] + # Retry thrice if TimeoutExpired exception is raised + MAX_RETRY = current_app.config.get('REMOTE_RULE_GIT_MAX_RETRY', 3) + git_archive = None + for _ in range(MAX_RETRY): + try: + git_archive = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error_output = git_archive.communicate( + timeout=current_app.config.get('REMOTE_RULE_GIT_TIMEOUT', 30) + ) + break + except subprocess.TimeoutExpired: + if git_archive: + git_archive.kill() + continue + + if git_archive.returncode != 0: + error_output = error_output.decode('utf-8') + if 'path not found' in error_output: + return None + + cmd_str = ' '.join(cmd) + log.error('The following exception occurred while running "%s": %s', cmd_str, error_output) + raise BadGateway(_retrieve_remote_rule_error) + + # Convert the output to a file-like object with BytesIO, then tar can read it + # in memory rather than writing it to a file first + remote_rule_archive = tarfile.open(fileobj=BytesIO(output)) + remote_rule_content = remote_rule_archive.extractfile(git_path).read().decode('utf-8') + return remote_rule_content + + # NOTE - not cached. def retrieve_decision(greenwave_url, data): response = requests_session.post(greenwave_url, json=data) diff --git a/greenwave/tests/test_app_factory.py b/greenwave/tests/test_app_factory.py index 1915a1d..9c176b9 100644 --- a/greenwave/tests/test_app_factory.py +++ b/greenwave/tests/test_app_factory.py @@ -29,7 +29,8 @@ def test_remote_rules_misconfigured(mock_load_policies): mock_load_policies.return_value = policies config = TestingConfig() - config.DIST_GIT_BASE_URL = '' + config.DIST_GIT_URL_TEMPLATE = '' + config.REMOTE_RULE_POLICIES = {} expected_error = 'If you want to apply a RemoteRule' @@ -37,23 +38,40 @@ def test_remote_rules_misconfigured(mock_load_policies): create_app(config) +def test_can_use_remote_rule_http_fallback(): + """ Test that _can_use_remote_rule verifies the configuration properly if HTTP is used. """ + config = { + 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub', + 'DIST_GIT_URL_TEMPLATE': + 'https://dist-git.domain.local/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml' + } + assert _can_use_remote_rule(config) is True + + def test_can_use_remote_rule_http(): """ Test that _can_use_remote_rule verifies the configuration properly if HTTP is used. """ config = { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub', - 'DIST_GIT_URL_TEMPLATE': ('{DIST_GIT_BASE_URL}{pkg_namespace}/{pkg_name}/raw/{rev}/f/' - 'gating.yaml') + 'REMOTE_RULE_POLICIES': { + '*': { + 'HTTP_URL_TEMPLATE': 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/' + 'raw/{rev}/f/gating.yaml' + } + } } assert _can_use_remote_rule(config) is True @pytest.mark.parametrize('config', ( { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', + 'REMOTE_RULE_POLICIES': { + 'brew-build-group': { + 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + }, }, { - 'DIST_GIT_BASE_URL': 'https://dist-git.domain.local', 'KOJI_BASE_URL': 'https://koji.domain.local/kojihub' } )) diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index 11845d4..83032fb 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -17,8 +17,8 @@ from greenwave.policies import ( TestResultMissing, TestResultFailed, TestResultPassed, - InvalidGatingYaml, - MissingGatingYaml, + InvalidRemoteRuleYaml, + MissingRemoteRuleYaml, OnDemandPolicy ) from greenwave.resources import ResultsRetriever @@ -550,7 +550,7 @@ def test_remote_rule_malformed_yaml(tmpdir): results = DummyResultsRetriever() decision = policy.check('fedora-26', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], InvalidGatingYaml) + assert isinstance(decision[0], InvalidRemoteRuleYaml) assert decision[0].is_satisfied is False @@ -640,7 +640,7 @@ def test_remote_rule_required(): results = DummyResultsRetriever() decision = policy.check('fedora-rawhide', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], MissingGatingYaml) + assert isinstance(decision[0], MissingRemoteRuleYaml) assert not decision[0].is_satisfied assert decision[0].subject.identifier == subject.identifier @@ -1109,7 +1109,7 @@ def test_remote_rule_policy_on_demand_policy_required(): results = DummyResultsRetriever() decision = policy.check('fedora-26', subject, results) assert len(decision) == 1 - assert isinstance(decision[0], MissingGatingYaml) + assert isinstance(decision[0], MissingRemoteRuleYaml) assert not decision[0].is_satisfied assert decision[0].subject.identifier == subject.identifier diff --git a/greenwave/tests/test_retrieve_gating_yaml.py b/greenwave/tests/test_retrieve_gating_yaml.py index 222d335..75a68ea 100644 --- a/greenwave/tests/test_retrieve_gating_yaml.py +++ b/greenwave/tests/test_retrieve_gating_yaml.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ +import subprocess +import io import socket from requests.exceptions import ConnectionError, HTTPError @@ -120,7 +122,9 @@ def test_retrieve_yaml_remote_rule_no_namespace(): response = mock.MagicMock() response.status_code = 404 session.request.return_value = response - retrieve_yaml_remote_rule("deadbeaf", "pkg", "") + retrieve_yaml_remote_rule( + "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*'] + ) expected_call = mock.call( 'HEAD', 'https://src.fedoraproject.org/pkg/raw/deadbeaf/f/gating.yaml') @@ -138,7 +142,9 @@ def test_retrieve_yaml_remote_rule_connection_error(): ] with pytest.raises(HTTPError) as excinfo: - retrieve_yaml_remote_rule("deadbeaf", "pkg", "") + retrieve_yaml_remote_rule( + "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*'] + ) assert str(excinfo.value) == ( '502 Server Error: Something went terribly wrong... for url: ' @@ -146,6 +152,69 @@ def test_retrieve_yaml_remote_rule_connection_error(): ) +@mock.patch('tarfile.open') +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive(mock_subp, mock_tar): + # Make the git archive call return bytes + mock_subp.return_value.communicate.return_value = (b'tar file', '') + mock_subp.return_value.returncode = 0 + # Make the tar archive, based on the return value of git archive, return a file-like + # object representing the gating.yaml file + mock_tar.return_value.extractfile.return_value = io.BytesIO(b'some gating yaml file') + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + with app.app_context(): + gating_yaml = retrieve_yaml_remote_rule('abcdef', 'python-requests', 'rpms', rr_config) + + assert gating_yaml == 'some gating yaml file' + expected_cmd = [ + 'git', 'archive', '--remote=git://dist-git.domain.local/abc/abc.git', + 'master', 'rpms/python-requests.yaml'] + mock_subp.assert_called_once_with(expected_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + tar_file = mock_tar.call_args[1]['fileobj'].read() + assert tar_file == b'tar file' + mock_tar.return_value.extractfile.assert_called_once_with('rpms/python-requests.yaml') + + +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive_no_file(mock_subp): + # Make the git archive command return an error saying the file isn't in the repo + mock_subp.return_value.communicate.return_value = \ + (None, b'remote: fatal: path not found: xxx.yaml') + mock_subp.return_value.returncode = 1 + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + with app.app_context(): + gating_yaml = retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config) + + assert gating_yaml is None + + +@mock.patch('subprocess.Popen') +def test_retrieve_yaml_remote_rule_git_archive_error(mock_subp): + # Make the git archive command return an error + mock_subp.return_value.communicate.return_value = (None, b'remote: fatal: some error') + mock_subp.return_value.returncode = 1 + + app = greenwave.app_factory.create_app() + rr_config = { + 'GIT_URL': 'git://dist-git.domain.local/abc/abc.git', + 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml' + } + expected_error = 'Error occurred while retrieving a remote rule file from the repo.' + with pytest.raises(BadGateway, match=expected_error): + with app.app_context(): + retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config) + + @mock.patch('greenwave.resources.xmlrpc.client.ServerProxy') def test_retrieve_scm_from_koji_build_socket_error(mock_xmlrpc_client): mock_auth_server = mock_xmlrpc_client.return_value diff --git a/greenwave/tests/test_summary.py b/greenwave/tests/test_summary.py index ccda425..202f5bd 100644 --- a/greenwave/tests/test_summary.py +++ b/greenwave/tests/test_summary.py @@ -6,7 +6,7 @@ from greenwave.policies import ( TestResultFailed, TestResultMissing, TestResultMissingWaived, - InvalidGatingYaml, + InvalidRemoteRuleYaml, ) from greenwave.subjects.subject import Subject @@ -23,7 +23,7 @@ testResultMissing = TestResultMissing( testSubject, 'test', None) testResultMissingWaived = TestResultMissingWaived( testSubject, 'test', None) -testInvalidGatingYaml = InvalidGatingYaml( +testInvalidGatingYaml = InvalidRemoteRuleYaml( testSubject, 'test', 'Missing !Policy tag') diff --git a/greenwave/tests/test_waive.py b/greenwave/tests/test_waive.py index 10b8615..054dbea 100644 --- a/greenwave/tests/test_waive.py +++ b/greenwave/tests/test_waive.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ from greenwave.policies import ( - InvalidGatingYaml, + InvalidRemoteRuleYaml, TestResultMissing, TestResultFailed, ) @@ -79,7 +79,7 @@ def test_waive_missing_result(): def test_waive_invalid_gatin_yaml(): answers = [ - InvalidGatingYaml( + InvalidRemoteRuleYaml( subject=test_subject(), test_case_name='invalid-gating-yaml', details='',