+     git-core \

+         git-core \

+ Tolerate an invalid remote rule file

+ The side effect is that all the policies defined in the remote rule

+ Missing remote rule file

+ Missing remote rule file (i.e. not present in the configured repo) is just skipped

+ and not treated as unsatisfied requirement by default. To change this, ``required`` boolean

+ For such policy, missing remote rule file could result in the following decision.

+ We need to write a remote rule file. The one for this example will

+ *NB*. It is not possible to insert a RemoteRule inside a remote rule file.

+ In case of a misconfigured remote rule you would need to repeate the

+ build. To avoid this it is possible to validate the remote rule file

+ If your remote rule file is misconfigured, Greenwave will reply

+ that the remote rule file is wrong. If you just want to skip this check

+ How is the remote rule file being retrieved?

+ --------------------------------------------

+ The remote rule file (usually called ``gating.yaml``) is downloaded

+ from a repository based on the source URL of a specific build in Koji.

+ Different URLs can be set for different subject types.

+ For HTTP method, the remote rule URL is constructed based on ``HTTP_URL_TEMPLATE``

+     http://example.com/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml

+ remote rule file in your repo. If you didn't configure any remote rule file

+ nothing will change.

+ Greenwave will check if a remote rule file exists, if it does, it pulls it

+ Greenwave requires these configuration parameters ``KOJI_BASE_URL`` and

+ ``REMOTE_RULE_POLICIES``.

+ 

+ ``REMOTE_RULE_POLICIES`` is a map, where the key is the subject type. There could be

+ a default pattern "*" used when no subject type matched. Old parameter ``DIST_GIT_URL_TEMPLATE``

+ is used if there is no default subject type, but please note that it is obsolete

+ and should not be used in new configurations. Each subject should contain a map of parameters

+ depending on a retrieval mechanism.

+ 

+ Greenwave has two mechanisms to retrieve the remote rule file: ``git archive`` is

+ using for side tags rules and using a git front-end. ``GIT_URL`` and ``GIT_PATH_TEMPLATE``

+ should be set for ``git archive`` mechanism, ``HTTP_URL_TEMPLATE``

+ should be set if you are going to use a git front-end.

+ 

+ Below is an example configuration where ``git archive`` is being used for "brew-build-group"

+ subject type and HTTP is being used for other:

+     REMOTE_RULE_POLICIES = {

+         'brew-build-group': {

+             'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git',

+             'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+         },

+         '*': {

+             'HTTP_URL_TEMPLATE': 'https://src.fedoraproject.org/{pkg_namespace}/{pkg_name}/raw/{rev}/f/gating.yaml'

+         }

+     }

+     KOJI_BASE_URL = 'https://koji.fedoraproject.org/kojihub'

+ 

+     if not config.get('KOJI_BASE_URL'):

+         return False

+ 

+     if config.get('DIST_GIT_URL_TEMPLATE'):

+         return True

+ 

+     if config.get('REMOTE_RULE_POLICIES'):

+         return all(

+             (conf_item.get('GIT_URL') and conf_item.get('GIT_PATH_TEMPLATE')) or

+             conf_item.get('HTTP_URL_TEMPLATE')

+             for conf_item in config.get('REMOTE_RULE_POLICIES').values()

+         )

+ 

+     return False

+             'If you want to apply a RemoteRule, you must have "KOJI_BASE_URL" and '

+             '"DIST_GIT_URL_TEMPLATE" or "REMOTE_RULE_POLICIES" or both set in your configuration. '

+             'Each field in "REMOTE_RULE_POLICIES" map have to contain either '

+             '"GIT_URL"/"GIT_PATH_TEMPLATE" or "HTTP_URL_TEMPLATE".'

+     return 'Health check OK', 200, [('Content-Type', 'text/plain')]

+     # NOTE: DIST_GIT_URL_TEMPLATE is obsolete and used here only for

+     # backward compatibility. They maybe removed in future versions. Use REMOTE_RULE_POLICIES['*']

+     # instead

+     DIST_GIT_URL_TEMPLATE = \

+         'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml'

+     REMOTE_RULE_POLICIES = {

+         'brew-build-group': {

+             'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git',

+             'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+         },

+         '*': {

+             'HTTP_URL_TEMPLATE':

+                 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml'

+         }

+     }

+     REMOTE_RULE_GIT_TIMEOUT = 30

+     REMOTE_RULE_GIT_MAX_RETRY = 3

+ class InvalidRemoteRuleYaml(RuleNotSatisfied):

+ class MissingRemoteRuleYaml(RuleNotSatisfied):

+     def _get_config_urls(self, rr_config, subject):

+         if subject in rr_config:

+             return rr_config[subject]

+         if '*' in rr_config:

+             return rr_config['*']

+         raise RuntimeError(f'Cannot use a remote rule for {subject} subject '

+                            f'as it has not been configured')

+ 

+         # remote rule file URL

+         rr_policies_conf = current_app.config.get('REMOTE_RULE_POLICIES')

+         if not rr_policies_conf or '*' not in rr_policies_conf:

+             rr_policies_conf['*'] = {

+                 'HTTP_URL_TEMPLATE': current_app.config['DIST_GIT_URL_TEMPLATE']

+             }

+         cur_subject_config = self._get_config_urls(rr_policies_conf, policy.subject_type)

+         response = greenwave.resources.retrieve_yaml_remote_rule(

+             rev, pkg_name, pkg_namespace, cur_subject_config

+         )

+                 InvalidRemoteRuleYaml(subject, 'invalid-gating-yaml', str(e))

+                 return [MissingRemoteRuleYaml(subject)]

+ from io import BytesIO

+ import tarfile

+ import subprocess

+ 

+ 

+             '(expected SCM URL in "source" attribute)'.format(nvr, koji_url)

+         )

+             '(missing URL fragment with SCM revision information)'.format(source, nvr, koji_url)

+         )

+ def retrieve_yaml_remote_rule(rev, pkg_name, pkg_namespace, rr_config):

+     """ Retrieve a remote rule file content from the given repo"""

+     if rr_config.get('GIT_URL') and rr_config.get('GIT_PATH_TEMPLATE'):

+         return _retrieve_yaml_remote_rule_git_archive(

+             pkg_name, pkg_namespace, rr_config['GIT_URL'], rr_config['GIT_PATH_TEMPLATE']

+         )

+     else:

+         return _retrieve_yaml_remote_rule_web(

+             rev, pkg_name, pkg_namespace, rr_config['HTTP_URL_TEMPLATE']

+         )

+ 

+ 

+ _retrieve_remote_rule_error = 'Error occurred while retrieving a remote rule file from the repo.'

+ 

+ 

+ def _retrieve_yaml_remote_rule_web(rev, pkg_name, pkg_namespace, url_template):

+     """ Retrieve a remote rule file content from the git web UI. """

+         "pkg_namespace": pkg_namespace + ('/' if pkg_namespace else ''),

+     url = url_template.format(**data)

+         raise BadGateway(_retrieve_remote_rule_error)

+     # remote rule file found...

+ def _retrieve_yaml_remote_rule_git_archive(pkg_name, pkg_namespace, git_url, path_template):

+     """ Retrieve  a remote rule file content from a git repo using git archive. """

+     git_path = path_template.format(pkg_name=pkg_name, pkg_namespace=pkg_namespace)

+     cmd = ['git', 'archive', f'--remote={git_url}', 'master', git_path]

+     # Retry thrice if TimeoutExpired exception is raised

+     MAX_RETRY = current_app.config.get('REMOTE_RULE_GIT_MAX_RETRY', 3)

+     git_archive = None

+     for _ in range(MAX_RETRY):

+         try:

+             git_archive = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

+             output, error_output = git_archive.communicate(

+                 timeout=current_app.config.get('REMOTE_RULE_GIT_TIMEOUT', 30)

+             )

+             break

+         except subprocess.TimeoutExpired:

+             if git_archive:

+                 git_archive.kill()

+             continue

+ 

+     if git_archive.returncode != 0:

+         error_output = error_output.decode('utf-8')

+         if 'path not found' in error_output:

+             return None

+ 

+         cmd_str = ' '.join(cmd)

+         log.error('The following exception occurred while running "%s": %s', cmd_str, error_output)

+         raise BadGateway(_retrieve_remote_rule_error)

+ 

+     # Convert the output to a file-like object with BytesIO, then tar can read it

+     # in memory rather than writing it to a file first

+     remote_rule_archive = tarfile.open(fileobj=BytesIO(output))

+     remote_rule_content = remote_rule_archive.extractfile(git_path).read().decode('utf-8')

+     return remote_rule_content

+ 

+ 

+     config.DIST_GIT_URL_TEMPLATE = ''

+     config.REMOTE_RULE_POLICIES = {}

+ def test_can_use_remote_rule_http_fallback():

+     """ Test that _can_use_remote_rule verifies the configuration properly if HTTP is used. """

+     config = {

+         'KOJI_BASE_URL': 'https://koji.domain.local/kojihub',

+         'DIST_GIT_URL_TEMPLATE':

+             'https://dist-git.domain.local/{pkg_namespace}{pkg_name}/raw/{rev}/f/gating.yaml'

+     }

+     assert _can_use_remote_rule(config) is True

+ 

+ 

+         'REMOTE_RULE_POLICIES': {

+             '*': {

+                 'HTTP_URL_TEMPLATE': 'https://src.fedoraproject.org/{pkg_namespace}{pkg_name}/'

+                                      'raw/{rev}/f/gating.yaml'

+             }

+         }

+         'REMOTE_RULE_POLICIES': {

+             'brew-build-group': {

+                 'GIT_URL': 'git@gitlab.cee.redhat.com:devops/greenwave-policies/side-tags.git',

+                 'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+             }

+         },

+     InvalidRemoteRuleYaml,

+     MissingRemoteRuleYaml,

+                     assert isinstance(decision[0], InvalidRemoteRuleYaml)

+                 assert isinstance(decision[0], MissingRemoteRuleYaml)

+                 assert isinstance(decision[0], MissingRemoteRuleYaml)

+ import subprocess

+ import io

+             retrieve_yaml_remote_rule(

+                 "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*']

+             )

+                 retrieve_yaml_remote_rule(

+                     "deadbeaf", "pkg", "", app.config['REMOTE_RULE_POLICIES']['*']

+                 )

+ @mock.patch('tarfile.open')

+ @mock.patch('subprocess.Popen')

+ def test_retrieve_yaml_remote_rule_git_archive(mock_subp, mock_tar):

+     # Make the git archive call return bytes

+     mock_subp.return_value.communicate.return_value = (b'tar file', '')

+     mock_subp.return_value.returncode = 0

+     # Make the tar archive, based on the return value of git archive, return a file-like

+     # object representing the gating.yaml file

+     mock_tar.return_value.extractfile.return_value = io.BytesIO(b'some gating yaml file')

+ 

+     app = greenwave.app_factory.create_app()

+     rr_config = {

+         'GIT_URL': 'git://dist-git.domain.local/abc/abc.git',

+         'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+     }

+     with app.app_context():

+         gating_yaml = retrieve_yaml_remote_rule('abcdef', 'python-requests', 'rpms', rr_config)

+ 

+     assert gating_yaml == 'some gating yaml file'

+     expected_cmd = [

+         'git', 'archive', '--remote=git://dist-git.domain.local/abc/abc.git',

+         'master', 'rpms/python-requests.yaml']

+     mock_subp.assert_called_once_with(expected_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)

+     tar_file = mock_tar.call_args[1]['fileobj'].read()

+     assert tar_file == b'tar file'

+     mock_tar.return_value.extractfile.assert_called_once_with('rpms/python-requests.yaml')

+ 

+ 

+ @mock.patch('subprocess.Popen')

+ def test_retrieve_yaml_remote_rule_git_archive_no_file(mock_subp):

+     # Make the git archive command return an error saying the file isn't in the repo

+     mock_subp.return_value.communicate.return_value = \

+         (None, b'remote: fatal: path not found: xxx.yaml')

+     mock_subp.return_value.returncode = 1

+ 

+     app = greenwave.app_factory.create_app()

+     rr_config = {

+         'GIT_URL': 'git://dist-git.domain.local/abc/abc.git',

+         'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+     }

+     with app.app_context():

+         gating_yaml = retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config)

+ 

+     assert gating_yaml is None

+ 

+ 

+ @mock.patch('subprocess.Popen')

+ def test_retrieve_yaml_remote_rule_git_archive_error(mock_subp):

+     # Make the git archive command return an error

+     mock_subp.return_value.communicate.return_value = (None, b'remote: fatal: some error')

+     mock_subp.return_value.returncode = 1

+ 

+     app = greenwave.app_factory.create_app()

+     rr_config = {

+         'GIT_URL': 'git://dist-git.domain.local/abc/abc.git',

+         'GIT_PATH_TEMPLATE': '{pkg_namespace}/{pkg_name}.yaml'

+     }

+     expected_error = 'Error occurred while retrieving a remote rule file from the repo.'

+     with pytest.raises(BadGateway, match=expected_error):

+         with app.app_context():

+             retrieve_yaml_remote_rule('master', 'python-requests', 'rpms', rr_config)

+ 

+ 

+     InvalidRemoteRuleYaml,

+ testInvalidGatingYaml = InvalidRemoteRuleYaml(

+     InvalidRemoteRuleYaml,

+         InvalidRemoteRuleYaml(