From 4ff5370a7ec973678b642146eadd2e13e4da7f63 Mon Sep 17 00:00:00 2001
From: Lukas Brabec <lbrabec@redhat.com>
Date: Mar 09 2018 08:52:49 +0000
Subject: distgit_directive: add param ignore_missing


This is helpful when you want to download files that might or might not
be there.

Fixes https://pagure.io/taskotron/libtaskotron/issue/384

---

diff --git a/libtaskotron/directives/distgit_directive.py b/libtaskotron/directives/distgit_directive.py
index 2c3552e..f50785e 100644
--- a/libtaskotron/directives/distgit_directive.py
+++ b/libtaskotron/directives/distgit_directive.py
@@ -11,6 +11,7 @@ from libtaskotron.directives import BaseDirective
 from libtaskotron import file_utils, python_utils
 from libtaskotron.ext.fedora import rpm_utils
 import libtaskotron.exceptions as exc
+from libtaskotron.logger import log
 
 
 DOCUMENTATION = """
@@ -73,6 +74,12 @@ parameters:
       Example: ``https://src.stg.fedoraproject.org``
     type: str
     default: https://src.fedoraproject.org
+  ignore_missing:
+    required: false
+    description: |
+      Ignore 404 error when requested files are missing in distgit
+    type: bool
+    default: False
 returns: |
   A dictionary containing following items:
 
@@ -148,6 +155,7 @@ class DistGitDirective(BaseDirective):
         namespace = params.get('namespace', namespace or 'rpms')
         baseurl = params.get('baseurl', BASEURL)
         target_dir = params['target_dir']
+        ignore_missing = params.get('ignore_missing', False)
 
         if not python_utils.iterable(params['path']):
             raise exc.TaskotronValueError("Incorrect value type of the 'path' argument: "
@@ -178,8 +186,15 @@ class DistGitDirective(BaseDirective):
             localpath = os.path.join(target_dir, localpath)
             file_utils.makedirs(os.path.dirname(localpath))
             url = URL_FMT.format(path=path, **format_fields)
-            output_data['downloaded_files'].append(
-                file_utils.download(url, '.', localpath)
-            )
+            try:
+                output_data['downloaded_files'].append(
+                    file_utils.download(url, '.', localpath)
+                )
+            except exc.TaskotronRemoteError, e:
+                if e.errno == 404 and ignore_missing:
+                    log.debug('File not found, ignoring: %s', url)
+                else:
+                    raise e
+
 
         return output_data
diff --git a/libtaskotron/exceptions.py b/libtaskotron/exceptions.py
index f5dd479..7dd30c8 100644
--- a/libtaskotron/exceptions.py
+++ b/libtaskotron/exceptions.py
@@ -35,7 +35,10 @@ class TaskotronDirectiveError(TaskotronError):
 
 class TaskotronRemoteError(TaskotronError):
     '''All network and remote-server related errors'''
-    pass
+    def __init__(self, e=None, errno=None):
+        if e:
+            super(TaskotronError, self).__init__(str(e))
+        self.errno = errno
 
 
 class TaskotronRemoteTimeoutError(TaskotronRemoteError):
diff --git a/libtaskotron/file_utils.py b/libtaskotron/file_utils.py
index f5283eb..fdf8e50 100644
--- a/libtaskotron/file_utils.py
+++ b/libtaskotron/file_utils.py
@@ -128,7 +128,7 @@ def download(url, dirname, filename=None, cachedir=None):
                     os.remove(dl_dest)
                 except OSError, e:
                     log.exception('Could not delete incomplete file: %s', dl_dest)
-            raise TaskotronRemoteError(e)
+            raise TaskotronRemoteError(e, errno=e.response.status_code)
 
     # create a symlink if the download was cached
     if cachedir:
diff --git a/testing/functest_file_utils.py b/testing/functest_file_utils.py
index 9be9f19..8969fe1 100644
--- a/testing/functest_file_utils.py
+++ b/testing/functest_file_utils.py
@@ -99,8 +99,9 @@ class TestDownload():
 
     def test_raise(self, monkeypatch):
         """download errors should be raised"""
+        mock_response = mock.Mock(status_code=404)
         _download_mocked = mock.Mock(side_effect=(
-            requests.exceptions.RequestException('fake download failed')))
+            requests.exceptions.RequestException('fake download failed', response=mock_response)))
         monkeypatch.setattr(file_utils, '_download', _download_mocked)
 
         with pytest.raises(exc.TaskotronRemoteError):
diff --git a/testing/test_distgit_directive.py b/testing/test_distgit_directive.py
index e64d94f..4cc8800 100644
--- a/testing/test_distgit_directive.py
+++ b/testing/test_distgit_directive.py
@@ -6,6 +6,7 @@
 import os.path
 import pytest
 from dingus import Dingus
+import mock
 
 from libtaskotron import file_utils
 from libtaskotron.directives import distgit_directive
@@ -137,3 +138,45 @@ class TestDistGitDirective():
         assert len(download_calls) == 1
         # self.ref_nvr was not touched, so if nvr parsing gets priority, the URL will not match
         assert download_calls[0][1][0] == self._get_url(self.ref_path[0])
+
+    def test_raises_on_404(self, monkeypatch):
+        '''Directive must raise when file is missing in distgit (404)'''
+        mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404))
+        monkeypatch.setattr(file_utils, 'download', mock_download)
+
+        with pytest.raises(exc.TaskotronRemoteError):
+            self.helper.process(self.ref_input, None)
+
+    def test_ignore_404(self, monkeypatch):
+        '''Directive ignores missing files (404) when ignore_missing is set to True'''
+        mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404))
+        monkeypatch.setattr(file_utils, 'download', mock_download)
+
+        self.ref_input['ignore_missing'] = True
+
+        self.helper.process(self.ref_input, None)
+
+        mock_download.assert_called_once()
+
+    def test_raises_on_non_404_with_ignore(self, monkeypatch):
+        '''Directive must raise if ignore_missing is set to True but error is not 404'''
+        mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=503))
+        monkeypatch.setattr(file_utils, 'download', mock_download)
+
+        self.ref_input['ignore_missing'] = True
+
+        with pytest.raises(exc.TaskotronRemoteError):
+            self.helper.process(self.ref_input, None)
+
+    def test_ignore_404_on_some_files(self, monkeypatch):
+        '''Directive returns list of downloaded files, missing files are ignored'''
+        side_effects = [None, exc.TaskotronRemoteError('', errno=404)]
+        mock_download = mock.Mock(side_effect=side_effects)
+        monkeypatch.setattr(file_utils, 'download', mock_download)
+
+        self.ref_input['ignore_missing'] = True
+        self.ref_input['path'] = ['foo.spec', 'foo.spek']
+
+        downloaded = self.helper.process(self.ref_input, None)
+
+        assert len(downloaded) == 1

