#404 distgit_directive: add param ignore_missing
Closed 6 years ago by kparal. Opened 6 years ago by lbrabec.

@@ -11,6 +11,7 @@ 

  from libtaskotron import file_utils, python_utils

  from libtaskotron.ext.fedora import rpm_utils

  import libtaskotron.exceptions as exc

+ from libtaskotron.logger import log

  

  

  DOCUMENTATION = """
@@ -73,6 +74,12 @@ 

        Example: ``https://src.stg.fedoraproject.org``

      type: str

      default: https://src.fedoraproject.org

+   ignore_missing:

+     required: false

+     description: |

+       Ignore 404 error when requested files are missing in distgit

+     type: bool

+     default: False

  returns: |

    A dictionary containing following items:

  
@@ -148,6 +155,7 @@ 

          namespace = params.get('namespace', namespace or 'rpms')

          baseurl = params.get('baseurl', BASEURL)

          target_dir = params['target_dir']

+         ignore_missing = params.get('ignore_missing', False)

  

          if not python_utils.iterable(params['path']):

              raise exc.TaskotronValueError("Incorrect value type of the 'path' argument: "
@@ -178,8 +186,15 @@ 

              localpath = os.path.join(target_dir, localpath)

              file_utils.makedirs(os.path.dirname(localpath))

              url = URL_FMT.format(path=path, **format_fields)

-             output_data['downloaded_files'].append(

-                 file_utils.download(url, '.', localpath)

-             )

+             try:

+                 output_data['downloaded_files'].append(

+                     file_utils.download(url, '.', localpath)

+                 )

+             except exc.TaskotronRemoteError, e:

+                 if e.errno == 404 and ignore_missing:

+                     log.debug('File not found, ignoring: %s', url)

+                 else:

+                     raise e

+ 

  

          return output_data

file modified
+4 -1
@@ -35,7 +35,10 @@ 

  

  class TaskotronRemoteError(TaskotronError):

      '''All network and remote-server related errors'''

-     pass

+     def __init__(self, e=None, errno=None):

+         if e:

+             super(TaskotronError, self).__init__(str(e))

+         self.errno = errno

  

  

  class TaskotronRemoteTimeoutError(TaskotronRemoteError):

file modified
+1 -1
@@ -128,7 +128,7 @@ 

                      os.remove(dl_dest)

                  except OSError, e:

                      log.exception('Could not delete incomplete file: %s', dl_dest)

-             raise TaskotronRemoteError(e)

+             raise TaskotronRemoteError(e, errno=e.response.status_code)

  

      # create a symlink if the download was cached

      if cachedir:

@@ -99,8 +99,9 @@ 

  

      def test_raise(self, monkeypatch):

          """download errors should be raised"""

+         mock_response = mock.Mock(status_code=404)

          _download_mocked = mock.Mock(side_effect=(

-             requests.exceptions.RequestException('fake download failed')))

+             requests.exceptions.RequestException('fake download failed', response=mock_response)))

          monkeypatch.setattr(file_utils, '_download', _download_mocked)

  

          with pytest.raises(exc.TaskotronRemoteError):

@@ -6,6 +6,7 @@ 

  import os.path

  import pytest

  from dingus import Dingus

+ import mock

  

  from libtaskotron import file_utils

  from libtaskotron.directives import distgit_directive
@@ -137,3 +138,45 @@ 

          assert len(download_calls) == 1

          # self.ref_nvr was not touched, so if nvr parsing gets priority, the URL will not match

          assert download_calls[0][1][0] == self._get_url(self.ref_path[0])

+ 

+     def test_raises_on_404(self, monkeypatch):

+         '''Directive must raise when file is missing in distgit (404)'''

+         mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404))

+         monkeypatch.setattr(file_utils, 'download', mock_download)

+ 

+         with pytest.raises(exc.TaskotronRemoteError):

+             self.helper.process(self.ref_input, None)

+ 

+     def test_ignore_404(self, monkeypatch):

+         '''Directive ignores missing files (404) when ignore_missing is set to True'''

+         mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=404))

+         monkeypatch.setattr(file_utils, 'download', mock_download)

+ 

+         self.ref_input['ignore_missing'] = True

+ 

+         self.helper.process(self.ref_input, None)

+ 

+         mock_download.assert_called_once()

+ 

+     def test_raises_on_non_404_with_ignore(self, monkeypatch):

+         '''Directive must raise if ignore_missing is set to True but error is not 404'''

+         mock_download = mock.Mock(side_effect=exc.TaskotronRemoteError('', errno=503))

+         monkeypatch.setattr(file_utils, 'download', mock_download)

+ 

+         self.ref_input['ignore_missing'] = True

+ 

+         with pytest.raises(exc.TaskotronRemoteError):

+             self.helper.process(self.ref_input, None)

+ 

+     def test_ignore_404_on_some_files(self, monkeypatch):

+         '''Directive returns list of downloaded files, missing files are ignored'''

+         side_effects = [None, exc.TaskotronRemoteError('', errno=404)]

+         mock_download = mock.Mock(side_effect=side_effects)

+         monkeypatch.setattr(file_utils, 'download', mock_download)

+ 

+         self.ref_input['ignore_missing'] = True

+         self.ref_input['path'] = ['foo.spec', 'foo.spek']

+ 

+         downloaded = self.helper.process(self.ref_input, None)

+ 

+         assert len(downloaded) == 1

For #384

If param ignore_missing is set to True and the requested files are missing in distgit (error code 404), no error is raised.

rebased onto eb00ba2

6 years ago

1 new commit added

  • add debug output
6 years ago

Looks good! Can you please add a unit test?

rebased onto 198dd72

6 years ago

rebased onto eb00ba2

6 years ago

rebased onto 4ff5370

6 years ago

Pull-Request has been closed by kparal

6 years ago