#983 fix downloads w/o content-length
Merged 4 years ago by tkopecek. Opened 5 years ago by tkopecek.
tkopecek/koji issue982  into  master

file modified
+7 -12
@@ -487,26 +487,20 @@ 

          else:

              print(_("Downloading: %s") % relpath)

  

- 

+     # closing needs to be used for requests < 2.18.0

      with closing(requests.get(url, stream=True)) as response:

          # raise error if occured

          response.raise_for_status()

-         length = response.headers.get('content-length')

-         f = open(relpath, 'wb')

-         if length is None:

-             f.write(response.content)

-             length = len(response.content)

-             if not (quiet or noprogress):

-                 _download_progress(length, length)

-         else:

+         length = int(response.headers.get('content-length') or 0)

+         with open(relpath, 'wb') as f:

              l = 0

-             length = int(length)

              for chunk in response.iter_content(chunk_size=65536):

                  l += len(chunk)

                  f.write(chunk)

                  if not (quiet or noprogress):

                      _download_progress(length, l)

-             f.close()

+             if not length and not (quiet or noprogress):

+                 _download_progress(l, l)

  

      if not (quiet or noprogress):

          print('')
@@ -515,9 +509,10 @@ 

  def _download_progress(download_t, download_d):

      if download_t == 0:

          percent_done = 0.0

+         percent_done_str = "???%"

      else:

          percent_done = float(download_d) / float(download_t)

-     percent_done_str = "%3d%%" % (percent_done * 100)

+         percent_done_str = "%3d%%" % (percent_done * 100)

      data_done = _format_size(download_d)

  

      sys.stdout.write("[% -36s] % 4s % 10s\r" % ('=' * (int(percent_done * 36)), percent_done_str, data_done))

@@ -84,18 +84,18 @@ 

          response = mock.MagicMock()

          self.requests_get.return_value = response

          response.headers.get.return_value = None # content-length

-         response.content = 'abcdef'

+         response.iter_content.return_value = ['a' * 65536, 'b' * 65536]

  

          rv = download_file("http://url", self.filename)

  

          actual = self.stdout.getvalue()

-         expected = 'Downloading: %s\n[====================================] 100%%     6.00 B\r\n' % self.filename

+         expected = 'Downloading: %s\n[                                    ] ???%%  64.00 KiB\r[                                    ] ???%% 128.00 KiB\r[====================================] 100%% 128.00 KiB\r\n' % self.filename

          self.assertMultiLineEqual(actual, expected)

  

          self.requests_get.assert_called_once()

          m_open.assert_called_once()

          response.headers.get.assert_called_once()

-         response.iter_content.assert_not_called()

+         response.iter_content.assert_called_once()

          self.assertIsNone(rv)

  

  
@@ -148,7 +148,7 @@ 

          _download_progress(1024 * 1024 * 1024 * 35, 1024 * 1024 * 1024 * 30)

          _download_progress(318921, 318921)

          actual = self.stdout.getvalue()

-         expected = '[                                    ]   0%     0.00 B\r' + \

+         expected = '[                                    ] ???%     0.00 B\r' + \

                     '[                                    ]   1%   1.00 KiB\r' + \

                     '[=================                   ]  47%  11.00 MiB\r' + \

                     '[==============================      ]  85%  30.00 GiB\r' + \

file modified
+3 -6
@@ -680,14 +680,11 @@ 

              else:

                  raise koji.BuildError('unsupported file type: %s' % type)

              koji.ensuredir(os.path.dirname(localpath))

+             # closing needs to be used for requests < 2.18.0

              with closing(requests.get(remote_url, stream=True)) as response:

                  with open(localpath, 'wb') as f:

-                     length = response.headers.get('content-length')

-                     if length is None:

-                         f.write(response.content)

-                     else:

-                         for chunk in response.iter_content(chunk_size=65536):

-                             f.write(chunk)

+                     for chunk in response.iter_content(chunk_size=65536):

+                         f.write(chunk)

              if type == 'rpm':

                  # rpm, check sigmd5. It is enough, as if content is broken,

                  # rpm will fail later

rebased onto aceb773e5d779da3869698102c639fc877046621

5 years ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

4 years ago

rebased onto 98b83ef

4 years ago

Commit 7301d58 fixes this pull-request

Pull-Request has been merged by tkopecek

4 years ago

Metadata Update from @jcupova:
- Pull-request tagged with: testing-done

4 years ago