From d7f2ed76c2114f70b35194899c7478d9e9ba2ab4 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Apr 11 2017 19:48:56 +0000
Subject: Use python-requests streaming for RPM downloads


Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

diff --git a/ChangeLog b/ChangeLog
index 5fd91d6..74dde53 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2017-04-11  Patrick Uiterwijk  <puiterwijk@redhat.com>
+
+	* src/bridge.py: Use requests instead of urlgrabber
+
 2017-03-14  Patrick Uiterwijk  <puiterwijk@redhat.com>
 
 	* src/server.py: Add version to generated container signatures
diff --git a/README b/README
index 71b804b..fbb5a2f 100644
--- a/README
+++ b/README
@@ -5,7 +5,7 @@ Required packages
 -----------------
 pexpect
 python-fedora
-python-urlgrabber
+python-requests
 python-sqlalchemy >= 0.5
 
 python-nss >= 0.11
diff --git a/src/bridge.py b/src/bridge.py
index e19e5ae..a90da87 100644
--- a/src/bridge.py
+++ b/src/bridge.py
@@ -38,7 +38,7 @@ import nss.error
 import nss.io
 import nss.nss
 import nss.ssl
-import urlgrabber.grabber
+import requests
 
 import double_tls
 import settings
@@ -154,26 +154,19 @@ def fas_user_is_in_group(config, user_name, group_name):
             fedora.client.FedoraServiceError), e:
         raise BridgeError('Error communicating with FAS: %s' % str(e))
 
-def urlgrabber_open(url):
+def urlopen(url):
     '''Open url.
 
     Return (file, file size).  Raise ForwardingError.
 
     '''
-    fd = urlgrabber.grabber.urlopen(url)
+    r = requests.get(url, stream=True)
     try:
-        try:
-            size = fd.size # urlgrabber using pycurl
-        except AttributeError:
-            try:
-                size = int(fd.hdr['Content-Length']) # Older urlgrabber
-            except KeyError:
-                raise ForwardingError('Content-Length not returned for %s' %
-                                      url)
-    except:
-        fd.close()
-        raise
-    return (fd, size)
+        size = int(r.headers['content-length'])
+    except KeyError:
+        raise ForwardingError('Content-Length not returned for %s' %
+                              url)
+    return (r, size)
 
 class RPMObject(object):
     '''Data about a single 'sign-rpms' subrequest.'''
@@ -467,13 +460,14 @@ class SignRPMRequestHandler(RequestHandler):
         self.__rpm.compute_payload_url(self.__koji_client)
         url = self.__rpm.request_payload_url
         try:
-            (src, payload_size) = urlgrabber_open(url)
+            (src, payload_size) = urlopen(url)
             try:
                 conn.server_buf.write(utils.u32_pack(payload_size))
-                copy_file_data(conn.server_buf, src, payload_size)
+                utils.copy_data(conn.server_buff.write, src.iter_content(4096),
+                                payload_size)
             finally:
                 src.close()
-        except urlgrabber.grabber.URLGrabError, e:
+        except requests.RequestException, e:
             raise ForwardingError('Error reading %s: %s' % (url, str(e)))
 
     def forward_reply_payload(self, conn):
@@ -711,14 +705,16 @@ class SignRPMsSendRequestThread(utils.WorkerThread):
         try:
             if payload_size != 0:
                 src = open(rpm.tmp_path, 'rb')
+                readfn = src.read
             else:
-                (src, payload_size) = urlgrabber_open(rpm.request_payload_url)
+                (src, payload_size) = urlopen(rpm.request_payload_url)
+                readfn = src.iter_content(4096)
             try:
                 self.__server_buf.write(utils.u32_pack(payload_size))
-                copy_file_data(self.__server_buf, src, payload_size)
+                utils.copy_data(self.__server_buf.write, readfn, payload_size)
             finally:
                 src.close()
-        except urlgrabber.grabber.URLGrabError, e:
+        except requests.RequestException, e:
             if payload_size != 0:
                 # This exception was not expected, let the default handling
                 # handle it
diff --git a/src/utils.py b/src/utils.py
index 52e7d45..015d5e3 100644
--- a/src/utils.py
+++ b/src/utils.py
@@ -36,6 +36,7 @@ import string
 import sys
 import tempfile
 import threading
+import types
 import xmlrpclib
 
 import nss.error
@@ -845,10 +846,14 @@ def sigterm_handler(*unused_args):
 
 def copy_data(write_fn, read_fn, size):
     '''Copy size bytes using write_fn and read_fn.'''
-    while size > 0:
-        data = read_fn(min(size, 4096))
-        write_fn(data)
-        size -= len(data)
+    if type(read_fn) is types.GeneratorType:
+        for data in read_fn:
+            write_fn(data)
+    else:
+        while size > 0:
+            data = read_fn(min(size, 4096))
+            write_fn(data)
+            size -= len(data)
 
 def file_size_in_blocks(fd):
     '''Return size of fd, taking into account block sizes.'''