From ccceff2232d8802dc680ecfab558c9a5649ce3f6 Mon Sep 17 00:00:00 2001
From: Till Maas <opensource@till.name>
Date: Jul 09 2018 15:46:20 +0000
Subject: mass_rebuild_file_bugs: Improve log attachment


- download data in chunks
- handle log files, that are too large

Signed-off-by: Till Maas <opensource@till.name>

---

diff --git a/scripts/mass_rebuild_file_bugs.py b/scripts/mass_rebuild_file_bugs.py
index 707618b..fe08a3e 100755
--- a/scripts/mass_rebuild_file_bugs.py
+++ b/scripts/mass_rebuild_file_bugs.py
@@ -112,13 +112,33 @@ def attach_logs(bug, logs):
         name = log.rsplit('/', 1)[-1]
         response = urllib2.urlopen(log)
         fp = tempfile.TemporaryFile()
-        fp.write(response.read())
-        fp.seek(0)
+
+        CHUNK = 2 ** 20
+        while True:
+            chunk = response.read(CHUNK)
+            if not chunk:
+                break
+            fp.write(chunk)
+
+        filesize = fp.tell()
+        # Bugzilla file limit, still possibly too much
+        # FILELIMIT = 20000 * 1024
+        # Just use 1 MiB:
+        FILELIMIT = 2 ** 10
+        if filesize > FILELIMIT:
+            fp.seek(filesize - FILELIMIT)
+            comment = "file {} too big, will only attach last {} bytes".format(
+                name, FILELIMIT)
+        else:
+            comment = ""
+            fp.seek(0)
         try:
             print('Attaching file %s to the ticket' % name)
             # arguments are: idlist, attachfile, description, ...
             attid = BZCLIENT.attachfile(
-                bug.id, fp, name, content_type='text/plain', file_name=name)
+                bug.id, fp, name, content_type='text/plain', file_name=name,
+                comment=comment
+            )
         except Fault as  ex:
             print(ex)
             raise