From bbd7d219f401054e5b461ea9d8b3ccd586c8fc91 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jun 27 2017 12:04:46 +0000
Subject: [PATCH 1/5] initial commit of recursive downloading of all dependent modules to one repository


---

diff --git a/moduleframework/common.py b/moduleframework/common.py
index ecfc00b..cf867a1 100644
--- a/moduleframework/common.py
+++ b/moduleframework/common.py
@@ -153,3 +153,6 @@ def print_debug(*args):
     """
     if is_debug():
         print_info(*args)
+
+def if_recursive_download():
+    return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD"))
\ No newline at end of file
diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index 4dc3091..e246d32 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -206,6 +206,29 @@ class PDCParser():
             out = {}
         return out
 
+    def download_tagged(self,dirname):
+        for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"):
+            pkgbouid = foo.strip().split(" ")[0]
+            if len(pkgbouid) > 4:
+                print_debug("DOWNLOADING: %s" % foo)
+
+                @Retry(attempts=DEFAULTRETRYCOUNT * 10, timeout=DEFAULTRETRYTIMEOUT * 60, delay=DEFAULTRETRYTIMEOUT,
+                       error=KojiExc(
+                           "RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT * 10)))
+                def tmpfunc():
+                    a = utils.process.run(
+                        "cd %s; koji download-build %s  -a %s -a noarch" %
+                        (dirname, pkgbouid, ARCH), shell=True, verbose=is_debug(), ignore_status=True)
+                    if a.exit_status == 1:
+                        if "packages available for" in a.stdout.strip():
+                            print_debug(
+                                'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command)
+                        else:
+                            raise KojiExc(
+                                'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
+
+                tmpfunc()
+
     def createLocalRepoFromKoji(self):
         """
         Return string of generated repository located LOCALLY
@@ -221,25 +244,9 @@ class PDCParser():
             pass
         else:
             os.mkdir(absdir)
-            for foo in utils.process.run(
-                    "koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"):
-                pkgbouid = foo.strip().split(" ")[0]
-                if len(pkgbouid) > 4:
-                    print_debug("DOWNLOADING: %s" % foo)
-
-                    @Retry(attempts=DEFAULTRETRYCOUNT * 10, timeout=DEFAULTRETRYTIMEOUT * 60, delay=DEFAULTRETRYTIMEOUT, error=KojiExc("RETRY: Unbale to fetch package from koji after %d attempts" % (DEFAULTRETRYCOUNT * 10)))
-                    def tmpfunc():
-                        a = utils.process.run(
-                            "cd %s; koji download-build %s  -a %s -a noarch" %
-                            (absdir, pkgbouid, ARCH), shell=True, verbose=is_debug(), ignore_status=True)
-                        if a.exit_status == 1:
-                            if "packages available for" in a.stdout.strip():
-                                print_debug(
-                                    'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command)
-                            else:
-                                raise KojiExc(
-                                    'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
-                    tmpfunc()
+            self.download_tagged(absdir)
+            if if_recursive_download():
+                self.generateDepModules()
             utils.process.run(
                 "cd %s; createrepo -v %s" %
                 (absdir, absdir), shell=True, verbose=is_debug())

From b4783e4193d049302662d2a52a589baff2ae65b2 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jun 27 2017 12:20:44 +0000
Subject: [PATCH 2/5] solved issues with multiple directories, create just one repo, and do not overwrite it


---

diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index e246d32..782921c 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -207,6 +207,7 @@ class PDCParser():
         return out
 
     def download_tagged(self,dirname):
+        print_info("DOWLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version))
         for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"):
             pkgbouid = foo.strip().split(" ")[0]
             if len(pkgbouid) > 4:
@@ -228,6 +229,7 @@ class PDCParser():
                                 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
 
                 tmpfunc()
+        print_info("DOWLOADING finished" % (self.name, self.stream, self.version))
 
     def createLocalRepoFromKoji(self):
         """
@@ -238,7 +240,10 @@ class PDCParser():
         """
         utils.process.run("{HOSTPACKAGER} install createrepo koji".format(
             **trans_dict), ignore_status=True)
-        dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version)
+        if if_recursive_download():
+            dirname="localrepo_recursive"
+        else:
+            dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version)
         absdir = os.path.abspath(dirname)
         if os.path.exists(absdir):
             pass
@@ -246,7 +251,12 @@ class PDCParser():
             os.mkdir(absdir)
             self.download_tagged(absdir)
             if if_recursive_download():
-                self.generateDepModules()
+                allmodules = self.generateDepModules()
+                for mo in allmodules:
+                    localrepo = PDCParser()
+                    localrepo.setLatestPDC(mo, allmodules[mo])
+                    localrepo.download_tagged(dirname)
+
             utils.process.run(
                 "cd %s; createrepo -v %s" %
                 (absdir, absdir), shell=True, verbose=is_debug())

From 964a66b26795d34a60b249cfb193059e18e88f76 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jun 27 2017 12:34:32 +0000
Subject: [PATCH 3/5] added new option to run-them script


---

diff --git a/tools/run-them.sh b/tools/run-them.sh
index 4e64c25..61f473c 100755
--- a/tools/run-them.sh
+++ b/tools/run-them.sh
@@ -52,6 +52,7 @@ mkdir -p $AVDIR
 export XUFILE="$AVDIR/out.xunit"
 export AVOCADOCMD="avocado run --xunit $XUFILE"
 export RESULTTOOLS=0
+export MTF_RECURSIVE_DOWNLOAD=yes
 
 function getparams_int(){
     ADDIT="$1"
@@ -122,7 +123,7 @@ function run_modulelint(){
 set -x
 inst_env
 RESULTTOOLS=$(($RESULTTOOLS+$?))
-PARAMS="`getparams_int`"
+PARAMS="`DEBUG=yes getparams_int`"
 RESULTTOOLS=$(($RESULTTOOLS+$?))
 export PARAMS
 

From 540a1397e3293f5c6d4617fd375b0c62d81d0ffb Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jun 27 2017 12:39:20 +0000
Subject: [PATCH 4/5] if is bad name prefix, is_ is better


---

diff --git a/moduleframework/common.py b/moduleframework/common.py
index cf867a1..1fbda74 100644
--- a/moduleframework/common.py
+++ b/moduleframework/common.py
@@ -154,5 +154,5 @@ def print_debug(*args):
     if is_debug():
         print_info(*args)
 
-def if_recursive_download():
+def is_recursive_download():
     return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD"))
\ No newline at end of file
diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index 782921c..f6cb66c 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -240,7 +240,7 @@ class PDCParser():
         """
         utils.process.run("{HOSTPACKAGER} install createrepo koji".format(
             **trans_dict), ignore_status=True)
-        if if_recursive_download():
+        if is_recursive_download():
             dirname="localrepo_recursive"
         else:
             dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version)
@@ -250,7 +250,7 @@ class PDCParser():
         else:
             os.mkdir(absdir)
             self.download_tagged(absdir)
-            if if_recursive_download():
+            if is_recursive_download():
                 allmodules = self.generateDepModules()
                 for mo in allmodules:
                     localrepo = PDCParser()

From b841e8e2ae86001e51dcd7db042fb6c7113bb1d7 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jun 27 2017 12:44:50 +0000
Subject: [PATCH 5/5] if is bad name prefix, is_ is better


---

diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index f6cb66c..5aef6d9 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -241,7 +241,7 @@ class PDCParser():
         utils.process.run("{HOSTPACKAGER} install createrepo koji".format(
             **trans_dict), ignore_status=True)
         if is_recursive_download():
-            dirname="localrepo_recursive"
+            dirname = "localrepo_recursive"
         else:
             dirname = "localrepo_%s_%s_%s" % (self.name, self.stream, self.version)
         absdir = os.path.abspath(dirname)

