From 0ae379874e53923e75d9d3b849179854391e2255 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 04 2017 10:17:55 +0000
Subject: [PATCH 1/7] fixed issues with deleting


---

diff --git a/moduleframework/common.py b/moduleframework/common.py
index e2d004c..f328dbf 100644
--- a/moduleframework/common.py
+++ b/moduleframework/common.py
@@ -105,6 +105,7 @@ trans_dict = {"HOSTIPADDR": hostipaddr,
               }
 
 
+BASEPATHDIR = "/opt"
 PDCURL = "https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants"
 URLBASECOMPOSE = "https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server"
 REPOMD = "repodata/repomd.xml"
diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py
index de3264b..4adb371 100755
--- a/moduleframework/module_framework.py
+++ b/moduleframework/module_framework.py
@@ -731,6 +731,7 @@ class NspawnHelper(RpmHelper):
         relative change root path
         """
         super(NspawnHelper, self).__init__()
+        self.baseprefix = os.path.join(BASEPATHDIR, "chroot_")
         self.__selinuxState = None
         time.time()
         actualtime = time.time()
@@ -738,9 +739,7 @@ class NspawnHelper(RpmHelper):
             self.jmeno = "%s_%r" % (self.moduleName, actualtime)
         else:
             self.jmeno = self.moduleName
-        self.chrootpath = os.path.abspath(
-            os.path.join(
-                "/opt", "chroot_%s" % self.jmeno))
+        self.chrootpath = os.path.abspath(self.baseprefix + self.jmeno)
         print_info("name of CHROOT directory:", self.chrootpath)
         trans_dict["ROOT"] = self.chrootpath
 
@@ -786,20 +785,37 @@ class NspawnHelper(RpmHelper):
                 return True
         raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT))
 
-    def __prepareSetup(self):
+    def __do_smart_start_cleanup(self):
         """
         Internal method, do not use it anyhow
 
         :return: None
         """
-        if get_if_do_cleanup() and os.path.exists(self.chrootpath):
-            shutil.rmtree(self.chrootpath, ignore_errors=True)
-            os.mkdir(self.chrootpath)
+
+        if get_if_do_cleanup():
+            # delete directory with same same (in case used option DO NOT CLEANUP)
+            if os.path.exists(self.chrootpath):
+                shutil.rmtree(self.chrootpath, ignore_errors=True)
+            # DELETE every chroot dir in case any exists
+            dirstodelete = glob.glob(self.baseprefix + "*")
+            if dirstodelete:
+                for dtd in dirstodelete:
+                    shutil.rmtree(dtd, ignore_errors=True)
+            # Terminate machine in case of same name and still running
         try:
             self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug(), ignore_status=True)
             self.__is_killed()
         except BaseException:
             pass
+        os.mkdir(self.chrootpath)
+
+    def __prepareSetup(self):
+        """
+        Internal method, do not use it anyhow
+
+        :return: None
+        """
+        self.__do_smart_start_cleanup()
         if not os.path.exists(os.path.join(self.chrootpath, "usr")):
             self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent())
             repos_to_use = ""
@@ -1023,7 +1039,14 @@ gpgcheck=0
             self.__is_killed()
         except Exception as poweroffex:
             print_info("Unable to stop machine via poweroff, terminating", poweroffex)
-            self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True)
+            try:
+                time.sleep(1)
+                self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True)
+                self.__is_killed()
+            except Exception as poweroffexterm:
+                print_info("Unable to stop machine via terminate, STRANGE", poweroffexterm)
+                time.sleep(DEFAULTRETRYTIMEOUT)
+                pass
             pass
 
         if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'):
diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index d43f61b..c3e68ac 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -245,7 +245,7 @@ class PDCParser():
 
         :return: str
         """
-        dir_prefix = "/opt"
+        dir_prefix = BASEPATHDIR
         utils.process.run("{HOSTPACKAGER} install createrepo koji".format(
             **trans_dict), ignore_status=True)
         if is_recursive_download():

From fbfffd3419a3331217bdf8fa412fa56ddfac67d1 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 04 2017 10:17:55 +0000
Subject: [PATCH 2/7] koji package downloading cleanup, moved to pdc instead of hardcoding in MTF main file


---

diff --git a/moduleframework/common.py b/moduleframework/common.py
index f328dbf..de22390 100644
--- a/moduleframework/common.py
+++ b/moduleframework/common.py
@@ -167,3 +167,36 @@ def is_recursive_download():
     :return: bool
     """
     return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD"))
+
+
+def get_if_do_cleanup():
+    """
+    Returns boolean value in case variable is set.
+     It is used internally in code
+
+    :return: bool
+    """
+    cleanup = os.environ.get('MTF_DO_NOT_CLEANUP')
+    return not bool(cleanup)
+
+
+def get_if_remoterepos():
+    """
+    Returns boolean value in case variable is set.
+    It is used internally in code
+
+    :return: bool
+    """
+    rreps = os.environ.get('MTF_REMOTE_REPOS')
+    return bool(rreps)
+
+
+def get_if_module():
+    """
+    Returns boolean value in case variable is set.
+    It is used internally in code
+
+    :return: bool
+    """
+    rreps = os.environ.get('MTF_DISABLE_MODULE')
+    return not bool(rreps)
diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py
index 4adb371..f46eec3 100755
--- a/moduleframework/module_framework.py
+++ b/moduleframework/module_framework.py
@@ -1464,42 +1464,6 @@ def get_latest_repo_url(wmodule="base-runtime", wstream="master", fake=False):
     if fake:
         return "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/"
     else:
-        localrepo = pdc_data.PDCParser()
-        localrepo.setLatestPDC(wmodule, wstream)
-        if get_if_remoterepos():
-            return localrepo.generateRepoUrl()
-        else:
-            return localrepo.createLocalRepoFromKoji()
-
-
-def get_if_do_cleanup():
-    """
-    Returns boolean value in case variable is set.
-     It is used internally in code
-
-    :return: bool
-    """
-    cleanup = os.environ.get('MTF_DO_NOT_CLEANUP')
-    return not bool(cleanup)
-
-
-def get_if_remoterepos():
-    """
-    Returns boolean value in case variable is set.
-    It is used internally in code
-
-    :return: bool
-    """
-    rreps = os.environ.get('MTF_REMOTE_REPOS')
-    return bool(rreps)
-
-
-def get_if_module():
-    """
-    Returns boolean value in case variable is set.
-    It is used internally in code
-
-    :return: bool
-    """
-    rreps = os.environ.get('MTF_DISABLE_MODULE')
-    return not bool(rreps)
+        tmp_pdc = pdc_data.PDCParser()
+        tmp_pdc.setLatestPDC(wmodule, wstream)
+        return tmp_pdc.generateRepoUrl()
diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index c3e68ac..b9105f3 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -152,8 +152,11 @@ class PDCParser():
         """
         # rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % (
         #    self.pdcdata["koji_tag"] + "-build", ARCH)
-        rpmrepo = "%s/%s/os/" % (URLBASECOMPOSE, ARCH)
-        return rpmrepo
+        if get_if_remoterepos():
+            rpmrepo = "%s/%s/os/" % (URLBASECOMPOSE, ARCH)
+            return rpmrepo
+        else:
+            return self.createLocalRepoFromKoji()
 
     def generateGitHash(self):
         """
@@ -271,15 +274,3 @@ class PDCParser():
                 (absdir, absdir), shell=True, verbose=is_debug())
         return "file://%s" % absdir
 
-    def generateParamsLocalKojiPkgs(self):
-        """
-        Return list of params what has to be set for automation like (local repo):
-        MODULE=nspawn MODULEMDURL=file:///...  URL=file:///localrepo
-
-        :return: list
-        """
-        output = []
-        output.append("URL=%s" % self.createLocalRepoFromKoji())
-        output.append("MODULEMDURL=%s" % self.generateModuleMDFile())
-        output.append("MODULE=%s" % "nspawn")
-        return output
diff --git a/tools/run-them.sh b/tools/run-them.sh
index 9193d2c..94253e0 100755
--- a/tools/run-them.sh
+++ b/tools/run-them.sh
@@ -21,16 +21,6 @@
 # Authors: Jan Scotka <jscotka@redhat.com>
 #
 
-SLEEP=15
-
-function sleep_a_while(){
-    echo "sleep for $SLEEP minutes to ensure that repos are ready"
-    for foo in `seq $SLEEP`; do
-        sleep 60
-        echo "$foo/$SLEEP minutes done"
-    done
-}
-
 function fedpkg_alt(){
     #LFILE="alt_fedpkg.conf"
     #cat /etc/rpkg/fedpkg.conf |sed 's/anongiturl.*/anongiturl = https:\/\/src.fedoraproject.org\/git\/%(module)s/' > $LFILE
@@ -54,12 +44,13 @@ export AVOCADOCMD="avocado run --xunit $XUFILE --show-job-log"
 export RESULTTOOLS=0
 export MTF_RECURSIVE_DOWNLOAD=yes
 
+
 function getparams_int(){
     ADDIT="$1"
     if [ "$PARSEITEMTYPE" = "" -o "$PARSEITEMTYPE" = "fedmsg" ]; then
-        python $MTF_PATH/tools/taskotron-msg-reader.py -f $PARSEITEM --localrepo $ADDIT
+        python $MTF_PATH/tools/taskotron-msg-reader.py -f $PARSEITEM $ADDIT
     elif [ "$PARSEITEMTYPE" = "taskotron" -o "$PARSEITEMTYPE" = "pdc" ]; then
-        python $MTF_PATH/tools/taskotron-msg-reader.py -r $PARSEITEM --localrepo $ADDIT
+        python $MTF_PATH/tools/taskotron-msg-reader.py -r $PARSEITEM $ADDIT
     elif [ -z $ADDIT -a "$PARSEITEMTYPE" = "compose" ]; then
         python $MTF_PATH/tools/compose_info_parser.py -c $PARSEITEM -m $MODULENAME
     fi
diff --git a/tools/taskotron-msg-reader.py b/tools/taskotron-msg-reader.py
index ce05dd6..77ec63e 100755
--- a/tools/taskotron-msg-reader.py
+++ b/tools/taskotron-msg-reader.py
@@ -41,12 +41,6 @@ if __name__ == '__main__':
     parser.add_option("-l", "--latest", dest="latest",
                       help="Use latest bits, build by MBS and stored in PDC")
     parser.add_option(
-        "--localrepo",
-        dest="localrepo",
-        action="store_true",
-        default=False,
-        help="generate local repository based on koji tag from PDC")
-    parser.add_option(
         "--commit",
         dest="commit",
         action="store_true",
@@ -55,10 +49,6 @@ if __name__ == '__main__':
 
     a = PDCParser()
     (options, args) = parser.parse_args()
-    if options.localrepo:
-        callfnc = "generateParamsLocalKojiPkgs"
-    else:
-        callfnc = "generateParams"
     if options.filename:
         flh = open(options.filename)
         stdinput = "".join(flh.readlines()).strip()
@@ -74,4 +64,4 @@ if __name__ == '__main__':
     if options.commit:
         print a.generateGitHash()
     else:
-        print " ".join(getattr(a, callfnc)())
+        print " ".join(a.generateParams())

From db091e3c83b465dff9fea711d455ed5caa14c0fc Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 04 2017 11:03:38 +0000
Subject: [PATCH 3/7] changed testing module to minimal config. mksh is not in compose


---

diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile
index 7a2aca2..2ca8869 100644
--- a/examples/testing-module/Makefile
+++ b/examples/testing-module/Makefile
@@ -5,11 +5,11 @@ export DEBUG=yes
 export DOCKERFILE=./Dockerfile
 
 check-docker:
-	MODULE=docker $(CMD) $(TESTS)
+	MODULE=docker CONFIG=./fullconfig.yaml $(CMD) $(TESTS)
 	@true
 
 check-minimal-config-docker:
-	MODULE=docker CONFIG=./minimal.yaml $(CMD) $(TESTS)
+	MODULE=docker $(CMD) $(TESTS)
 	@true
 
 
diff --git a/examples/testing-module/config.yaml b/examples/testing-module/config.yaml
index c74964b..71ca43d 120000
--- a/examples/testing-module/config.yaml
+++ b/examples/testing-module/config.yaml
@@ -1 +1 @@
-../../docs/example-config.yaml
\ No newline at end of file
+../../docs/example-config-minimal.yaml
\ No newline at end of file
diff --git a/examples/testing-module/fullconfig.yaml b/examples/testing-module/fullconfig.yaml
new file mode 120000
index 0000000..c74964b
--- /dev/null
+++ b/examples/testing-module/fullconfig.yaml
@@ -0,0 +1 @@
+../../docs/example-config.yaml
\ No newline at end of file
diff --git a/examples/testing-module/minimal.yaml b/examples/testing-module/minimal.yaml
deleted file mode 120000
index 71ca43d..0000000
--- a/examples/testing-module/minimal.yaml
+++ /dev/null
@@ -1 +0,0 @@
-../../docs/example-config-minimal.yaml
\ No newline at end of file

From 1616ff34db4a1b3687220193b8ff13a352241755 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 04 2017 11:41:27 +0000
Subject: [PATCH 5/7] minimal config path fix


---

diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile
index 2ca8869..2a4217c 100644
--- a/examples/testing-module/Makefile
+++ b/examples/testing-module/Makefile
@@ -14,11 +14,11 @@ check-minimal-config-docker:
 
 
 check-rpm:
-	MODULE=nspawn $(CMD) $(TESTS)
+	MODULE=nspawn CONFIG=./fullconfig.yaml $(CMD) $(TESTS)
 	@true
 
 check-minimal-config-rpm:
-	MODULE=nspawn CONFIG=./minimal.yaml $(CMD) $(TESTS)
+	MODULE=nspawn $(CMD) $(TESTS)
 	@true
 
 check-behave-docker:
diff --git a/tools/run-them.sh b/tools/run-them.sh
index 94253e0..ce8f6fe 100755
--- a/tools/run-them.sh
+++ b/tools/run-them.sh
@@ -33,8 +33,9 @@ export PARSEITEM=$2
 # compose fedmsg or None same as fedmsg
 export PARSEITEMTYPE=$3
 export SELECTION=$4
-export MTF_PATH=/usr/share/moduleframework
+export MTF_PATH="/usr/share/moduleframework"
 export MODULE_LINT="$MTF_PATH/tools/modulelint/*.py"
+export MINIMAL_CONFIG="$MTF_PATH/docs/example-config-minimal.yaml"
 export MODULE_TESTS="*.py *.sh"
 
 export AVDIR=~/avocado
@@ -44,7 +45,6 @@ export AVOCADOCMD="avocado run --xunit $XUFILE --show-job-log"
 export RESULTTOOLS=0
 export MTF_RECURSIVE_DOWNLOAD=yes
 
-
 function getparams_int(){
     ADDIT="$1"
     if [ "$PARSEITEMTYPE" = "" -o "$PARSEITEMTYPE" = "fedmsg" ]; then
@@ -110,7 +110,7 @@ function run_modulelint(){
 
     TESTS="`ls $MODULE_LINT`"
     echo "RUN AT LEAST MODULE LINTER: $TESTS"
-    eval $PARAMS CONFIG=$MTF_PATH/docs/example-config-minimal.yaml $AVOCADOCMD $TESTS
+    eval $PARAMS CONFIG=$MINIMAL_CONFIG $AVOCADOCMD $TESTS
 }
 
 set -x

From bc67e1c7b5bad851218d8860c1e9fff22c55b9c5 Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 04 2017 12:11:11 +0000
Subject: [PATCH 6/7] make docker linter faster, not need to invoke parent setup in own setup class, because it does offline checking


---

diff --git a/tools/modulelint.py b/tools/modulelint.py
index 245a452..122f379 100644
--- a/tools/modulelint.py
+++ b/tools/modulelint.py
@@ -39,7 +39,6 @@ class DockerfileLinter(module_framework.AvocadoTest):
     def setUp(self):
         # it is not intended just for docker, but just docker packages are
         # actually properly signed
-        super(self.__class__, self).setUp()
         self.dp = dockerlinter.DockerfileLinter(os.path.join(os.getcwd(), ".."))
         if self.dp.dockerfile is None:
             self.skip()

From 2d60093fe2e8a1a012a86e57328853dddd26999c Mon Sep 17 00:00:00 2001
From: Jan Scotka <jscotka@redhat.com>
Date: Jul 07 2017 08:43:15 +0000
Subject: [PATCH 7/7] Added unitetest to pdc module


---

diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py
index b9105f3..354070e 100644
--- a/moduleframework/pdc_data.py
+++ b/moduleframework/pdc_data.py
@@ -217,7 +217,7 @@ class PDCParser():
         :param dirname: string
         :return: None
         """
-        print_info("DOWLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version))
+        print_info("DOWNLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version))
         for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"):
             pkgbouid = foo.strip().split(" ")[0]
             if len(pkgbouid) > 4:
@@ -239,7 +239,7 @@ class PDCParser():
                                 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
 
                 tmpfunc()
-        print_info("DOWLOADING finished")
+        print_info("DOWNLOADING finished")
 
     def createLocalRepoFromKoji(self):
         """
@@ -274,3 +274,14 @@ class PDCParser():
                 (absdir, absdir), shell=True, verbose=is_debug())
         return "file://%s" % absdir
 
+if __name__ == "__main__":
+    a = PDCParser()
+    a.setLatestPDC(name="memcached", stream="f26")
+    dependencies = a.generateDepModules()
+    get_if_remoterepos = (lambda: True)
+    assert dependencies == {'base-runtime': 'f26', 'shared-userspace': 'f26', 'perl': 'f26'}
+    assert "https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server/x86_64/os/" == a.generateRepoUrl()
+    assert "URL=https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server/x86_64/os/" in a.generateParams()
+    assert "MODULE=nspawn" in a.generateParams()
+    assert len(a.generateGitHash()) == 41
+    assert "Memcached is a high-performance, distributed" in a.getmoduleMD()['data']['description']

