From 7792b971f2321881ddb3480ae44adffeaddd5c9f Mon Sep 17 00:00:00 2001
From: Jan Kaluza <jkaluza@redhat.com>
Date: Jul 22 2019 09:07:58 +0000
Subject: Cleanup lost Mock chroots before starting new one.


The Mock chroot can be lost as a result of VM reboot or when
Pungi gets killed for whatever reason.

We still need to ensure that those chroots are removed somehow.
This PR does that by finding Moch chroot older than Pungi timeout
and removes them in safe way before starting any new chroot task.

---

diff --git a/server/odcs/server/mock_runroot.py b/server/odcs/server/mock_runroot.py
index 5882098..491f6a6 100644
--- a/server/odcs/server/mock_runroot.py
+++ b/server/odcs/server/mock_runroot.py
@@ -42,6 +42,7 @@
 #      the output of this command can be stored there.
 
 from __future__ import print_function
+import time
 import platform
 import sys
 import koji
@@ -151,6 +152,31 @@ def rmtree_skip_mounts(rootdir, mounts, rootdir_mounts=None):
     return subdirectory_skipped
 
 
+def cleanup_old_runroots():
+    """
+    Checks the /var/lib/mock diretory for old runroot chroots and remove them.
+
+    Those chroots can be lost there for example when the host reboots in the
+    middle of runroot generation or in case Pungi task gets killed for
+    whatever reason.
+
+    We need to ensure that the chroot data are removed from the filesystem
+    in these cases.
+    """
+    now = time.time()
+    mounts = [conf.target_dir] + conf.runroot_extra_mounts
+    mock_root = "/var/lib/mock"
+    for runroot_key in os.listdir(mock_root):
+        chroot_path = os.path.join(mock_root, runroot_key, "root")
+        # Skip the chroot_path if it does not exist or is not old enough.
+        try:
+            if os.stat(chroot_path).st_mtime > now - conf.pungi_timeout:
+                continue
+        except OSError:
+            continue
+        rmtree_skip_mounts(chroot_path, mounts)
+
+
 def runroot_tmp_path(runroot_key):
     """
     Creates and returns the temporary path to store the configuration files
@@ -202,6 +228,10 @@ def mock_runroot_init(tag_name):
     # would confuse Pungi when calling this method.
     logging.disable(logging.CRITICAL)
 
+    # At first run the cleanup task to remove possible old lost runroot
+    # chroots.
+    cleanup_old_runroots()
+
     # Get the latest Koji repo associated with the tag.
     koji_module = koji.get_profile_module(conf.koji_profile)
     koji_session = create_koji_session()
diff --git a/server/tests/test_mock_runroot.py b/server/tests/test_mock_runroot.py
index 7930706..3fbfdee 100644
--- a/server/tests/test_mock_runroot.py
+++ b/server/tests/test_mock_runroot.py
@@ -20,12 +20,13 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import time
 import unittest
 from mock import patch, mock_open, MagicMock
 
 from odcs.server.mock_runroot import (
     mock_runroot_init, raise_if_runroot_key_invalid, mock_runroot_run,
-    mock_runroot_install, rmtree_skip_mounts)
+    mock_runroot_install, rmtree_skip_mounts, cleanup_old_runroots)
 from .utils import AnyStringWith
 
 
@@ -41,8 +42,10 @@ class TestMockRunroot(unittest.TestCase):
     @patch("odcs.server.mock_runroot.execute_mock")
     @patch("odcs.server.mock_runroot.print", create=True)
     @patch("odcs.server.mock_runroot.rmtree_skip_mounts")
+    @patch("odcs.server.mock_runroot.cleanup_old_runroots")
     def test_mock_runroot_init(
-            self, rmtree_skip_mounts, fake_print, execute_mock, create_koji_session):
+            self, cleanup_old_runroots, rmtree_skip_mounts, fake_print,
+            execute_mock, create_koji_session):
         execute_mock.side_effect = RuntimeError("Expected exception")
         koji_session = create_koji_session.return_value
         koji_session.getRepo.return_value = {"id": 1}
@@ -57,6 +60,7 @@ class TestMockRunroot(unittest.TestCase):
 
         execute_mock.assert_called_once_with(AnyStringWith("-"), ['--init'])
         rmtree_skip_mounts.assert_called_once()
+        cleanup_old_runroots.assert_called_once()
 
     def test_raise_if_runroot_key_invalid(self):
         with self.assertRaises(ValueError):
@@ -155,3 +159,31 @@ class TestMockRunroot(unittest.TestCase):
 
         rmdir.assert_called_once_with("/var/lib/mock/foo-bar/root/mnt/koji")
         remove.assert_called_once_with("/var/lib/mock/foo-bar/root/x")
+
+    @patch("odcs.server.mock_runroot.os.listdir")
+    @patch("odcs.server.mock_runroot.os.stat")
+    @patch("odcs.server.mock_runroot.rmtree_skip_mounts")
+    def test_cleanup_old_runroot(self, rmtree_skip_mounts, stat, listdir):
+        listdir.return_value = ["foo", "bar", "already-removed"]
+
+        def mocked_stat(path):
+            stat_result = MagicMock()
+            if path.endswith("/foo/root"):
+                # The "foo/root" is 1 day old, so should be removed.
+                stat_result.st_mtime = time.time() - 24 * 3600
+            elif path.endswith("/already-removed/root"):
+                # The "already-removed/root" is already removed, so raise an
+                # exception.
+                raise OSError("No such file")
+            elif path.endswith("/bar/root"):
+                # The "bar/root" is just 10 seconds old, so should not be
+                # removed.
+                stat_result.st_mtime = time.time() - 10
+            else:
+                raise ValueError("stat called for unexpected file.")
+            return stat_result
+        stat.side_effect = mocked_stat
+
+        cleanup_old_runroots()
+
+        rmtree_skip_mounts.assert_called_once_with("/var/lib/mock/foo/root", AnyStringWith("test_composes"))