#285 Cleanup lost Mock chroots before starting new one.
Merged 4 years ago by jkaluza. Opened 4 years ago by jkaluza.
jkaluza/odcs cleanu-chroot  into  master

@@ -42,6 +42,7 @@ 

  #      the output of this command can be stored there.

  

  from __future__ import print_function

+ import time

  import platform

  import sys

  import koji
@@ -151,6 +152,31 @@ 

      return subdirectory_skipped

  

  

+ def cleanup_old_runroots():

+     """

+     Checks the /var/lib/mock diretory for old runroot chroots and remove them.

+ 

+     Those chroots can be lost there for example when the host reboots in the

+     middle of runroot generation or in case Pungi task gets killed for

+     whatever reason.

+ 

+     We need to ensure that the chroot data are removed from the filesystem

+     in these cases.

+     """

+     now = time.time()

+     mounts = [conf.target_dir] + conf.runroot_extra_mounts

+     mock_root = "/var/lib/mock"
cqi commented 4 years ago

Would it be good to make this configurable?

It would be cleaner than hardcoding this, but it is default Mock directory which we do not change on our deployments.

+     for runroot_key in os.listdir(mock_root):

+         chroot_path = os.path.join(mock_root, runroot_key, "root")

+         # Skip the chroot_path if it does not exist or is not old enough.

+         try:

+             if os.stat(chroot_path).st_mtime > now - conf.pungi_timeout:

+                 continue

+         except OSError:

+             continue

+         rmtree_skip_mounts(chroot_path, mounts)

+ 

+ 

  def runroot_tmp_path(runroot_key):

      """

      Creates and returns the temporary path to store the configuration files
@@ -202,6 +228,10 @@ 

      # would confuse Pungi when calling this method.

      logging.disable(logging.CRITICAL)

  

+     # At first run the cleanup task to remove possible old lost runroot

+     # chroots.

+     cleanup_old_runroots()

+ 

      # Get the latest Koji repo associated with the tag.

      koji_module = koji.get_profile_module(conf.koji_profile)

      koji_session = create_koji_session()

@@ -20,12 +20,13 @@ 

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

  

+ import time

  import unittest

  from mock import patch, mock_open, MagicMock

  

  from odcs.server.mock_runroot import (

      mock_runroot_init, raise_if_runroot_key_invalid, mock_runroot_run,

-     mock_runroot_install, rmtree_skip_mounts)

+     mock_runroot_install, rmtree_skip_mounts, cleanup_old_runroots)

  from .utils import AnyStringWith

  

  
@@ -41,8 +42,10 @@ 

      @patch("odcs.server.mock_runroot.execute_mock")

      @patch("odcs.server.mock_runroot.print", create=True)

      @patch("odcs.server.mock_runroot.rmtree_skip_mounts")

+     @patch("odcs.server.mock_runroot.cleanup_old_runroots")

      def test_mock_runroot_init(

-             self, rmtree_skip_mounts, fake_print, execute_mock, create_koji_session):

+             self, cleanup_old_runroots, rmtree_skip_mounts, fake_print,

+             execute_mock, create_koji_session):

          execute_mock.side_effect = RuntimeError("Expected exception")

          koji_session = create_koji_session.return_value

          koji_session.getRepo.return_value = {"id": 1}
@@ -57,6 +60,7 @@ 

  

          execute_mock.assert_called_once_with(AnyStringWith("-"), ['--init'])

          rmtree_skip_mounts.assert_called_once()

+         cleanup_old_runroots.assert_called_once()

  

      def test_raise_if_runroot_key_invalid(self):

          with self.assertRaises(ValueError):
@@ -155,3 +159,31 @@ 

  

          rmdir.assert_called_once_with("/var/lib/mock/foo-bar/root/mnt/koji")

          remove.assert_called_once_with("/var/lib/mock/foo-bar/root/x")

+ 

+     @patch("odcs.server.mock_runroot.os.listdir")

+     @patch("odcs.server.mock_runroot.os.stat")

+     @patch("odcs.server.mock_runroot.rmtree_skip_mounts")

+     def test_cleanup_old_runroot(self, rmtree_skip_mounts, stat, listdir):

+         listdir.return_value = ["foo", "bar", "already-removed"]

+ 

+         def mocked_stat(path):

+             stat_result = MagicMock()

+             if path.endswith("/foo/root"):

+                 # The "foo/root" is 1 day old, so should be removed.

+                 stat_result.st_mtime = time.time() - 24 * 3600

+             elif path.endswith("/already-removed/root"):

+                 # The "already-removed/root" is already removed, so raise an

+                 # exception.

+                 raise OSError("No such file")

+             elif path.endswith("/bar/root"):

+                 # The "bar/root" is just 10 seconds old, so should not be

+                 # removed.

+                 stat_result.st_mtime = time.time() - 10

+             else:

+                 raise ValueError("stat called for unexpected file.")

+             return stat_result

+         stat.side_effect = mocked_stat

+ 

+         cleanup_old_runroots()

+ 

+         rmtree_skip_mounts.assert_called_once_with("/var/lib/mock/foo/root", AnyStringWith("test_composes"))

The Mock chroot can be lost as a result of VM reboot or when
Pungi gets killed for whatever reason.

We still need to ensure that those chroots are removed somehow.
This PR does that by finding Moch chroot older than Pungi timeout
and removes them in safe way before starting any new chroot task.

Would it be good to make this configurable?

It would be cleaner than hardcoding this, but it is default Mock directory which we do not change on our deployments.

Pull-Request has been merged by jkaluza

4 years ago