#245 Remove old cached data from Koji tag cache directory.
Merged 5 months ago by cqi. Opened 5 months ago by jkaluza.
jkaluza/odcs remove-old-cache  into  master

@@ -225,6 +225,10 @@ 

                  self._remove_compose_dir(path)

                  continue

  

+         # Remove old Koji tag data from Koji tag cache.

+         koji_tag_cache = KojiTagCache()

+         koji_tag_cache.remove_old_koji_tag_cache_data()

+ 

  

  def create_koji_session():

      """

@@ -21,6 +21,7 @@ 

  #

  # Written by Jan Kaluza <jkaluza@redhat.com>

  

+ import time

  import os

  import shutil

  import threading

@@ -51,6 +52,35 @@ 

          with KojiTagCache._cache_lock:

              makedirs(self.cache_dir)

  

+     def remove_old_koji_tag_cache_data(self):

+         """

+         Removes the old unused Koji tag cache directories which has not been

+         used for more than `conf.koji_tag_cache_cleanup_timeout` days.

+         """

+         # The koji_tag_cache_cleanup_timeout is in days, so convert it to

+         # seconds.

+         older_than_seconds = conf.koji_tag_cache_cleanup_timeout * 24 * 3600

+         threshold = time.time() - older_than_seconds

+ 

+         for cached_dir in os.listdir(self.cache_dir):

+             path = os.path.join(self.cache_dir, cached_dir)

+             try:

+                 mtime = os.path.getmtime(path)

+             except OSError:

+                 # Directory might have been removed in meantime by some

+                 # 3rd party process.

+                 log.exception(

+                     "Old koji tag cache directory %s removed while checking "

+                     "Koji cache." % path)

+                 continue

+ 

+             if mtime > threshold:

+                 # Koji tag directory is not old enough to be removed.

+                 continue

+ 

+             log.info("Removing old Koji tag cache data in %s." % path)

+             shutil.rmtree(path)

+ 

      def cached_compose_dir(self, compose):

          """

          Returns the full path to caching directory belonging to `compose`.

@@ -326,6 +326,12 @@ 

              'type': str,

              'default': None,

              'desc': 'Kerberos principal to use for Koji auth.'},

+         'koji_tag_cache_cleanup_timeout': {

+             'type': int,

+             'default': 30,

+             'desc': 'Number of days after which the cached Koji tag data '

+                     'stored in the "koji_tag_cache" directory will be '

+                     'removed.'},

          'raw_config_urls': {

              'type': dict,

              'default': {},

@@ -20,6 +20,7 @@ 

  #

  # Written by Jan Kaluza <jkaluza@redhat.com>

  

+ import time

  import os

  from mock import patch

  

@@ -137,3 +138,28 @@ 

          self.cache.update_cache(c)

          rmtree.assert_called_once_with(

              os.path.join(conf.target_dir, "koji_tag_cache/f26-0--x86_64"))

+ 

+     @patch("os.listdir")

+     @patch("os.path.getmtime")

+     @patch("shutil.rmtree")

+     def test_remove_old_koji_tag_cache_data(self, rmtree, getmtime, listdir):

+         now = time.time()

+         listdir.return_value = ["foo", "bar"]

+         # Default timeout is 30 days, so set 31 for first dir and 29 for

+         # second dir.

+         getmtime.side_effect = [now - 31 * 24 * 3600, now - 29 * 24 * 3600]

+ 

+         self.cache.remove_old_koji_tag_cache_data()

+         rmtree.assert_called_once_with(

+             os.path.join(self.cache.cache_dir, "foo"))

+ 

+     @patch("os.listdir")

+     @patch("os.path.getmtime")

+     @patch("shutil.rmtree")

+     def test_remove_old_koji_tag_cache_data_getmtime_raises(

+             self, rmtree, getmtime, listdir):

+         listdir.return_value = ["foo", "bar"]

+         getmtime.side_effect = OSError("path does not exist")

+ 

+         self.cache.remove_old_koji_tag_cache_data()

+         rmtree.assert_not_called()

Each directory in the Koji tag cache directory is unique to particular
compose input data. In case when for example compose is generated from
particular Koji tag just once and never again, the cached data would
stay in NFS storage forever.

This is not good, because the NFS storage is not big and generally
we should cleanup the unused data.

In this commit, the cached data which has not been used for more than
30 days are removed. This is configurable.

rebased onto dd1218a

5 months ago

Pull-Request has been merged by cqi

5 months ago