PR#143: WIP: If pungi_runroot_enabled is true, use koji runroot also to remove expired composes. - odcs

odcs

#143 WIP: If pungi_runroot_enabled is true, use koji runroot also to remove expired composes.

Closed 6 years ago by jkaluza. Opened 6 years ago by jkaluza.

jkaluza/odcs session-rollbac into master

If pungi_runroot_enabled is true, use koji runroot also to remove expired composes.

Jan Kaluza • 6 years ago

c4078c1

server/odcs/server/backend.py

file modified

+57 -3

		`@@ -143,6 +143,33 @@`
		`else:`
		`shutil.rmtree(toplevel_dir)`

		`+ def _get_remove_compose_dir_cmds(self, toplevel_dir):`
		`+ """`
		`+ Returns the commands needed to remove the compose with toplevel_dir`
		`+ compose dir as list.`
		`+ """`
		`+`
		`+ # Be nice and don't fail when directory does not exist.`
		`+ if not os.path.exists(toplevel_dir):`
		`+ log.warn("Cannot remove directory %s, it does not exist",`
		`+ toplevel_dir)`
		`+ return []`
		`+`
		`+ cmds = []`
		`+`
		`+ # If toplevel_dir is a symlink, remove the symlink and`
		`+ # its target. If toplevel_dir is normal directory, just`
		`+ # remove it using rmtree.`
		`+ if os.path.realpath(toplevel_dir) != toplevel_dir:`
		`+ targetpath = os.path.realpath(toplevel_dir)`
		`+ cmds += ["rm", "-f", toplevel_dir]`
		`+ if os.path.exists(targetpath):`
		`+ cmds += ["&&", "rm", "-rf", targetpath]`
		`+ else:`
		`+ cmds += ["rm", "-rf", toplevel_dir]`
		`+`
		`+ return cmds`
		`+`
		`def _get_compose_id_from_path(self, path):`
		`"""`
		`Returns the ID of compose from directory path in conf.target_dir.`
		`@@ -163,6 +190,15 @@`
		`"""`
		`log.info("Checking for expired composes")`

		`+ # We store all the compose directories to remove in this list and`
		`+ # removes them all in the end of this method.`
		`+ # The reason to do it this way is that if "conf.pungi_runroot_enabled"`
		`+ # is True, we spawn Koji runroot task which calls "rm" commands for`
		`+ # every compose directory we want to remove. We need a list`
		`+ # of directories to be able to generate that set of rm commands later`
		`+ # if neede and if not, just remove the directories by this thread.`
		`+ compose_dirs_to_remove = []`
		`+`
		`composes = Compose.composes_to_expire()`
		`for compose in composes:`
		`log.info("%r: Removing compose", compose)`
		`@@ -170,7 +206,7 @@`
		`compose.time_removed = datetime.utcnow()`
		`db.session.commit()`
		`if not compose.reused_id:`
		`- self._remove_compose_dir(compose.toplevel_dir)`
		`+ compose_dirs_to_remove.append(str(compose.toplevel_dir))`

		`# In case of ODCS error, there might be left-over directories`
		`# belonging to already expired composes. Try to find them in the`
		`@@ -197,16 +233,34 @@`
		`if not composes:`
		`log.info("Removing data of compose %d - it is not in "`
		`"database: %s", compose_id, path)`
		`- self._remove_compose_dir(path)`
		`+ compose_dirs_to_remove.append(path)`
		`continue`

		`compose = composes[0]`
		`if compose.state == COMPOSE_STATES["removed"]:`
		`log.info("%r: Removing data of compose - it has already "`
		`"expired some time ago: %s", compose_id, path)`
		`- self._remove_compose_dir(path)`
		`+ compose_dirs_to_remove.append(path)`
		`continue`

		`+ koji_cmds = []`
		`+ for compose_dir in compose_dirs_to_remove:`
		`+ if conf.pungi_runroot_enabled:`
		`+ cmds = self._get_remove_compose_dir_cmds(compose_dir)`
		`+ if cmds:`
		`+ koji_cmds += cmds`
		`+ koji_cmds += ["&&"]`
		`+ else:`
		`+ self._remove_compose_dir(compose_dir)`
		`+`
		`+ # In case we have some commands to run in Koji, spawn the Koji runroot`
		`+ # task and wait for result.`
		`+ if koji_cmds:`
		`+ # Remove last "&&"`
		`+ koji_cmds = koji_cmds[:-1]`
		`+ koji_session = odcs.server.utils.make_koji_session()`
		`+ odcs.server.utils.run_koji_runroot(koji_session, koji_cmds)`
		`+`

		`def create_koji_session():`
		`"""`

server/odcs/server/pungi.py

file modified

+4 -69

		`@@ -25,8 +25,6 @@`
		`import shutil`
		`import tempfile`
		`import jinja2`
		`- import koji`
		`- import munch`
		`import time`
		`import random`
		`import string`
		`@@ -35,7 +33,8 @@`
		`from odcs.server import conf, log`
		`from odcs.server import comps`
		`from odcs.common.types import PungiSourceType, COMPOSE_RESULTS`
		`- from odcs.server.utils import makedirs, download_file`
		`+ from odcs.server.utils import (`
		`+ makedirs, download_file, make_koji_session, run_koji_runroot)`


		`class PungiConfig(object):`
		`@@ -185,47 +184,6 @@`
		`output_path = os.path.join(topdir, "pungi.conf")`
		`download_file(self.pungi_cfg, output_path)`

		`- def make_koji_session(self):`
jkaluza commented 6 years ago `make_koji_session` is just moved to utils.py, so we can use it from other parts of ODCS too.
		`- """`
		`- Creates new KojiSession according to odcs.server.conf, logins to`
		`- Koji using this session and returns it.`
		`- :rtype: koji.KojiSession`
		`- :return: KojiSession`
		`- """`
		`- koji_config = munch.Munch(koji.read_config(`
		`- profile_name=conf.koji_profile,`
		`- user_config=conf.koji_config,`
		`- ))`
		`-`
		`- address = koji_config.server`
		`- authtype = koji_config.authtype`
		`- log.info("Connecting to koji %r with %r." % (address, authtype))`
		`- koji_session = koji.ClientSession(address, opts=koji_config)`
		`- if authtype == "kerberos":`
		`- ccache = getattr(conf, "krb_ccache", None)`
		`- keytab = getattr(conf, "krb_keytab", None)`
		`- principal = getattr(conf, "krb_principal", None)`
		`- log.debug(" ccache: %r, keytab: %r, principal: %r" % (`
		`- ccache, keytab, principal))`
		`- if keytab and principal:`
		`- koji_session.krb_login(`
		`- principal=principal,`
		`- keytab=keytab,`
		`- ccache=ccache,`
		`- )`
		`- else:`
		`- koji_session.krb_login(ccache=ccache)`
		`- elif authtype == "ssl":`
		`- koji_session.ssl_login(`
		`- os.path.expanduser(koji_config.cert),`
		`- None,`
		`- os.path.expanduser(koji_config.serverca),`
		`- )`
		`- else:`
		`- raise ValueError("Unrecognized koji authtype %r" % authtype)`
		`-`
		`- return koji_session`
		`-`
		`def get_pungi_cmd(self, conf_topdir, targetdir):`
		`"""`
		`Returns list with pungi command line arguments needed to generate`
		`@@ -305,7 +263,7 @@`
		`makedirs(conf_topdir)`
		`self._write_cfgs(conf_topdir)`

		`- koji_session = self.make_koji_session()`
		`+ koji_session = make_koji_session()`
		`serverdir = self.upload_files_to_koji(koji_session, conf_topdir)`

		`# TODO: Copy keytab from secret repo and generate koji profile.`
		`@@ -313,30 +271,7 @@`
		`cmd += ["cp", "/mnt/koji/work/%s/*" % serverdir, ".", "&&"]`
		`cmd += self.get_pungi_cmd("./", conf.pungi_runroot_target_dir)`

		`- kwargs = {`
jkaluza commented 6 years ago This is just moved to utils.py as `run_koji_runroot`, so we can use it from other parts of ODCS too.
		`- 'channel': conf.pungi_parent_runroot_channel,`
		`- 'packages': conf.pungi_parent_runroot_packages,`
		`- 'mounts': conf.pungi_parent_runroot_mounts,`
		`- 'weight': conf.pungi_parent_runroot_weight`
		`- }`
		`-`
		`- task_id = koji_session.runroot(`
		`- conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch,`
		`- " ".join(cmd), **kwargs)`
		`-`
		`- while True:`
		`- # wait for the task to finish`
		`- if koji_session.taskFinished(task_id):`
		`- break`
		`- log.info("Waiting for Koji runroot task %r to finish...", task_id)`
		`- time.sleep(60)`
		`-`
		`- info = koji_session.getTaskInfo(task_id)`
		`- if info is None:`
		`- raise RuntimeError("Cannot get status of Koji task %r" % task_id)`
		`- state = koji.TASK_STATES[info['state']]`
		`- if state in ('FAILED', 'CANCELED'):`
		`- raise RuntimeError("Koji runroot task %r failed." % task_id)`
		`+ run_koji_runroot(koji_session, cmd)`

		`def run(self):`
		`"""`

server/odcs/server/utils.py

file modified

+75

		`@@ -26,11 +26,86 @@`
		`import time`
		`import subprocess`
		`import requests`
		`+ import munch`
		`+ import koji`
		`from distutils.spawn import find_executable`

		`from odcs.server import conf, log`


		`+ def make_koji_session():`
		`+ """`
		`+ Creates new KojiSession according to odcs.server.conf, logins to`
		`+ Koji using this session and returns it.`
		`+ :rtype: koji.KojiSession`
		`+ :return: KojiSession`
		`+ """`
		`+ koji_config = munch.Munch(koji.read_config(`
		`+ profile_name=conf.koji_profile,`
		`+ user_config=conf.koji_config,`
		`+ ))`
		`+`
		`+ address = koji_config.server`
		`+ authtype = koji_config.authtype`
		`+ log.info("Connecting to koji %r with %r." % (address, authtype))`
		`+ koji_session = koji.ClientSession(address, opts=koji_config)`
		`+ if authtype == "kerberos":`
		`+ ccache = getattr(conf, "krb_ccache", None)`
		`+ keytab = getattr(conf, "krb_keytab", None)`
		`+ principal = getattr(conf, "krb_principal", None)`
		`+ log.debug(" ccache: %r, keytab: %r, principal: %r" % (`
		`+ ccache, keytab, principal))`
		`+ if keytab and principal:`
		`+ koji_session.krb_login(`
		`+ principal=principal,`
		`+ keytab=keytab,`
		`+ ccache=ccache,`
		`+ )`
		`+ else:`
		`+ koji_session.krb_login(ccache=ccache)`
		`+ elif authtype == "ssl":`
		`+ koji_session.ssl_login(`
		`+ os.path.expanduser(koji_config.cert),`
		`+ None,`
		`+ os.path.expanduser(koji_config.serverca),`
		`+ )`
		`+ else:`
		`+ raise ValueError("Unrecognized koji authtype %r" % authtype)`
		`+`
		`+ return koji_session`
		`+`
		`+`
		`+ def run_koji_runroot(koji_session, cmd):`
		`+ """`
		`+ Runs the cmd defined as list of commands in Koji runroot and waits for the`
		`+ task to finish. Raises RuntimeError in case the task execution failed.`
		`+ """`
		`+ kwargs = {`
		`+ 'channel': conf.pungi_parent_runroot_channel,`
		`+ 'packages': conf.pungi_parent_runroot_packages,`
		`+ 'mounts': conf.pungi_parent_runroot_mounts,`
		`+ 'weight': conf.pungi_parent_runroot_weight`
		`+ }`
		`+`
		`+ task_id = koji_session.runroot(`
		`+ conf.pungi_parent_runroot_tag, conf.pungi_parent_runroot_arch,`
		`+ " ".join(cmd), **kwargs)`
		`+`
		`+ while True:`
		`+ # wait for the task to finish`
		`+ if koji_session.taskFinished(task_id):`
		`+ break`
		`+ log.info("Waiting for Koji runroot task %r to finish...", task_id)`
		`+ time.sleep(60)`
		`+`
		`+ info = koji_session.getTaskInfo(task_id)`
		`+ if info is None:`
		`+ raise RuntimeError("Cannot get status of Koji task %r" % task_id)`
		`+ state = koji.TASK_STATES[info['state']]`
		`+ if state in ('FAILED', 'CANCELED'):`
		`+ raise RuntimeError("Koji runroot task %r failed." % task_id)`
		`+`
		`+`
		`def download_file(url, output_path):`
		`"""`
		Downloads file from URL `url` to `output_path`.

server/tests/test_pungi.py

file modified

+1 -1

		`@@ -201,7 +201,7 @@`
		`self.config_patcher.patch('pungi_runroot_target_dir_url', 'http://kojipkgs.fedoraproject.org/compose/odcs')`
		`self.config_patcher.start()`

		`- self.patch_make_koji_session = patch("odcs.server.pungi.Pungi.make_koji_session")`
		`+ self.patch_make_koji_session = patch("odcs.server.pungi.make_koji_session")`
		`self.make_koji_session = self.patch_make_koji_session.start()`
		`self.koji_session = MagicMock()`
		`self.koji_session.runroot.return_value = 123`

server/tests/test_remove_expired_composes_thread.py

file modified

+149 -2

		`@@ -21,17 +21,18 @@`
		`# Written by Jan Kaluza <jkaluza@redhat.com>`

		`from odcs.server import db, conf`
		`+ import odcs.server.backend`
		`from odcs.server.models import Compose`
		`from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS`
		`from odcs.server.backend import RemoveExpiredComposesThread`
		`from odcs.server.pungi import PungiSourceType`
		`from datetime import datetime, timedelta`

		`- from .utils import ModelsBaseTest`
		`+ from .utils import ModelsBaseTest, ConfigPatcher`

		`import os`
		`import mock`
		`- from mock import patch`
		`+ from mock import patch, MagicMock`


		`class TestRemoveExpiredComposesThread(ModelsBaseTest):`
		`@@ -211,3 +212,149 @@`
		`self.thread._remove_compose_dir(toplevel_dir)`
		`unlink.assert_not_called()`
		`rmtree.assert_called_once()`
		`+`
		`+`
		`+ class TestRemoveExpiredComposesThreadPungiRunroot(ModelsBaseTest):`
		`+ maxDiff = None`
		`+`
		`+ def setUp(self):`
		`+ super(TestRemoveExpiredComposesThreadPungiRunroot, self).setUp()`
		`+`
		`+ compose = Compose.create(`
		`+ db.session, "unknown", PungiSourceType.MODULE, "testmodule-master",`
		`+ COMPOSE_RESULTS["repository"], 60)`
		`+ db.session.add(compose)`
		`+ db.session.commit()`
		`+`
		`+ self.config_patcher = ConfigPatcher(odcs.server.backend.conf)`
		`+ self.config_patcher.patch("pungi_runroot_enabled", True)`
		`+ self.config_patcher.start()`
		`+`
		`+ self.thread = RemoveExpiredComposesThread()`
		`+`
		`+ self.patch_make_koji_session = patch("odcs.server.utils.make_koji_session")`
		`+ self.make_koji_session = self.patch_make_koji_session.start()`
		`+ self.koji_session = MagicMock()`
		`+ self.make_koji_session.return_value = self.koji_session`
		`+`
		`+ self.patch_run_koji_runroot = patch("odcs.server.utils.run_koji_runroot")`
		`+ self.run_koji_runroot = self.patch_run_koji_runroot.start()`
		`+`
		`+ def tearDown(self):`
		`+ super(TestRemoveExpiredComposesThreadPungiRunroot, self).tearDown()`
		`+ self.config_patcher.stop()`
		`+ self.patch_make_koji_session.stop()`
		`+`
		`+ def _mock_glob(self, glob, dirs):`
		`+ glob_ret_values = [[], []]`
		`+ for d in dirs:`
		`+ path = os.path.join(conf.target_dir, d)`
		`+ if d.startswith("latest-"):`
		`+ glob_ret_values[0].append(path)`
		`+ else:`
		`+ glob_ret_values[1].append(path)`
		`+ glob.side_effect = glob_ret_values`
		`+`
		`+ @patch("os.path.isdir")`
		`+ @patch("glob.glob")`
		`+ def test_remove_left_composes(self, glob, isdir):`
		`+ isdir.return_value = True`
		`+ self._mock_glob(glob, ["latest-odcs-96-1", "odcs-96-1-20171005.n.0"])`
		`+ self.thread.do_work()`
		`+`
		`+ self.run_koji_runroot.assert_not_called()`
		`+`
		`+ @patch("os.path.isdir")`
		`+ @patch("glob.glob")`
		`+ def test_remove_left_composes_not_dir(`
		`+ self, glob, isdir):`
		`+ isdir.return_value = False`
		`+ self._mock_glob(glob, ["latest-odcs-96-1"])`
		`+ self.thread.do_work()`
		`+ self.run_koji_runroot.assert_not_called()`
		`+`
		`+ @patch("os.path.isdir")`
		`+ @patch("glob.glob")`
		`+ def test_remove_left_composes_wrong_dir(`
		`+ self, glob, isdir):`
		`+ isdir.return_value = True`
		`+ self._mock_glob(glob, ["latest-odcs-", "odcs-", "odcs-abc"])`
		`+ self.thread.do_work()`
		`+ self.run_koji_runroot.assert_not_called()`
		`+`
		`+ @patch("os.path.isdir")`
		`+ @patch("glob.glob")`
		`+ def test_remove_left_composes_valid_compose(`
		`+ self, glob, isdir):`
		`+ isdir.return_value = True`
		`+ self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"])`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ c.state = COMPOSE_STATES["done"]`
		`+ db.session.add(c)`
		`+ db.session.commit()`
		`+ self.thread.do_work()`
		`+ self.run_koji_runroot.assert_not_called()`
		`+`
		`+ @patch("os.path.isdir")`
		`+ @patch("glob.glob")`
		`+ @patch("os.path.realpath")`
		`+ @patch("os.path.exists")`
		`+ def test_remove_left_composes_expired_compose(`
		`+ self, exists, realpath, glob, isdir):`
		`+ exists.return_value = True`
		`+ realpath.return_value = "/odcs-real"`
		`+ isdir.return_value = True`
		`+ self._mock_glob(glob, ["latest-odcs-1-1", "odcs-1-1-2017.n.0"])`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ c.state = COMPOSE_STATES["removed"]`
		`+ db.session.add(c)`
		`+ db.session.commit()`
		`+ self.thread.do_work()`
		`+ self.run_koji_runroot.assert_called_once_with(`
		`+ self.koji_session,`
		`+ ['rm', '-f', os.path.join(conf.target_dir, 'latest-odcs-1-1'), '&&',`
		`+ 'rm', '-rf', '/odcs-real', '&&',`
		`+ 'rm', '-f', os.path.join(conf.target_dir, 'odcs-1-1-2017.n.0'), '&&',`
		`+ 'rm', '-rf', '/odcs-real'])`
		`+`
		`+ @patch("os.path.realpath")`
		`+ @patch("os.path.exists")`
		`+ def test_remove_compose_dir_symlink(`
		`+ self, exists, realpath):`
		`+ exists.return_value = True`
		`+ toplevel_dir = "/odcs"`
		`+ realpath.return_value = "/odcs-real"`
		`+`
		`+ cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)`
		`+ self.assertEqual(`
		`+ cmds, ['rm', '-f', '/odcs', '&&', 'rm', '-rf', '/odcs-real'])`
		`+`
		`+ @patch("shutil.rmtree")`
		`+ @patch("os.unlink")`
		`+ @patch("os.path.realpath")`
		`+ @patch("os.path.exists")`
		`+ def test_remove_compose_dir_broken_symlink(`
		`+ self, exists, realpath, unlink, rmtree):`
		`+ def mocked_exists(p):`
		`+ return p != "/odcs-real"`
		`+ exists.side_effect = mocked_exists`
		`+ toplevel_dir = "/odcs"`
		`+ realpath.return_value = "/odcs-real"`
		`+`
		`+ cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)`
		`+ self.assertEqual(`
		`+ cmds, ['rm', '-f', '/odcs'])`
		`+`
		`+ @patch("shutil.rmtree")`
		`+ @patch("os.unlink")`
		`+ @patch("os.path.realpath")`
		`+ @patch("os.path.exists")`
		`+ def test_remove_compose_dir_real_dir(`
		`+ self, exists, realpath, unlink, rmtree):`
		`+ exists.return_value = True`
		`+ toplevel_dir = "/odcs"`
		`+ realpath.return_value = "/odcs"`
		`+`
		`+ cmds = self.thread._get_remove_compose_dir_cmds(toplevel_dir)`
		`+ self.assertEqual(`
		`+ cmds, ['rm', '-rf', '/odcs'])`

jkaluza commented 6 years ago

This is needed for ODCS koji runroog pungi backend. We will generate composes to read-only /mnt/koji and therefore we need to run the tasks to remove the old composes there using the koji runroot too.

jkaluza commented on line 24 of server/odcs/server/pungi.py 6 years ago

make_koji_session is just moved to utils.py, so we can use it from other parts of ODCS too.

jkaluza commented on line 81 of server/odcs/server/pungi.py 6 years ago

This is just moved to utils.py as run_koji_runroot, so we can use it from other parts of ODCS too.

rebased onto c4078c1

6 years ago

jkaluza commented 6 years ago

We have in the end decided that from security reasons the data on koji storage will be removed by separate external process not running on the same machine as ODCS.

Pull-Request has been closed by jkaluza

6 years ago