PR#398: Skip composeinfo-base.json creation if CTS is configured. - odcs

odcs

#398 Skip composeinfo-base.json creation if CTS is configured.

Merged 3 years ago by lsedlar. Opened 3 years ago by jkaluza.

jkaluza/odcs cts into master

Skip composeinfo-base.json creation if CTS is configured.

Jan Kaluza • 3 years ago

0bc8992

server/odcs/server/pungi.py

file modified

+53 -4

		`@@ -27,6 +27,7 @@`
		`import tempfile`
		`import jinja2`
		`import time`
		`+ import threading`
		`from productmd.composeinfo import ComposeInfo`
		`from kobo.conf import PyConfigParser`

		`@@ -298,6 +299,37 @@`
		`self._write(os.path.join(topdir, "comps.xml"), comps_cfg)`


		`+ class ReadComposeIdThread(threading.Thread):`
		`+ def __init__(self, compose):`
		`+ threading.Thread.__init__(self)`
		`+ self._stop_event = threading.Event()`
		`+ self.compose = compose`
		`+`
		`+ def stop(self):`
		`+ self._stop_event.set()`
		`+`
		`+ def run(self):`
		`+ p = os.path.join(self.compose.toplevel_dir, "work", "global", "composeinfo-base.json")`
		`+ while not self._stop_event.is_set():`
		`+ time.sleep(1)`
		`+`
		`+ # File does not exist yet.`
		`+ if not os.path.exists(p):`
		`+ continue`
		`+`
		`+ ci = ComposeInfo()`
		`+ try:`
		`+ ci.load(p)`
		`+ except Exception:`
		`+ # This should happen only if file exists, but the data is not`
		`+ # written yet.`
		`+ continue`
		`+`
		`+ self.compose.pungi_compose_id = ci.compose.id`
		`+ db.session.commit()`
		`+ break`
		`+`
		`+`
		`class Pungi(object):`
		`def __init__(self, compose_id, pungi_cfg, koji_event=None, old_compose=None):`
		`self.compose_id = compose_id`
		`@@ -364,15 +396,18 @@`
		`pungi_cmd += ["--old-composes", self.old_compose]`
		`return pungi_cmd`

		`- def _prepare_compose_dir(self, compose, conf_topdir):`
		`+ def _prepare_compose_dir(self, compose, conf):`
		`"""`
		`Creates the compose directory and returns the full path to it.`
		`"""`
		`compose_date = time.strftime("%Y%m%d", time.localtime())`
		`makedirs(compose.toplevel_dir)`

		`- conf = PyConfigParser()`
		`- conf.load_from_file(os.path.join(conf_topdir, "pungi.conf"))`
		`+ # If Compose Tracking Service is configured in the config file,`
		`+ # we skip the ComposeInfo creation completely and instead let`
		`+ # the Pungi to ask CTS to generate unique ComposeInfo.`
		`+ if "cts_url" in conf and "cts_keytab" in conf:`
		`+ return compose.toplevel_dir`

		`ci = ComposeInfo()`
		`ci.release.name = conf["release_name"]`
		`@@ -414,10 +449,16 @@`
		`Runs local Pungi compose.`
		`"""`
		`td = None`
		`+ compose_id_thread = None`
		`try:`
		`td = tempfile.mkdtemp()`
		`self._write_cfgs(td)`
		`- compose_dir = self._prepare_compose_dir(compose, td)`
		`+`
		`+ # Load pungi configuration file.`
		`+ conf = PyConfigParser()`
		`+ conf.load_from_file(os.path.join(td, "pungi.conf"))`
		`+`
		`+ compose_dir = self._prepare_compose_dir(compose, conf)`
		`self.pungi_cfg.validate(td, compose_dir)`
		`pungi_cmd = self.get_pungi_cmd(td, compose, compose_dir)`

		`@@ -426,6 +467,12 @@`
		`# cached locally in the SQLAlchemy.`
		`db.session.commit()`

		`+ # If Compose Tracking Service is configured in the config file,`
		`+ # we need to get the Compose ID from Pungi in separate thread.`
		`+ if "cts_url" in conf and "cts_keytab" in conf:`
		`+ compose_id_thread = ReadComposeIdThread(compose)`
		`+ compose_id_thread.start()`
		`+`
		`log_out_path = os.path.join(compose_dir, "pungi-stdout.log")`
		`log_err_path = os.path.join(compose_dir, "pungi-stderr.log")`

		`@@ -435,6 +482,8 @@`
		`pungi_cmd, cwd=td, timeout=self.pungi_cfg.pungi_timeout,`
		`stdout=log_out, stderr=log_err)`
		`finally:`
		`+ if compose_id_thread:`
		`+ compose_id_thread.stop()`
		`try:`
		`if td is not None:`
		`shutil.rmtree(td)`

server/tests/test_pungi.py

file modified

+59

		`@@ -25,6 +25,7 @@`
		`import tempfile`
		`import unittest`
		`import time`
		`+ from productmd import ComposeInfo`

		`from mock import patch, MagicMock, mock_open, call`
		`from kobo.conf import PyConfigParser`
		`@@ -337,6 +338,12 @@`
		`)`


		`+ class FakePyConfigParser(dict):`
		`+`
		`+ def load_from_file(self, args, *kwargs):`
		`+ pass`
		`+`
		`+`
		`class TestPungi(ModelsBaseTest):`

		`def setUp(self):`
		`@@ -376,6 +383,8 @@`
		`self.compose.compose_type = "test"`
		`self.compose.label = None`

		`+ makedirs(self.compose.toplevel_dir)`
		`+`
		`def tearDown(self):`
		`super(TestPungi, self).tearDown()`

		`@@ -383,6 +392,8 @@`
		`self.patch_makedirs.stop()`
		`self.patch_ci_dump.stop()`

		`+ shutil.rmtree(self.compose.toplevel_dir)`
		`+`
		`@patch("odcs.server.utils.execute_cmd")`
		`def test_pungi_run(self, execute_cmd):`
		`pungi_cfg = PungiConfig("MBS-512", "1", PungiSourceType.MODULE,`
		`@@ -405,6 +416,54 @@`
		`stdout=AnyStringWith("pungi-stdout.log"))`

		`@patch("odcs.server.utils.execute_cmd")`
		`+ @patch("odcs.server.pungi.PyConfigParser")`
		`+ def test_pungi_run_cts(self, py_config_parser, execute_cmd):`
		`+ self.patch_ci_dump.stop()`
		`+ py_config_parser.return_value = FakePyConfigParser({`
		`+ "cts_url": "https://cts.localhost.tld/",`
		`+ "cts_keytab": "/tmp/some.keytab",`
		`+ })`
		`+`
		`+ def fake_execute_cmd(args, *kwargs):`
		+ # Fake `execute_cmd` method which creates composeinfo-base.json file
		`+ # and waits for three seconds to test that ODCS picks up the compose`
		`+ # ID from this file.`
		`+ p = os.path.join(self.compose.toplevel_dir, "work", "global", "composeinfo-base.json")`
		`+ makedirs(os.path.dirname(p))`
		`+ ci = ComposeInfo()`
		`+ ci.compose.id = "Fedora-Rawhide-20200517.n.1"`
		`+ ci.compose.type = "nightly"`
		`+ ci.compose.date = "20200517"`
		`+ ci.compose.respin = 1`
		`+ ci.release.name = "Fedora"`
		`+ ci.release.short = "Fedora"`
		`+ ci.release.version = "Rawhide"`
		`+ ci.release.is_layered = False`
		`+ ci.release.type = "ga"`
		`+ ci.release.internal = False`
		`+ ci.dump(p)`
		`+ time.sleep(3)`
		`+`
		`+ execute_cmd.side_effect = fake_execute_cmd`
		`+`
		`+ pungi_cfg = PungiConfig("MBS-512", "1", PungiSourceType.MODULE,`
		`+ "testmodule:master:1:1")`
		`+ pungi = Pungi(1, pungi_cfg)`
		`+ pungi.run(self.compose)`
		`+`
		`+ self.makedirs.assert_called_with(`
		`+ AnyStringWith("test_composes/odcs-1"))`
		`+`
		`+ execute_cmd.assert_called_once_with(`
		`+ ['pungi-koji', AnyStringWith('pungi.conf'),`
		`+ AnyStringWith('--compose-dir='), '--test'],`
		`+ cwd=AnyStringWith('/tmp/'), timeout=3600,`
		`+ stderr=AnyStringWith("pungi-stderr.log"),`
		`+ stdout=AnyStringWith("pungi-stdout.log"))`
		`+`
		`+ self.assertEqual(self.compose.pungi_compose_id, "Fedora-Rawhide-20200517.n.1")`
		`+`
		`+ @patch("odcs.server.utils.execute_cmd")`
		`def test_pungi_run_compose_type(self, execute_cmd):`
		`for compose_type in [None, "test", "ci", "nightly", "production"]:`
		`self.makedirs.reset_mock()`

jkaluza commented 3 years ago

If cts_url and cts_keytab is defined in the Pungi configuration,
the composeinfo-base.json is not created and instead, the Pungi
is used to generate this file and ODCS reads it in separate thread
to get the Compose.pungi_compose_id.

Signed-off-by: Jan Kaluza jkaluza@redhat.com

rebased onto fc57e88de111490be19d1ac36100c49109525e4b

3 years ago

rebased onto 0bc8992

3 years ago