From a018f43e18f8ec3dbad53feafc029cac534593f2 Mon Sep 17 00:00:00 2001 From: Haibo Lin Date: Mar 12 2021 07:22:10 +0000 Subject: Reuse in-progress composes Previously only finished composes can be reused, with this patch, in-progress composes also could be reused and this could be useful especially for large composes. JIRA: RHELCMP-4000 Signed-off-by: Haibo Lin --- diff --git a/server/odcs/server/backend.py b/server/odcs/server/backend.py index 5314783..75365c9 100644 --- a/server/odcs/server/backend.py +++ b/server/odcs/server/backend.py @@ -517,8 +517,9 @@ def get_reusable_compose(compose): composes = ( db.session.query(Compose) .filter( - Compose.state == COMPOSE_STATES["done"], + Compose.state.in_([COMPOSE_STATES["done"], COMPOSE_STATES["generating"]]), Compose.source_type == compose.source_type, + Compose.id != compose.id, ) .all() ) @@ -910,26 +911,29 @@ def generate_pungi_compose(compose): if compose.source_type != PungiSourceType.RAW_CONFIG: _write_repo_file(compose) - # Raises an exception if invalid - validate_pungi_compose(compose) - - # Generate symlinks pointing to latest build of raw_config compose. - if compose.source_type == PungiSourceType.RAW_CONFIG: - generate_compose_symlink(compose) - if compose.compose_type in ["production", "development"]: - pungi_logs = PungiLogs(compose) - config_dump = pungi_logs.get_config_dump() - if not config_dump: - msg = "%r: Cannot load Pungi config dump." % compose - log.error(msg) - raise RuntimeError(msg) - compose.pungi_config_dump = config_dump - - # If there is no exception generated by the pungi.run() and if - # validation didn't fail, then we know the compose has been - # successfully generated. - compose.transition(COMPOSE_STATES["done"], "Compose is generated successfully") - log.info("%r: Compose done", compose) + # Generate symlinks pointing to latest build of raw_config compose. + if compose.source_type == PungiSourceType.RAW_CONFIG: + generate_compose_symlink(compose) + if compose.compose_type in ["production", "development"]: + pungi_logs = PungiLogs(compose) + config_dump = pungi_logs.get_config_dump() + if not config_dump: + msg = "%r: Cannot load Pungi config dump." % compose + log.error(msg) + raise RuntimeError(msg) + compose.pungi_config_dump = config_dump + + if compose_to_reuse and compose_to_reuse.state == COMPOSE_STATES["generating"]: + log.info("%r: Reuse compose %r which is generating", compose, compose_to_reuse) + else: + # Raises an exception if invalid + validate_pungi_compose(compose) + + # If there is no exception generated by the pungi.run() and if + # validation didn't fail, then we know the compose has been + # successfully generated. + compose.transition(COMPOSE_STATES["done"], "Compose is generated successfully") + log.info("%r: Compose done", compose) if not compose_to_reuse: koji_tag_cache.update_cache(compose) diff --git a/server/odcs/server/models.py b/server/odcs/server/models.py index ad09d00..8f8fc42 100644 --- a/server/odcs/server/models.py +++ b/server/odcs/server/models.py @@ -538,6 +538,8 @@ class Compose(ODCSBase): if to_state in (COMPOSE_STATES["done"], COMPOSE_STATES["failed"]): ttl = self.time_to_expire - self.time_submitted self.time_to_expire = (happen_on or datetime.utcnow()) + ttl + for reusing in self.get_reusing_composes(): + reusing.transition(to_state, reason, happen_on=self.time_done) db.session.commit() diff --git a/server/tests/test_backend.py b/server/tests/test_backend.py index 9d2ccda..ececa21 100644 --- a/server/tests/test_backend.py +++ b/server/tests/test_backend.py @@ -527,6 +527,44 @@ class TestBackend(ModelsBaseTest): @patch("odcs.server.backend.koji_get_inherited_tags") @patch("odcs.server.backend.create_koji_session") + def test_get_reusable_tag_compose_in_generating( + self, create_koji_session, koji_get_inherited_tags + ): + koji_get_inherited_tags.return_value = ["foo", "bar"] + koji_session = MagicMock() + create_koji_session.return_value = koji_session + koji_session.tagChangedSinceEvent.return_value = False + + old_c = Compose.create( + db.session, + "me", + PungiSourceType.KOJI_TAG, + "foo", + COMPOSE_RESULTS["repository"], + 3600, + packages="ed", + ) + old_c.koji_event = 1 + old_c.state = COMPOSE_STATES["generating"] + c = Compose.create( + db.session, + "me", + PungiSourceType.KOJI_TAG, + "foo", + COMPOSE_RESULTS["repository"], + 3600, + packages="ed", + ) + c.koji_event = 2 + db.session.add(old_c) + db.session.add(c) + db.session.commit() + + reused_c = get_reusable_compose(c) + self.assertEqual(reused_c, old_c) + + @patch("odcs.server.backend.koji_get_inherited_tags") + @patch("odcs.server.backend.create_koji_session") def test_get_reusable_tag_compose_none_koji_event( self, create_koji_session, koji_get_inherited_tags ): diff --git a/server/tests/test_models.py b/server/tests/test_models.py index db1b0b2..ea4ac68 100644 --- a/server/tests/test_models.py +++ b/server/tests/test_models.py @@ -337,3 +337,19 @@ class ComposeModel(ModelsBaseTest): in_five_minutes = now + timedelta(seconds=300) self.c1.transition(COMPOSE_STATES["generating"], "it started", in_five_minutes) assert self.c1.time_started == in_five_minutes + + def test_transition_to_done_updates_reusing_compose(self): + reason = "it's finished" + self.c3.transition(COMPOSE_STATES["done"], reason) + self.assertEqual(self.c1.state, COMPOSE_STATES["done"]) + self.assertEqual(self.c1.state_reason, reason) + self.assertEqual(self.c2.state, COMPOSE_STATES["done"]) + self.assertEqual(self.c2.state_reason, reason) + + def test_transition_to_failed_updates_reusing_compose(self): + reason = "it's failed" + self.c3.transition(COMPOSE_STATES["failed"], reason) + self.assertEqual(self.c1.state, COMPOSE_STATES["failed"]) + self.assertEqual(self.c1.state_reason, reason) + self.assertEqual(self.c2.state, COMPOSE_STATES["failed"]) + self.assertEqual(self.c2.state_reason, reason)