From 2e0500d7ce0d9becb3f59186c1f007898aba76ad Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Jun 19 2023 15:54:42 +0000 Subject: Schedule tests for ELN composes I promised to do this for @dcavalca , so here it is. Needs the most recent fedfind version (which I promise to release in just a second). Signed-off-by: Adam Williamson --- diff --git a/sample-configs/fedora_openqa_scheduler.stg.toml b/sample-configs/fedora_openqa_scheduler.stg.toml index 11650ab..04f9417 100644 --- a/sample-configs/fedora_openqa_scheduler.stg.toml +++ b/sample-configs/fedora_openqa_scheduler.stg.toml @@ -48,7 +48,8 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.coreos.build.state.change", "org.fedoraproject.prod.bodhi.update.request.testing", "org.fedoraproject.prod.bodhi.update.edit", - "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"] + "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete", + "org.fedoraproject.prod.odcs.compose.state-changed"] # need this to receive messages from ZMQ->AMQP bridge [[bindings]] queue = "00000000-0000-0000-0000-000000000000" @@ -58,7 +59,8 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.coreos.build.state.change", "org.fedoraproject.prod.bodhi.update.request.testing", "org.fedoraproject.prod.bodhi.update.edit", - "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"] + "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete", + "org.fedoraproject.prod.odcs.compose.state-changed"] [consumer_config] # host to schedule tests on diff --git a/sample-configs/fedora_openqa_scheduler.toml b/sample-configs/fedora_openqa_scheduler.toml index b4935b9..a501d6b 100644 --- a/sample-configs/fedora_openqa_scheduler.toml +++ b/sample-configs/fedora_openqa_scheduler.toml @@ -43,7 +43,8 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.coreos.build.state.change", "org.fedoraproject.prod.bodhi.update.request.testing", "org.fedoraproject.prod.bodhi.update.edit", - "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"] + "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete", + "org.fedoraproject.prod.odcs.compose.state-changed"] # need this to receive messages from ZMQ->AMQP bridge [[bindings]] queue = "00000000-0000-0000-0000-000000000000" @@ -52,7 +53,8 @@ routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.coreos.build.state.change", "org.fedoraproject.prod.bodhi.update.request.testing", "org.fedoraproject.prod.bodhi.update.edit", - "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"] + "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete", + "org.fedoraproject.prod.odcs.compose.state-changed"] [consumer_config] # host to schedule tests on diff --git a/src/fedora_openqa/config.py b/src/fedora_openqa/config.py index 014f7ad..17e19a2 100644 --- a/src/fedora_openqa/config.py +++ b/src/fedora_openqa/config.py @@ -117,6 +117,22 @@ WANTED = [ }, { "match": { + "subvariant": "BaseOS", + "type": "dvd", + "format": "iso", + "arch": "x86_64", + }, + }, + { + "match": { + "subvariant": "BaseOS", + "type": "boot", + "format": "iso", + "arch": "x86_64", + }, + }, + { + "match": { "subvariant": "Cloud_Base", "type": "qcow2", "format": "qcow2", diff --git a/src/fedora_openqa/consumer.py b/src/fedora_openqa/consumer.py index 70dc754..1b7dbd0 100755 --- a/src/fedora_openqa/consumer.py +++ b/src/fedora_openqa/consumer.py @@ -68,6 +68,8 @@ class OpenQAScheduler(object): return self._consume_compose(message.body) elif 'coreos' in message.topic: return self._consume_fcosbuild(message.body) + elif 'odcs' in message.topic: + return self._consume_odcs(message.body) elif 'bodhi.update.status.testing' in message.topic: if message.body.get("re-trigger"): return self._consume_retrigger(message.body) @@ -117,6 +119,23 @@ class OpenQAScheduler(object): else: self.logger.debug("No openQA jobs run, likely already tested") + def _compose_schedule(self, location, description): + """Shared schedule, log, return code for _consume_compose and + _consume_odcs. + """ + self.logger.info("Scheduling openQA jobs for compose %s", description) + try: + # pylint: disable=no-member + (compose, jobs) = schedule.jobs_from_compose(location, openqa_hostname=self.openqa_hostname) + except schedule.TriggerException as err: + self.logger.warning("No openQA jobs run! %s", err) + return + if jobs: + self.logger.info("openQA jobs run on compose %s: " + "%s", compose, ' '.join(str(job) for job in jobs)) + else: + self.logger.warning("No openQA jobs run!") + def _consume_retrigger(self, body): """ Consume a re-trigger request messages. These are published @@ -154,18 +173,7 @@ class OpenQAScheduler(object): if 'FINISHED' in status and location: # We have a complete pungi4 compose - self.logger.info("Scheduling openQA jobs for compose %s", compstr) - try: - # pylint: disable=no-member - (compose, jobs) = schedule.jobs_from_compose(location, openqa_hostname=self.openqa_hostname) - except schedule.TriggerException as err: - self.logger.warning("No openQA jobs run! %s", err) - return - if jobs: - self.logger.info("openQA jobs run on compose %s: " - "%s", compose, ' '.join(str(job) for job in jobs)) - else: - self.logger.warning("No openQA jobs run!") + self._compose_schedule(location, compstr) def _consume_fcosbuild(self, body): """Consume an FCOS build state change message.""" @@ -183,6 +191,25 @@ class OpenQAScheduler(object): else: self.logger.info("No openQA jobs run!") + def _consume_odcs(self, body): + """Consume an ODCS compose state change message.""" + # this is intentionally written to blow up if required info is + # missing from the message, as that would likely indicate a + # message format change and we'd need to handle that + if body["compose"]["state"] != 2: + self.logger.debug("Not a 'finished success' message, ignoring") + return + if body["event"] != "state-changed": + self.logger.debug("Not a state change message, ignoring") + return + cid = body["compose"].get("pungi_compose_id", "") + if not cid.startswith("Fedora-ELN"): + self.logger.debug("Not an ELN compose, ignoring") + return + url = body["compose"]["toplevel_url"] + desc = f"{cid} at {url}" + self._compose_schedule(url, desc) + def _consume_update(self, body, force=True): """ Given a message containing an "update" dict, decide whether diff --git a/src/fedora_openqa/schedule.py b/src/fedora_openqa/schedule.py index a7e2a02..d05ba33 100644 --- a/src/fedora_openqa/schedule.py +++ b/src/fedora_openqa/schedule.py @@ -352,8 +352,8 @@ def jobs_from_compose(location, wanted=None, force=False, extraparams=None, open # image. We want to schedule them when testing 'full' composes # that have a generic tree, but not when testing 'partial' # composes that only produce images. fedfind's https_url_generic - # is a good indicator of this. - if rel.https_url_generic: + # is a good indicator of this. we also don't schedule for ELN + if rel.https_url_generic and rel.release.lower() != "eln": images.extend( [ ("Workstation-upgrade", "x86_64", {}, "Workstation", "upgrade"), @@ -399,7 +399,12 @@ def jobs_from_compose(location, wanted=None, force=False, extraparams=None, open # don't do this for post-release nightlies that are *always* # candidates, though # using getattr as the 'respin' composes don't have these attrs - if getattr(rel, 'type', '') == 'production' and getattr(rel, 'product', '') == 'Fedora' and jobs: + if ( + jobs + and getattr(rel, 'type', '') == 'production' + and getattr(rel, 'product', '') == 'Fedora' + and getattr(rel, 'release', '').lower() != 'eln' + ): client = OpenQA_Client(openqa_hostname) # we expect group 1 to be 'fedora'. I think this is reliable. params = {'text': "tag:{0}:important:candidate".format(rel.cid)} diff --git a/tests/test_consumers.py b/tests/test_consumers.py index 4722e70..974dc15 100644 --- a/tests/test_consumers.py +++ b/tests/test_consumers.py @@ -374,6 +374,33 @@ ELNREADY = Message( } ) +# ELN successful compose message. Tests should run +ELNCOMPOSE = Message( + topic="org.fedoraproject.prod.odcs.compose.state-changed", + body={ + "compose": { + "pungi_compose_id": "Fedora-ELN-20230619.1", + "state": 2, + "state_name": "done", + "state_reason": "Compose is generated successfully", + "toplevel_url": "https://odcs.fedoraproject.org/composes/odcs-28282", + }, + "event": "state-changed" + } +) + +# ELN compose message which wasn't for a state change +ELNCOMPOSENOTSC = copy.deepcopy(ELNCOMPOSE) +ELNCOMPOSENOTSC.body["event"] = "someotherevent" + +# ELN compose message where the state isn't 'done' +ELNCOMPOSENOTDONE = copy.deepcopy(ELNCOMPOSE) +ELNCOMPOSENOTDONE.body["compose"]["state"] = 3 + +# ODCS compose message which isn't for ELN +ODCSCOMPOSENOTELN = copy.deepcopy(ELNCOMPOSE) +ODCSCOMPOSENOTELN.body["compose"]["pungi_compose_id"] = "odcs-28283-1-20230619.t.0" + # Successful FCOS build message FCOSBUILD = Message( topic="org.fedoraproject.prod.coreos.build.state.change", @@ -504,6 +531,10 @@ class TestConsumers: (NONRETRIGGER, True, None, "FEDORA-2023-1f3e17882f", "39"), (NONFRETRIGGER, True, False, None, None), (ELNREADY, True, False, None, None), + (ELNCOMPOSE, True, None, None, None), + (ELNCOMPOSENOTSC, True, False, None, None), + (ELNCOMPOSENOTDONE, True, False, None, None), + (ODCSCOMPOSENOTELN, True, False, None, None), (FCOSBUILD, False, None, None, None), (FCOSBUILDNOTF, False, False, None, None), (FCOSBUILDNOTS, False, False, None, None)