#94 Handle "re-trigger tests" requests from Bodhi (#91)
Merged 2 years ago by adamwill. Opened 2 years ago by adamwill.

@@ -46,7 +46,8 @@ 

  # we subscribe to prod not staging keys here as per above.

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

-                 "org.fedoraproject.prod.bodhi.update.edit"]

+                 "org.fedoraproject.prod.bodhi.update.edit",

+                 "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

  # need this to receive messages from ZMQ->AMQP bridge

  [[bindings]]

  queue = "00000000-0000-0000-0000-000000000000"
@@ -54,7 +55,8 @@ 

  # we subscribe to prod not staging keys here as per above.

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

-                 "org.fedoraproject.prod.bodhi.update.edit"]

+                 "org.fedoraproject.prod.bodhi.update.edit",

+                 "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

  

  [consumer_config]

  # host to schedule tests on

@@ -41,14 +41,16 @@ 

  exchange = "amq.topic"

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

-                 "org.fedoraproject.prod.bodhi.update.edit"]

+                 "org.fedoraproject.prod.bodhi.update.edit",

+                 "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

  # need this to receive messages from ZMQ->AMQP bridge

  [[bindings]]

  queue = "00000000-0000-0000-0000-000000000000"

  exchange = "zmq.topic"

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

-                 "org.fedoraproject.prod.bodhi.update.edit"]

+                 "org.fedoraproject.prod.bodhi.update.edit",

+                 "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

  

  [consumer_config]

  # host to schedule tests on

file modified
+63 -18
@@ -26,6 +26,7 @@ 

  

  # external imports

  import fedora_messaging.config

+ from openqa_client.client import OpenQA_Client

  

  # internal imports

  from .config import UPDATETL
@@ -63,6 +64,8 @@ 

          """Consume incoming message."""

          if 'pungi' in message.topic:

              return self._consume_compose(message)

+         elif 'bodhi.update.status.testing' in message.topic:

+             return self._consume_retrigger(message)

          elif 'bodhi' in message.topic:

              return self._consume_update(message)

  
@@ -99,6 +102,64 @@ 

  

          return

  

+     def _update_schedule(self, advisory, version, flavors):

+         """

+         Shared schedule, log, return code for _consume_retrigger and

+         _consume_update.

+         """

+         # flavors

+         # being None here results in our desired behaviour (jobs will

+         # be created for *all* flavors)

+         jobs = []

+         # pylint: disable=no-member

+         for arch in self.update_arches:

+             jobs.extend(schedule.jobs_from_update(

+                 advisory, version, flavors=flavors,

+                 openqa_hostname=self.openqa_hostname, force=True, arch=arch))

+         if jobs:

+             self.logger.info("openQA jobs run on update %s: "

+                       "%s", advisory, ' '.join(str(job) for job in jobs))

+         else:

+             self.logger.warning("No openQA jobs run!")

+             return

+ 

+         self.logger.debug("Finished")

+ 

+     def _consume_retrigger(self, message):

+         """Consume a 're-trigger tests' type message."""

+         body = _find_true_body(message)

+         if not body.get("re-trigger"):

+             self.logger.debug("Not a re-trigger request, ignoring")

+             return

+         advisory = body.get("artifact", {}).get("id", "")

+         self.logger.info("Handling test re-trigger request for update %s", advisory)

+         if not advisory.startswith("FEDORA-2"):

+             self.logger.info("Update %s does not look like a Fedora package update, ignoring", advisory)

+             return

+         # there's an extra weird UUID string on the end of the update

+         # ID in these messages for some reason, split it off

+         advisory = "-".join(advisory.split("-")[0:3])

+         # we get the 'dist tag' as the 'version' here, need to trim

+         # the leading "f"

+         version = body.get("artifact", {}).get("release", "")[1:]

+         # these messages do not include critpath status, so we can't

+         # "decide" whether to test the update. instead, as they're

+         # re-trigger messages, we'll just see whether we already

+         # tested the update, and for what flavors if so, and re-test

+         # in that same context

+         client = OpenQA_Client(self.openqa_hostname)

+         build = f"Update-{advisory}"

+         existjobs = client.openqa_request("GET", "jobs", params={"build": build})["jobs"]

+         flavors = [job.get("settings", {}).get("FLAVOR", "") for job in existjobs]

+         # strip the updates- prefix and ignore empty strings

+         flavors = [flavor.split("updates-")[-1] for flavor in flavors if flavor]

+         flavors = set(flavors)

+         if flavors:

+             self.logger.info("Re-running tests for update %s, flavors %s", advisory, ", ".join(flavors))

+             self._update_schedule(advisory, version, flavors)

+         else:

+             self.logger.info("No existing jobs found, so not scheduling any!")

+ 

      def _consume_update(self, message):

          """Consume an 'update' type message."""

          body = _find_true_body(message)
@@ -147,24 +208,8 @@ 

              self.logger.debug("Update is not critical path and no packages in test list, no jobs scheduled")

              return

  

-         # Finally, now we've decided on flavors, run the jobs. flavors

-         # being None here results in our desired behaviour (jobs will

-         # be created for *all* flavors)

-         jobs = []

-         # pylint: disable=no-member

-         for arch in self.update_arches:

-             jobs.extend(schedule.jobs_from_update(

-                 advisory, version, flavors=flavors,

-                 openqa_hostname=self.openqa_hostname, force=True, arch=arch))

-         if jobs:

-             self.logger.info("openQA jobs run on update %s: "

-                       "%s", advisory, ' '.join(str(job) for job in jobs))

-         else:

-             self.logger.warning("No openQA jobs run!")

-             return

- 

-         self.logger.debug("Finished")

-         return

+         # Finally, now we've decided on flavors, run the jobs.

+         self._update_schedule(advisory, version, flavors)

  

  

  # WIKI REPORTER

file modified
+123 -16
@@ -183,6 +183,82 @@ 

  EPELEDIT = copy.deepcopy(CRITPATHEDIT)

  EPELEDIT.body['update']['release']['id_prefix'] = 'FEDORA-EPEL'

  

+ # Bodhi 'update ready for testing' message which is a re-trigger

+ # request for a Fedora update

+ RETRIGGER = Message(

+     topic="org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete",

+     body={

+         "re-trigger": True,

+         "artifact": {

+           "release": "f34",

+           "type": "koji-build-group",

+           "builds": [

+             {

+               "nvr": "gdb-11.1-5.fc34",

+               "task_id": 78709925,

+               "scratch": False,

+               "component": "gdb",

+               "type": "koji-build",

+               "id": 1854854,

+               "issuer": "kevinb"

+             }

+           ],

+           "repository": "https://bodhi.fedoraproject.org/updates/FEDORA-2021-53a7dfa185",

+           "id": "FEDORA-2021-53a7dfa185-bb4a8e2be6997eb20655fa079af87470fe416415"

+         },

+         "contact": {

+           "docs": "https://docs.fedoraproject.org/en-US/ci/",

+           "team": "Fedora CI",

+           "email": "admin@fp.o",

+           "name": "Bodhi"

+         },

+         "version": "0.2.2",

+         "agent": "adamwill",

+         "generated_at": "2021-11-12T22:25:12.966558Z"

+     }

+ )

+ 

+ # Bodhi 'update ready for testing' message which is not a re-trigger

+ # request

+ NONRETRIGGER = copy.deepcopy(RETRIGGER)

+ NONRETRIGGER.body["re-trigger"] = False

+ NONRETRIGGER.body["agent"] = "bodhi"

+ 

+ # Bodhi 'update ready for testing' message which is a re-trigger

+ # request, but not for a regular Fedora package update

+ NONFRETRIGGER = Message(

+     topic="org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete",

+     body={

+         "re-trigger": True,

+         "artifact": {

+           "release": "epel8",

+           "type": "koji-build-group",

+           "builds": [

+             {

+               "nvr": "libpinyin-epel-2.2.0-2.el8",

+               "task_id": 78921389,

+               "scratch": False,

+               "component": "libpinyin-epel",

+               "type": "koji-build",

+               "id": 1856264,

+               "issuer": "tdawson"

+             }

+           ],

+           "repository": "https://bodhi.fedoraproject.org/updates/FEDORA-EPEL-2021-049da15976",

+           "id": "FEDORA-EPEL-2021-049da15976-37a14b6e2476318d8338b1aa4a5a3190428eb328"

+         },

+         "contact": {

+           "docs": "https://docs.fedoraproject.org/en-US/ci/",

+           "team": "Fedora CI",

+           "email": "admin@fp.o",

+           "name": "Bodhi"

+         },

+         "version": "0.2.2",

+         "agent": "tdawson",

+         "generated_at": "2021-11-17T01:41:15.438773Z"

+     }

+ )

+ 

  # initialize a few test consumers with different configs

  PRODCONF = {

      'consumer_config': {
@@ -249,7 +325,7 @@ 

          ]

      )

      @pytest.mark.parametrize(

-         "message,flavors",

+         "message,gotjobs,flavors,advisory,version",

          [

              # For 'flavors': 'False' means no jobs should be created. For

              # compose scheduling, any other value just means "jobs should
@@ -257,27 +333,40 @@ 

              # means "jobs should be created, and this is the expected value

              # of the 'flavors' kwarg to the fake_update call" (remember,

              # None means "run tests for all flavors").

-             (STARTEDCOMPOSE, False),

-             (DOOMEDCOMPOSE, False),

-             (FINISHEDCOMPOSE, True),

-             (FINCOMPLETE, True),

+             # if 'gotjobs' is True, we mock a query of existing jobs to

+             # return some. otherwise, we mock it to return nothing. This

+             # is for testing re-trigger request scheduling in both cases,

+             # it is irrelevant to 'new/edited update' and 'new compose'

+             # message handling.

+             # if "advisory" and/or "version" is a string, we'll check

+             # that value is used in the update scheduling request. This is

+             # for re-trigger request handling, where we do non-trivial

+             # parsing to get those values.

+             (STARTEDCOMPOSE, False, False, None, None),

+             (DOOMEDCOMPOSE, False, False, None, None),

+             (FINISHEDCOMPOSE, False, True, None, None),

+             (FINCOMPLETE, False, True, None, None),

              # for all critpath updates we should schedule for all flavors

-             (CRITPATHCREATE, None),

-             (CRITPATHEDIT, None),

-             (NONCRITCREATE, False),

-             (NONCRITEDIT, False),

-             (EPELCREATE, False),

-             (EPELEDIT, False),

+             (CRITPATHCREATE, False, None, None, None),

+             (CRITPATHEDIT, False, None, None, None),

+             (NONCRITCREATE, False, False, None, None),

+             (NONCRITEDIT, False, False, None, None),

+             (EPELCREATE, False, False, None, None),

+             (EPELEDIT, False, False, None, None),

              # TLCREATE contains only a 'server'-listed package

-             (TLCREATE, {'server', 'server-upgrade'}),

+             (TLCREATE, False, {'server', 'server-upgrade'}, None, None),

              # TLEDIT contains both 'server' and 'workstation-live-iso'-listed

              # packages

-             (TLEDIT, {'server', 'server-upgrade', 'workstation-live-iso'}),

+             (TLEDIT, False, {'server', 'server-upgrade', 'workstation-live-iso'}, None, None),

              # TLALLEDIT contains an 'all flavors'-listed package

-             (TLALLEDIT, None),

+             (TLALLEDIT, False, None, None, None),

+             (RETRIGGER, True, {'server', 'workstation'}, "FEDORA-2021-53a7dfa185", "34"),

+             (RETRIGGER, False, False, None, None),

+             (NONRETRIGGER, True, False, None, None),

+             (NONFRETRIGGER, True, False, None, None),

          ]

      )

-     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, flavors):

+     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, gotjobs, flavors, advisory, version):

          """Test the job scheduling consumers do their thing. The

          parametrization pairs are:

          1. (consumer, expected openQA hostname)
@@ -286,7 +375,21 @@ 

          and that the hostname is as expected. If jobs aren't expected,

          we check the schedule function was not hit.

          """

-         consumer(message)

+         with mock.patch("openqa_client.client.OpenQA_Client.openqa_request", autospec=True) as fake_request:

+             if gotjobs:

+                 # we mock four existing jobs across two flavors, to

+                 # make sure we don't duplicate flavors in the request

+                 fake_request.return_value = {

+                     'jobs': [

+                         {'id': 1, 'settings': {'FLAVOR': 'updates-server'}, 'state': 'done'},

+                         {'id': 2, 'settings': {'FLAVOR': 'updates-server'}, 'state': 'done'},

+                         {'id': 3, 'settings': {'FLAVOR': 'updates-workstation'}, 'state': 'done'},

+                         {'id': 4, 'settings': {'FLAVOR': 'updates-workstation'}, 'state': 'done'},

+                     ]

+                 }

+             else:

+                 fake_request.return_value = {'jobs': []}

+             consumer(message)

          if flavors is False:

              assert fake_schedule.call_count + fake_update.call_count == 0

          else:
@@ -302,6 +405,10 @@ 

              else:

                  assert fake_update.call_args[1]['openqa_hostname'] == oqah

                  assert fake_update.call_args[1]['flavors'] == flavors

+                 if advisory:

+                     assert fake_update.call_args[0][0] == advisory

+                 if version:

+                     assert fake_update.call_args[0][1] == version

          #fake_schedule.reset_mock()

  

  

Bodhi has a button that lets an update owner (or proven packager)
request a re-run of automated tests on the update. When the
button is clicked, Bodhi publishes a message with the topic
"bodhi.update.status.testing.koji-build-group.build.complete",
which is also what it does when an update is pushed to testing
after being created or edited. We did not previously handle these
messages because there was no way to distinguish between "new/
edited update" messages and "re-trigger" messages due to a couple
of Bodhi bugs, but those are now fixed, and the messages can be
identified by the "re-trigger" key (it's True for re-trigger
requests, False for new/edited update messages). So, this handles
them.

The messages don't have all the info about the update that other
Bodhi messages do, in particular they don't have the best
'version' field (so we have to parse something that's close
enough), the advisory ID has an appendix (so we have to strip
that), and there's no indication whether the update is critpath
(so instead of 'deciding' whether to test it as we do with
new/edited updates, we just see if we have already tested the
same update, and if so, we test it again, on the same flavors -
which is the spirit of the request anyway).

Signed-off-by: Adam Williamson awilliam@redhat.com

@zlopez @asaleh @mattia for reference, let me know if you see any potential problems here. Note I still don't want to use these messages for scheduling tests when updates are created or edited, because it's unnecessarily slow for openQA, which doesn't need them to be pushed to testing in order to test them, only created.

Build succeeded.

rebased onto 00a6a54

2 years ago

Build succeeded.

rebased onto 1d6335f

2 years ago

Build succeeded.

rebased onto 76df1f5

2 years ago

Build succeeded.

Pull-Request has been merged by adamwill

2 years ago