#97 Switch FCOS scheduling to use build metadata, add consumer (#85)
Merged 3 years ago by adamwill. Opened 3 years ago by adamwill.

@@ -45,6 +45,7 @@ 

  exchange = "amq.topic"

  # we subscribe to prod not staging keys here as per above.

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

+                 "org.fedoraproject.prod.coreos.build.state.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

                  "org.fedoraproject.prod.bodhi.update.edit",

                  "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]
@@ -54,6 +55,7 @@ 

  exchange = "zmq.topic"

  # we subscribe to prod not staging keys here as per above.

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

+                 "org.fedoraproject.prod.coreos.build.state.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

                  "org.fedoraproject.prod.bodhi.update.edit",

                  "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

@@ -40,6 +40,7 @@ 

  queue = "00000000-0000-0000-0000-000000000000"

  exchange = "amq.topic"

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

+                 "org.fedoraproject.prod.coreos.build.state.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

                  "org.fedoraproject.prod.bodhi.update.edit",

                  "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]
@@ -48,6 +49,7 @@ 

  queue = "00000000-0000-0000-0000-000000000000"

  exchange = "zmq.topic"

  routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",

+                 "org.fedoraproject.prod.coreos.build.state.change",

                  "org.fedoraproject.prod.bodhi.update.request.testing",

                  "org.fedoraproject.prod.bodhi.update.edit",

                  "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete"]

file modified
+6 -7
@@ -76,16 +76,16 @@ 

      sys.exit()

  

  def command_fcosbuild(args):

-     """Schedule openQA jobs for a Fedora CoreOS build (current build

-     from specified stream). Currently assumes x86_64 as that's all

-     Fedora CoreOS builds for.

+     """

+     Schedule openQA jobs for a specific Fedora CoreOS build. Takes

+     the build directory URL.

      """

      flavors = None

      if args.flavors:

          flavors = args.flavors.split(',')

  

-     jobs = schedule.jobs_from_fcosbuild(stream=args.stream, flavors=flavors, force=args.force,

-                                             openqa_hostname=args.openqa_hostname)

+     jobs = schedule.jobs_from_fcosbuild(args.buildurl, flavors=flavors, force=args.force,

+                                         openqa_hostname=args.openqa_hostname)

      print("Scheduled jobs: {0}".format(', '.join((str(job) for job in jobs))))

      sys.exit()

  
@@ -223,8 +223,7 @@ 

      parser_fcosbuild = subparsers.add_parser(

          "fcosbuild", description="Schedule jobs for a Fedora CoreOS build stream."

      )

-     parser_fcosbuild.add_argument("--stream", "-s", default="next", help="The stream to test the current build "

-                                   "from", metavar="STREAM", choices=("rawhide", "next", "testing", "stable"))

+     parser_fcosbuild.add_argument("buildurl", help="The URL to the build directory", metavar="BUILDURL")

      parser_fcosbuild.add_argument("--flavors", help="Comma-separated list of flavors to schedule jobs for "

                                    "(if not specified, all flavors will be scheduled)", metavar="FLAVORS")

      parser_fcosbuild.add_argument("--openqa-hostname", help="openQA host to schedule jobs on (default: "

@@ -64,6 +64,8 @@ 

          """Consume incoming message."""

          if 'pungi' in message.topic:

              return self._consume_compose(message)

+         elif 'coreos' in message.topic:

+             return self._consume_fcosbuild(message)

          elif 'bodhi.update.status.testing' in message.topic:

              return self._consume_retrigger(message)

          elif 'bodhi' in message.topic:
@@ -102,6 +104,26 @@ 

  

          return

  

+     def _consume_fcosbuild(self, message):

+         """Consume an FCOS build state change message."""

+         body = _find_true_body(message)

+         # this is intentionally written to blow up if required info is

+         # missing from the message, as that would likely indicate a

+         # message format change and we'd need to handle that

+         if body["state"] != "FINISHED" or body["result"] != "SUCCESS":

+             self.logger.debug("Not a 'finished success' message, ignoring")

+             return

+         builddir = body["build_dir"]

+         self.logger.info("Scheduling openQA jobs for FCOS build %s", builddir)

+         jobs = schedule.jobs_from_fcosbuild(builddir, openqa_hostname=self.openqa_hostname)

+         if jobs:

+             self.logger.info("openQA jobs run: %s", ' '.join(str(job) for job in jobs))

+         else:

+             self.logger.info("No openQA jobs run!")

+             return

+         self.logger.debug("Finished")

+         return

+ 

      def _update_schedule(self, advisory, version, flavors):

          """

          Shared schedule, log, return code for _consume_retrigger and

file modified
+19 -16
@@ -390,23 +390,24 @@ 

      return (rel.cid, jobs)

  

  

- def jobs_from_fcosbuild(stream="next", flavors=None, force=False, extraparams=None, openqa_hostname=None):

-     """Schedule jobs for the current artifacts from the given Fedora

-     CoreOS stream (valid streams are "rawhide", "next", "testing",

-     "stable").

+ def jobs_from_fcosbuild(buildurl, flavors=None, force=False, extraparams=None, openqa_hostname=None):

+     """Schedule jobs for the Fedora CoreOS build at the given URL

+     (should be the top-level URL with meta.json in it).

      flavors can be an iterable of flavors to schedule, otherwise all

-     known CoreOS flavors will be scheduled. If force is False, we

-     will not create jobs if some already exist for the same version

-     and flavor; if it's True, we will always create jobs.

+     known CoreOS flavors that are present in the build will be

+     scheduled.

+     If force is False, we will not create jobs if some already exist

+     for the same version and flavor; if it's True, we will always

+     create jobs.

      """

      flavdict = {

-         "CoreOS-colive-iso": ("iso", "colive", "ISO_URL"),

+         "CoreOS-colive-iso": ("live-iso", "colive", "ISO_URL"),

      }

-     url = f"https://builds.coreos.fedoraproject.org/streams/{stream}.json"

-     # assumes x86_64 for now, if CoreOS starts building for other

-     # arches will have to adjust this

-     artifacts = fedfind.helpers.download_json(url)["architectures"]["x86_64"]["artifacts"]["metal"]

-     version = artifacts["release"]

+     url = f"{buildurl}/meta.json"

+     metadata = fedfind.helpers.download_json(url)

+     arch = metadata["coreos-assembler.basearch"]

+     images = metadata["images"]

+     version = metadata["buildid"]

      relnum = version.split(".")[0]

      build = f"Fedora-CoreOS-{version}"

      logger.info("Scheduling jobs for CoreOS release %s", version)
@@ -415,19 +416,21 @@ 

          if flavors and flavor not in flavors:

              # filtered out!

              continue

-         location = artifacts["formats"].get(form, {}).get("disk", {}).get("location")

+         path = images.get(form, {}).get("path")

+         location = f"{buildurl}/{path}"

          if location:

              param_urls = {

                  param: location,

              }

          else:

-             # no artifact found, onto the next

+             # no image found, onto the next

              continue

+         logger.debug("Arch: %s", arch)

          logger.debug("Flavor: %s", flavor)

          logger.debug("Format: %s", form)

          logger.debug("Image type: %s", imagetype)

          logger.debug("Location: %s", location)

-         jobs.extend(run_openqa_jobs(param_urls, flavor, "x86_64", "CoreOS", imagetype, build,

+         jobs.extend(run_openqa_jobs(param_urls, flavor, arch, "CoreOS", imagetype, build,

                                      relnum, "", force=force, extraparams=extraparams,

                                      openqa_hostname=openqa_hostname))

      return jobs

file modified
+6 -13
@@ -215,12 +215,13 @@ 

      @mock.patch('fedora_openqa.schedule.jobs_from_fcosbuild', return_value=[1, 2], autospec=True)

      def test_fcosbuild(self, fakejff, capsys):

          """General tests for command_fcosbuild."""

-         args = cli.parse_args(["fcosbuild"])

+         buildurl = "https://builds.coreos.fedoraproject.org/prod/streams/rawhide/builds/36.20211123.91.0/x86_64"

+         args = cli.parse_args(["fcosbuild", buildurl])

          with pytest.raises(SystemExit) as excinfo:

              cli.command_fcosbuild(args)

          (out, _) = capsys.readouterr()

-         # stream kwarg should be default stream ("next")

-         assert fakejff.call_args[1]["stream"] == "next"

+         # first arg should be buildurl

+         assert fakejff.call_args[0][0] == buildurl

          # flavors kwarg should be false-y (not, e.g., [None])

          assert not fakejff.call_args[1]["flavors"]

          # shouldn't force
@@ -230,16 +231,8 @@ 

          # should exit 0

          assert not excinfo.value.code

  

-         # check explicit stream selection

-         args = cli.parse_args(["fcosbuild", "--stream", "testing"])

-         with pytest.raises(SystemExit) as excinfo:

-             cli.command_fcosbuild(args)

-         # should exit 0

-         assert not excinfo.value.code

-         assert fakejff.call_args[1]["stream"] == "testing"

- 

          # check 'flavors'

-         args = cli.parse_args(["fcosbuild", "--flavors", "foo,bar"])

+         args = cli.parse_args(["fcosbuild", buildurl, "--flavors", "foo,bar"])

          with pytest.raises(SystemExit) as excinfo:

              cli.command_fcosbuild(args)

          # should exit 0
@@ -247,7 +240,7 @@ 

          assert fakejff.call_args[1]["flavors"] == ["foo", "bar"]

  

          # check 'force'

-         args = cli.parse_args(["fcosbuild", "--force"])

+         args = cli.parse_args(["fcosbuild", buildurl, "--force"])

          with pytest.raises(SystemExit) as excinfo:

              cli.command_fcosbuild(args)

          # should exit 0

file modified
+33 -6
@@ -259,6 +259,25 @@ 

      }

  )

  

+ # Successful FCOS build message

+ FCOSBUILD = Message(

+     topic="org.fedoraproject.prod.coreos.build.state.change",

+     body={

+         "build_id": "36.20211123.91.0",

+         "stream": "rawhide",

+         "basearch": "x86_64",

+         "build_dir": "https://builds.coreos.fedoraproject.org/prod/streams/rawhide/builds/36.20211123.91.0/x86_64",

+         "state": "FINISHED",

+         "result": "SUCCESS"

+     }

+ )

+ # Not "finished" FCOS build message

+ FCOSBUILDNOTF = copy.deepcopy(FCOSBUILD)

+ FCOSBUILDNOTF.body["state"] = "STARTED"

+ # Not "successful" FCOS build message

+ FCOSBUILDNOTS = copy.deepcopy(FCOSBUILD)

+ FCOSBUILDNOTS.body["result"] = "FAILURE"

+ 

  # initialize a few test consumers with different configs

  PRODCONF = {

      'consumer_config': {
@@ -316,6 +335,7 @@ 

  

      @mock.patch('fedora_openqa.schedule.jobs_from_compose', return_value=('somecompose', [1]), autospec=True)

      @mock.patch('fedora_openqa.schedule.jobs_from_update', return_value=[1], autospec=True)

+     @mock.patch('fedora_openqa.schedule.jobs_from_fcosbuild', return_value=[1], autospec=True)

      @pytest.mark.parametrize(

          "consumer,oqah",

          [
@@ -364,9 +384,13 @@ 

              (RETRIGGER, False, False, None, None),

              (NONRETRIGGER, True, False, None, None),

              (NONFRETRIGGER, True, False, None, None),

+             (FCOSBUILD, False, None, None, None),

+             (FCOSBUILDNOTF, False, False, None, None),

+             (FCOSBUILDNOTS, False, False, None, None)

          ]

      )

-     def test_scheduler(self, fake_update, fake_schedule, consumer, oqah, message, gotjobs, flavors, advisory, version):

+     def test_scheduler(self, fake_fcosbuild, fake_update, fake_schedule, consumer, oqah,

+                        message, gotjobs, flavors, advisory, version):

          """Test the job scheduling consumers do their thing. The

          parametrization pairs are:

          1. (consumer, expected openQA hostname)
@@ -391,17 +415,20 @@ 

                  fake_request.return_value = {'jobs': []}

              consumer(message)

          if flavors is False:

-             assert fake_schedule.call_count + fake_update.call_count == 0

+             assert fake_schedule.call_count + fake_update.call_count + fake_fcosbuild.call_count == 0

          else:

              archcount = len(consumer.update_arches)

-             compcalls = fake_schedule.call_count

+             compcalls = fake_schedule.call_count + fake_fcosbuild.call_count

              updcalls = fake_update.call_count

-             # for a compose test on any consumer, the method should be

-             # hit once. for an update test, the method should be hit

-             # as many times as the consumer has arches configured

+             # for a compose/fcosbuild test on any consumer, the method

+             # should be hit once. for an update test, the method should

+             # be hit as many times as the consumer has arches

+             # configured

              assert (compcalls == 1 and updcalls == 0) or (compcalls == 0 and updcalls == archcount)

              if fake_schedule.call_count == 1:

                  assert fake_schedule.call_args[1]['openqa_hostname'] == oqah

+             elif fake_fcosbuild.call_count == 1:

+                 assert fake_fcosbuild.call_args[1]['openqa_hostname'] == oqah

              else:

                  assert fake_update.call_args[1]['openqa_hostname'] == oqah

                  assert fake_update.call_args[1]['flavors'] == flavors

file modified
+22 -66
@@ -63,59 +63,14 @@ 

  # trimmed CoreOS build metadata JSON, used for scheduling CoreOS jobs

  COREOSJSON = {

      # pylint: disable=line-too-long

-     "stream": "next",

-     "metadata": {

-         "last-modified": "2020-08-25T19:20:43Z"

-     },

-     "architectures": {

-         "x86_64": {

-             "artifacts": {

-                 "metal": {

-                     "release": "32.20200824.1.0",

-                     "formats": {

-                         "4k.raw.xz": {

-                             "disk": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-metal4k.x86_64.raw.xz",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-metal4k.x86_64.raw.xz.sig",

-                                 "sha256": "124b9ef7aa58d4c85c59259a6eae6e55175f1f8aa00f9df4e28f6e1fe77170df"

-                             }

-                         },

-                         "iso": {

-                             "disk": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live.x86_64.iso",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live.x86_64.iso.sig",

-                                 "sha256": "23b979b1675fcd1cc1972f77cce802e585eb123f4ef1770448a6085b574527fb"

-                             }

-                         },

-                         "pxe": {

-                             "kernel": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-kernel-x86_64",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-kernel-x86_64.sig",

-                                 "sha256": "249d9f9b59bc96904de096a5b6a15a7892a596bdc109e6b7b123fe69f5969b9c"

-                             },

-                             "initramfs": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-initramfs.x86_64.img",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-initramfs.x86_64.img.sig",

-                                 "sha256": "92ef4de1daa0d65c8981a1f5f6e9b80cb3f95ba5a08f25f1621ac0ecb934f9b5"

-                             },

-                             "rootfs": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-rootfs.x86_64.img",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live-rootfs.x86_64.img.sig",

-                                 "sha256": "8581ac303b9f0465f910229863505a0ca8d937924bfa5ee2f4db61daa29d94da"

-                             }

-                         },

-                         "raw.xz": {

-                             "disk": {

-                                 "location": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-metal.x86_64.raw.xz",

-                                 "signature": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-metal.x86_64.raw.xz.sig",

-                                 "sha256": "8fc5dd1ab58acebd69b90b4d8dc46bc8e7166ace101fdd092e6526941632ece2"

-                             }

-                         }

-                     }

-                 }

-             }

+     "buildid": "36.20211123.91.0",

+     "images": {

+         "live-iso": {

+             "path": "fedora-coreos-36.20211123.91.0-live.x86_64.iso",

+             "sha256": "ad0d16c772c1ba6c3f1189d6de37e25e251b3d11ce96b9fe2c44fb4178a320b9"

          }

-     }

+     },

+     "coreos-assembler.basearch": "x86_64"

  }

  

  @pytest.mark.usefixtures("ffmock02")
@@ -979,8 +934,8 @@ 

      }

  

  @mock.patch("fedfind.helpers.download_json", return_value=COREOSJSON)

- @mock.patch("fedfind.helpers.get_current_stables", return_value=[31, 32])

- @mock.patch("fedfind.helpers.get_current_release", return_value=32)

+ @mock.patch("fedfind.helpers.get_current_stables", return_value=[33, 34, 35])

+ @mock.patch("fedfind.helpers.get_current_release", return_value=35)

  @mock.patch("fedora_openqa.schedule.OpenQA_Client", autospec=True)

  def test_jobs_from_fcosbuild(fakeclient, fakecurrr, fakecurrs, fakejson):

      """Test scheduling jobs from a Fedora CoreOS build."""
@@ -989,9 +944,10 @@ 

      # for now, return no 'jobs' (for the dupe query), one 'id' (for

      # the post request)

      fakeinst.openqa_request.return_value = {"jobs": [], "ids": [1]}

-     # simple case, default stream is next

-     ret = schedule.jobs_from_fcosbuild()

-     # should get one job and the build back

+     # simple case, COREOSJSON is trimmed from the real meta.json here

+     buildurl = "https://builds.coreos.fedoraproject.org/prod/streams/rawhide/builds/36.20211123.91.0/x86_64"

+     ret = schedule.jobs_from_fcosbuild(buildurl)

+     # should get one job back

      assert ret == [1]

      # find the POST calls

      posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == "POST"]
@@ -1000,28 +956,28 @@ 

      parmdict = posts[0][0][2]

      assert parmdict == {

          "DISTRI": "fedora",

-         "VERSION": "32",

+         "VERSION": "36",

          "ARCH": "x86_64",

-         "BUILD": "Fedora-CoreOS-32.20200824.1.0",

+         "BUILD": "Fedora-CoreOS-36.20211123.91.0",

          "_OBSOLETE": "1",

          "_ONLY_OBSOLETE_SAME_BUILD": "1",

          "QEMU_HOST_IP": "172.16.2.2",

          "NICTYPE_USER_OPTIONS": "net=172.16.2.0/24",

          "FLAVOR": "CoreOS-colive-iso",

-         "CURRREL": "32",

-         "PREVREL": "31",

-         "RAWREL": "33",

-         "UP1REL": "31",

-         "UP2REL": "30",

+         "CURRREL": "35",

+         "PREVREL": "34",

+         "RAWREL": "36",

+         "UP1REL": "35",

+         "UP2REL": "34",

          "IMAGETYPE": "colive",

          # pylint: disable=line-too-long

-         "ISO_URL": "https://builds.coreos.fedoraproject.org/prod/streams/next/builds/32.20200824.1.0/x86_64/fedora-coreos-32.20200824.1.0-live.x86_64.iso",

+         "ISO_URL": f"{buildurl}/fedora-coreos-36.20211123.91.0-live.x86_64.iso",

          "LOCATION": "",

          "SUBVARIANT": "CoreOS",

      }

  

      # test we get no jobs if we specify a non-found flavor

-     ret = schedule.jobs_from_fcosbuild(flavors=["nonexistent"])

+     ret = schedule.jobs_from_fcosbuild(buildurl, flavors=["nonexistent"])

      assert ret == []

      posts = [call for call in fakeinst.openqa_request.call_args_list if call[0][0] == "POST"]

      # should still only have one call

There are now messages (coreos.build.state.change) we can use
to trigger tests of FCOS builds. This includes builds that aren't
published to the public streams, so in order to test those, we
change the triggering code to run from the 'build URL' (which
is provided in these messages), using a per-build metadata file
(meta.json) which should always be present. To manually trigger
tests you need to know this build URL. This change also allows
for scheduling for aarch64, though the templates aren't yet set
up for this.

Signed-off-by: Adam Williamson awilliam@redhat.com

Build succeeded.

Pull-Request has been merged by adamwill

3 years ago