#35 Add support for Google Cloud Platform
Merged 4 months ago by jcline. Opened 5 months ago by jcline.
gcp  into  main

@@ -1,7 +1,8 @@ 

- __version__ = "1.1.1"

+ __version__ = "1.2.0"

  

  from .publish import (  # noqa: F401

      AwsPublishedV1,

      AzurePublishedV1,

      ContainerPublishedV1,

+     GcpPublishedV1,

  )

@@ -250,3 +250,79 @@ 

              f"\tTags: {', '.join(self.body['tags'])}\n"

              f"\tArchitectures: {', '.join(self.body['architectures'])}\n"

          )

+ 

+ 

+ class GcpPublishedV1(_PublishedV1):

+     topic = ".".join([_PublishedV1.topic, "gcp"])

+     body_schema = {

+         "id": f"{SCHEMA_URL}/v1/{'.'.join([_PublishedV1.topic, 'gcp'])}",

+         "$schema": "https://json-schema.org/draft/2019-09/schema",

+         "description": (

+             "Schema for messages sent by fedora-image-uploader when a "

+             "new Google Cloud Platform image is published."

+         ),

+         "type": "object",

+         "properties": {

+             "architecture": {

+                 "type": "string",

+                 "description": "The machine architecture of the image (x86_64, aarch64, etc).",

+             },

+             "compose_id": {

+                 "type": "string",

+                 "description": "The compose ID this image was created from.",

+             },

+             "release": {

+                 "type": "integer",

+                 "description": "The release number associated with the image.",

+             },

+             "subvariant": {

+                 "type": "string",

+                 "description": "The subvariant of the image (e.g. Cloud_Base).",

+             },

+             "family": {

+                 "type": "string",

+                 "description": "The Google Compute Engine OS family for the image.",

+             },

+             "image_name": {

+                 "type": "string",

+                 "description": "The name of the image.",

+             },

+             "image_url": {

+                 "type": "string",

+                 "description": "The URL of the image in Google Cloud Engine.",

+             },

+             "storage_locations": {

+                 "type": "array",

+                 "items": {"type": "string"},

+                 "description": "The geographic location codes where the image is available.",

+             },

+         },

+         "required": [

+             "architecture",

+             "compose_id",

+             "release",

+             "subvariant",

+             "family",

+             "image_name",

+             "image_url",

+             "storage_locations",

+         ],

+     }

+ 

+     @property

+     def summary(self):

+         return (

+             f"{self.app_name} published the {self.body['architecture']} image from compose"

+             f" {self.body['compose_id']} to the {self.body['family']} family in Google"

+             " Cloud Platform"

+         )

+ 

+     def __str__(self):

+         return (

+             "A new image has been published to Google Cloud Platform:\n\n"

+             f"\tArchitecture: {self.body['architecture']}\n"

+             f"\tCompose ID: {self.body['compose_id']}\n"

+             f"\tFamily: {self.body['family']}\n"

+             f"\tImage Name: {self.body['image_name']}\n"

+             f"\tImage URL: {self.body['image_url']}\n"

+         )

@@ -45,6 +45,7 @@ 

  "fedora_image_uploader.published.v1.aws" = "fedora_image_uploader_messages.publish:AwsPublishedV1"

  "fedora_image_uploader.published.v1.azure" = "fedora_image_uploader_messages.publish:AzurePublishedV1"

  "fedora_image_uploader.published.v1.container" = "fedora_image_uploader_messages.publish:ContainerPublishedV1"

+ "fedora_image_uploader.published.v1.gcp" = "fedora_image_uploader_messages.publish:GcpPublishedV1"

  

  [tool.hatch.version]

  path = "fedora_image_uploader_messages/__init__.py"

@@ -11,6 +11,7 @@ 

      AwsPublishedV1,

      AzurePublishedV1,

      ContainerPublishedV1,

+     GcpPublishedV1,

  )

  

  
@@ -192,3 +193,34 @@ 

      message = ContainerPublishedV1(body=body)

      assert expected_summary == message.summary

      assert expected_str == str(message)

+ 

+ 

+ def test_gcp_str():

+     message = GcpPublishedV1(

+         topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.aarch64",

+         body={

+             "architecture": "x86_64",

+             "compose_id": "Fedora-40-20240414.0",

+             "release": 40,

+             "subvariant": "Cloud_Base",

+             "family": "fedora-cloud-40",

+             "image_name": "fedora-cloud-40-1-14-x86-64",

+             "image_url": "https://example.com/link",

+             "storage_locations": ["us"],

+         },

+     )

+     expected_summary = (

+         "fedora-image-uploader published the x86_64 image from compose Fedora-40-20240414.0 to "

+         "the fedora-cloud-40 family in Google Cloud Platform"

+     )

+     expected_str = (

+         "A new image has been published to Google Cloud Platform:\n\n"

+         "\tArchitecture: x86_64\n"

+         "\tCompose ID: Fedora-40-20240414.0\n"

+         "\tFamily: fedora-cloud-40\n"

+         "\tImage Name: fedora-cloud-40-1-14-x86-64\n"

+         "\tImage URL: https://example.com/link\n"

+     )

+ 

+     assert message.summary == expected_summary

+     assert str(message) == expected_str

@@ -0,0 +1,435 @@ 

+ import datetime

+ import hashlib

+ import logging

+ import tempfile

+ import uuid

+ 

+ from fedfind import release as ff_release

+ from fedora_image_uploader_messages import GcpPublishedV1

+ from fedora_messaging import config

+ from fedora_messaging import exceptions as fm_exceptions

+ from google.api_core import exceptions as google_exceptions

+ from google.cloud import compute_v1, storage

+ 

+ from .utils import (

+     download_image,

+     fallible_publish,

+     get_eol,

+     get_milestone,

+     parse_release,

+ )

+ 

+ _log = logging.getLogger(__name__)

+ 

+ 

+ class Gcp:

+     """

+     Uploader for Google Cloud Platform.

+ 

+     This handler uses the following configuration keys under the "gcp" section of "consumer_config":

+         - project: The GCP project to upload images to.

+         - bucket_name: The name of the GCP storage bucket to upload images to.

+         - storage_locations: A list of GCP storage locations to store the image in. Currently this

+                              list should contain only one location since GCP rejects it otherwise.

+         - publish_amqp_messages: Whether to publish messages to the AMQP broker.

+     """

+ 

+     SUPPORTED_ARCHES = {

+         "x86_64": compute_v1.Image.Architecture.X86_64,

+         "aarch64": compute_v1.Image.Architecture.ARM64,

+     }

+ 

+     def __init__(self):

+         self.conf = config.conf["consumer_config"]["gcp"]

+         self.storage_client = storage.Client(project=self.conf["project"])

+         self.images_client = compute_v1.ImagesClient()

+ 

+     def __call__(self, image: dict, ffrel: ff_release.Release):

+         if image.get("arch") not in self.SUPPORTED_ARCHES.keys():

+             _log.debug(f"Skipping unsupported arch '{image['arch']}' for GCP")

+             return

+         if image.get("format") != "tar.gz":

+             _log.debug(f"Skipping unsupported format '{image['format']}' (need tar.gz) for GCP")

+             return

+         if image.get("subvariant") not in ("Cloud_Base", "BaseOS"):

+             _log.debug(

+                 "Skipping %s for GCP: subvariant is %s", image.get("path"), image.get("arch")

+             )

+             return

+         if ffrel.relnum < 40 and ffrel.release.lower() != "eln":

+             # The format changed with F40 and I can't be bothered to support the old format for

+             # a month

+             _log.debug("Skipping %s for GCP: F{ffrel.relnum} not supported", image.get("path"))

+             return

+ 

+         blob = self.upload_disk_image(image)

+         try:

+             uploaded_image = self.import_image(image, ffrel, blob)

+         except google_exceptions.Conflict as e:

+             # The name conflicts; requests are idempotent if and only if the image sha256 is the

+             # same. Something's gone quite wrong here so log at error level, but don't nack the

+             # message since there's not much we can programmatically do to fix this.

+             _log.error(

+                 "Unable to import image from %s as it conflicts with an existing image (%s)",

+                 blob.self_link,

+                 str(e),

+             )

+             return

+ 

+         message = GcpPublishedV1(

+             topic=".".join(

+                 [GcpPublishedV1.topic, get_milestone(ffrel), image["subvariant"], image["arch"]]

+             ),

+             body={

+                 "architecture": image["arch"],

+                 "compose_id": ffrel.cid,

+                 "release": ffrel.relnum,

+                 "subvariant": image["subvariant"],

+                 "family": uploaded_image.family,

+                 "image_name": uploaded_image.name,

+                 "image_url": uploaded_image.self_link,

+                 "storage_locations": uploaded_image.storage_locations,

+             },

+         )

+ 

+         if self.conf.get("publish_amqp_messages", False):

+             fallible_publish(message)

+ 

+         try:

+             self.promote_image(uploaded_image)

+         except google_exceptions.GoogleAPICallError as e:

+             # We'll try again next time

+             _log.error("Failed to promote images: %s", str(e))

+ 

+         try:

+             self.cleanup_old_images()

+         except google_exceptions.GoogleAPICallError as e:

+             # We'll try again next time

+             _log.error("Failed to clean up images: %s", str(e))

+ 

+     def import_image(

+         self, image: dict, ffrel: ff_release.Release, blob: storage.Blob

+     ) -> compute_v1.Image:

+         image_suffix, y_release, z_release = parse_release(ffrel)

+         image_version = f"{ffrel.relnum}.{y_release}.{z_release}"

+         image_name = (

+             f"fedora-cloud-{ffrel.relnum}-{y_release}-{z_release}-{image['arch']}".lower().replace(

+                 "_", "-"

+             )

+         )

+         family = f"fedora-cloud-{image_suffix.lower()}"

+         architecture = self.SUPPORTED_ARCHES[image["arch"]]

+         eol = get_eol(ffrel)

+         if eol:

+             eol = eol.strftime("%Y-%m-%d")

+         else:

+             eol = "none"

+ 

+         # This list may not be complete...

+         features = [

+             compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name),

+             compute_v1.GuestOsFeature(

+                 type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name

+             ),

+             compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),

+             compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),

+         ]

+         # Once aarch64 is signed for Secure Boot, we can include this unconditionally

+         if architecture == compute_v1.Image.Architecture.X86_64:

+             features.append(

+                 compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.SECURE_BOOT.name),

+             )

+ 

+         # With the exception of Rawhide and ELN, we initially upload the image as deprecated so

+         # thatit doesn't became the default for the image family. Images can then be selectively

+         # promoted to be "ACTIVE".

+         #

+         # https://cloud.google.com/compute/docs/images/deprecate-custom#deprecation-states

+         deprecated = compute_v1.DeprecationStatus.State.DEPRECATED.name

+         if ffrel.release.lower() in ("rawhide", "eln"):

+             deprecated = compute_v1.DeprecationStatus.State.ACTIVE.name

+         # Also, if this is the first image of a family, make it active

+         request = compute_v1.ListImagesRequest(

+             project=self.conf["project"],

+             filter=f"(family = {family}) AND (architecture = {architecture.name})",

+             max_results=1,

+         )

+         if len(list(self.images_client.list(request=request, timeout=60))) == 0:

+             deprecated = compute_v1.DeprecationStatus.State.ACTIVE.name

+ 

+         disk = compute_v1.RawDisk(source=blob.self_link)

+         image_resource = compute_v1.Image(

+             architecture=architecture.name,

+             family=family,

+             description=f"Fedora Cloud base image version {image_version}",

+             # Despite being able to set this, it seems to be ignored when adding the image,

+             # so there's a call below to set it again.

+             deprecated=compute_v1.DeprecationStatus(state=deprecated),

+             guest_os_features=features,

+             labels={

+                 # Sadly only ASCII lowercase, numeric, underscore and dash are allowed

+                 "fedora-compose-id": ffrel.cid.lower().replace(".", "-"),

+                 "fedora-subvariant": image["subvariant"].lower(),

+                 "fedora-release": ffrel.release.lower(),

+                 "fedora-version": str(ffrel.relnum),

+                 "end-of-life": eol,

+                 "fedora-image-uploader-managed": "true",

+             },

+             # The API rejects this as an "invalid URL".

+             # licenses=["https://docs.fedoraproject.org/en-US/legal/fedora-linux-license/"],

+             name=image_name,

+             raw_disk=disk,

+             # Despite the name, it seems to only accept a list of length 1.

+             storage_locations=self.conf["storage_locations"],

+         )

+         image_request = compute_v1.InsertImageRequest(

+             project=self.conf["project"],

+             image_resource=image_resource,

+             request_id=str(uuid.UUID(hex=image["checksums"]["sha256"][:32])),

+         )

+         _log.info(

+             "Requesting import of image %s to the %s family in GCP as %s",

+             blob.self_link,

+             family,

+             image_name,

+         )

+         try:

+             response = self.images_client.insert(image_request, timeout=60)

+             response.result(timeout=60 * 30)

+             response.exception(timeout=5)

+             if response.warnings:

+                 for warning in response.warnings:

+                     _log.warning("Warning during image import: %s", warning.message)

+         except google_exceptions.Conflict:

+             # This specific error needs to be handled by this function's caller since retrying is

+             # not going to fix it.

+             raise

+         except google_exceptions.GoogleAPICallError as e:

+             _log.error("Failed to import image %s: %s", image_name, str(e))

+             raise fm_exceptions.Nack()

+ 

+         # GCE doesn't actually mark the image deprecated despite our request (maybe they'll

+         # fix that some day?), so do it a second time. This obviously introduces a race where

+         # booting from the family during this time gets the new image, but there's not much we

+         # can do about that.

+         if deprecated == compute_v1.DeprecationStatus.State.DEPRECATED.name:

+             try:

+                 _log.info(

+                     "Requesting new image %s in the %s family be marked DEPRECATED",

+                     image_name,

+                     family,

+                 )

+                 request = compute_v1.DeprecateImageRequest(

+                     project=self.conf["project"],

+                     image=image_name,

+                     deprecation_status_resource=compute_v1.DeprecationStatus(

+                         state=compute_v1.DeprecationStatus.State.DEPRECATED.name,

+                     ),

+                     request_id=str(uuid.UUID(hex=image["checksums"]["sha256"][:32])),

+                 )

+                 response = self.images_client.deprecate(request=request, timeout=60)

+                 response.result(timeout=60 * 10)

+                 response.exception(timeout=5)

+             except google_exceptions.GoogleAPICallError as e:

+                 _log.error("Failed to deprecate image %s: %s", image_name, str(e))

+                 raise fm_exceptions.Nack()

+ 

+         try:

+             image_request = compute_v1.GetImageRequest(

+                 project=self.conf["project"], image=image_name

+             )

+             image_resource = self.images_client.get(image_request, timeout=30)

+         except google_exceptions.GoogleAPICallError as e:

+             _log.error("Failed to read image details for %s: %s", image_name, str(e))

+             raise fm_exceptions.Nack()

+ 

+         _log.info(

+             "Successfully uploaded image %s to GCP under the %s family",

+             image_resource.name,

+             image_resource.family,

+         )

+         return image_resource

+ 

+     def upload_disk_image(self, image: dict) -> storage.Blob:

+         """

+         Upload the disk image to a GCP storage bucket.

+ 

+         This is a pre-requisite for importing the image as a machine image.

+         """

+         bucket = self.storage_client.bucket(self.conf["bucket_name"])

+         blob_name = f"{image['checksums']['sha256']}.tar.gz"

+         blob = bucket.blob(blob_name)

+ 

+         if blob.exists():

+             blob.reload()

+             _log.info("Skipping upload of GCP image to %s as it already exists", blob.self_link)

+         else:

+             with tempfile.TemporaryDirectory() as workdir:

+                 image_path = download_image(image, workdir, decompress=False)

+                 _log.info("Starting upload of GCP image to %s", bucket.name)

+                 try:

+                     blob.upload_from_filename(image_path, if_generation_match=0, timeout=60 * 30)

+                     _log.info("Finished upload of GCP image as %s", blob.self_link)

+                 except google_exceptions.GoogleAPICallError as e:

+                     _log.error("Failed to upload image to bucket: %s", str(e))

+                     raise fm_exceptions.Nack()

+ 

+         return blob

+ 

+     def promote_image(self, new_image: compute_v1.Image):

+         """

+         Promote an image from deprecated to active state if it's been 2 weeks since the last

+         promotion.

+         """

+         # These are always active

+         if new_image.labels["fedora-release"] in ("rawhide", "eln"):

+             return

+ 

+         # We can't filter on deprecated.state = ACTIVE since that's not set initially and no value

+         # is the same thing as ACTIVE, apparently.

+         request = compute_v1.ListImagesRequest(

+             project=self.conf["project"],

+             filter=(

+                 f"(family = {new_image.family}) AND " f"(architecture = {new_image.architecture})"

+             ),

+         )

+         response = self.images_client.list(request=request, timeout=60)

+ 

+         today = datetime.datetime.now(tz=datetime.UTC)

+         two_weeks_ago = today - datetime.timedelta(days=14)

+         active_images = [

+             image

+             for image in response

+             if image.deprecated != compute_v1.DeprecationStatus.State.DEPRECATED.name

+         ]

+         active_images.sort(

+             key=lambda image: datetime.datetime.fromisoformat(image.creation_timestamp),

+             reverse=True,

+         )

+ 

+         try:

+             latest_active = active_images[0]

+             if two_weeks_ago > datetime.datetime.fromisoformat(latest_active.creation_timestamp):

+                 _log.info(

+                     "The latest active image in the %s family, %s, is more than two weeks old",

+                     latest_active.family,

+                     latest_active.name,

+                 )

+             else:

+                 _log.info(

+                     "The latest active image in the %s family, %s, is less than two weeks old (%s)",

+                     latest_active.family,

+                     latest_active.name,

+                     str(latest_active.creation_timestamp),

+                 )

+                 return

+         except IndexError:

+             _log.error(

+                 "No active images found in the %s family, but the first image should have been"

+                 " active!",

+                 new_image.family,

+             )

+             return

+ 

+         request = compute_v1.DeprecateImageRequest(

+             project=self.conf["project"],

+             image=new_image.name,

+             deprecation_status_resource=compute_v1.DeprecationStatus(

+                 state=compute_v1.DeprecationStatus.State.ACTIVE.name,

+             ),

+         )

+         response = self.images_client.deprecate(request=request, timeout=60)

+         _log.info("Requested image %s in the %s be marked ACTIVE", new_image.name)

+         response.result(timeout=60 * 10)

+         response.exception(timeout=5)

+         if response.warnings:

+             for warning in response.warnings:

+                 _log.warning("Warning while marking image active: %s", warning.message)

+ 

+     def cleanup_old_images(self):

+         """

+         Remove old images from GCP.

+ 

+         The retention policy implemented here is as follows:

+           - Images for stable releases marked as active are retained until their EOL date.

+           - Images for stable releases marked as deprecated are deleted after 14 days.

+           - Rawhide and ELN images are retained for 14 days.

+         """

+         # Start by cleaning up deprecated and Rawhide/ELN images

+         request = compute_v1.ListImagesRequest(

+             project=self.conf["project"],

+             filter=(

+                 "(deprecated.state = DEPRECATED) OR "

+                 "(labels.fedora-release = rawhide) OR "

+                 "(labels.fedora-release = eln)"

+             ),

+         )

+         today = datetime.datetime.today().replace(tzinfo=datetime.UTC)

+         two_weeks_ago = today - datetime.timedelta(days=14)

+         delete_responses = []

+         response = self.images_client.list(request=request, timeout=60)

+         for image in response:

+             if image.labels.get("fedora-image-uploader-managed") != "true":

+                 # We didn't upload this image, so we shouldn't delete it

+                 continue

+             image_creation = datetime.datetime.fromisoformat(image.creation_timestamp)

+             if two_weeks_ago > image_creation:

+                 _log.info(

+                     "Requesting delete for image %s since it's older than %s",

+                     image.name,

+                     str(two_weeks_ago),

+                 )

+                 request = compute_v1.DeleteImageRequest(

+                     project=self.conf["project"],

+                     image=image.name,

+                     request_id=str(

+                         uuid.UUID(

+                             hex=hashlib.sha256(image.name.encode(errors="ignore")).hexdigest()[:32]

+                         )

+                     ),

+                 )

+                 delete_responses.append(self.images_client.delete(request, timeout=60 * 10))

+ 

+         # Next, remove any images with an EOL date that has passed. There's probably a fancy filter

+         # to do this, but I don't know the right syntax and we're going to be dealing with no more

+         # than a couple hundred images at a time.

+         request = compute_v1.ListImagesRequest(project=self.conf["project"])

+         response = self.images_client.list(request=request, timeout=60)

+         for image in response:

+             if image.labels.get("fedora-image-uploader-managed") != "true":

+                 # We didn't upload this image, so we shouldn't delete it

+                 continue

+ 

+             try:

+                 eol = datetime.datetime.fromisoformat(image.labels["end-of-life"]).replace(

+                     tzinfo=datetime.UTC

+                 )

+             except ValueError:

+                 # Pre-release, Rawhide, and ELN have an EOL of "none"

+                 _log.debug(

+                     "Skipping image %s with EOL of %s", image.name, image.labels["end-of-life"]

+                 )

+                 continue

+             except KeyError:

+                 _log.error("Image %s is missing an EOL label", image.name)

+                 continue

+ 

+             if today > eol:

+                 _log.info("Requesting delete for image %s since it's past EOL", image.name)

+                 request = compute_v1.DeleteImageRequest(

+                     project=self.conf["project"],

+                     image=image.name,

+                     request_id=str(

+                         uuid.UUID(

+                             hex=hashlib.sha256(image.name.encode(errors="ignore")).hexdigest()[:32]

+                         )

+                     ),

+                 )

+                 delete_responses.append(self.images_client.delete(request, timeout=60 * 10))

+ 

+         for response in delete_responses:

+             response.result(timeout=60 * 30)

+             response.exception(timeout=5)

+             if response.warnings:

+                 for warning in response.warnings:

+                     _log.warning("Warning during image import: %s", warning.message)

@@ -30,6 +30,11 @@ 

  except ImportError:

      _log.info("Install the 'azure' extra for Azure support")

  

+ try:

+     from . import gcp

+ except ImportError:

+     _log.info("Install the 'gcp' extra for Google Cloud Platform support")

+ 

  

  DOCKER_ARCHES = {"amd64": "x86_64", "arm64": "aarch64", "ppc64le": "ppc64le", "s390x": "s390x"}

  
@@ -76,6 +81,8 @@ 

              self.handlers["azure"] = azure.Azure()

          if "aws" in self.conf.keys():

              self.handlers["aws"] = aws.Aws()

+         if "gcp" in self.conf.keys():

+             self.handlers["gcp"] = gcp.Gcp()

  

          # tracks the container repos we got images for, for manifest

          # creation purposes

@@ -30,8 +30,12 @@ 

      "azure-mgmt-compute",

      "azure-storage-blob",

  ]

+ gcp = [

+     "google-cloud-compute",

+     "google-cloud-storage",

+ ]

  dev = [

-     "fedora-image-uploader[aws,azure]",

+     "fedora-image-uploader[aws,azure,gcp]",

      "black",

      "isort",

      "flake8",

@@ -0,0 +1,574 @@ 

+ import json

+ import os

+ from unittest import mock

+ 

+ import freezegun

+ import pytest

+ from fedora_image_uploader_messages import GcpPublishedV1

+ from fedora_messaging import config, message

+ from fedora_messaging import testing as fm_testing

+ from google.cloud import compute_v1, storage

+ 

+ from fedora_image_uploader import Uploader

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+                 "publish_amqp_messages": True,

+             }

+         },

+     },

+ )

+ def test_image_filter(fixtures_dir):

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.upload_disk_image = mock.Mock()

+     image = {

+         "arch": "aarch64",

+         "format": "tar.gz",

+         "subvariant": "Cloud_Base",

+     }

+     ffrel = mock.Mock(relnum=42, release="Rawhide")

+ 

+     # Arch isn't supported

+     image["arch"] = "ppc64le"

+     handler(image, ffrel)

+     handler.upload_disk_image.call_count == 0

+     image["arch"] = "aarch64"

+ 

+     # Format isn't tar.gz

+     image["format"] = "qcow2"

+     handler(image, ffrel)

+     handler.upload_disk_image.call_count == 0

+     image["format"] = "tar.gz"

+ 

+     # Subvariant isn't supported

+     image["subvariant"] = "CoreOS"

+     handler(image, ffrel)

+     handler.upload_disk_image.call_count == 0

+ 

+     # Don't bother with 39

+     handler(image, mock.Mock(relnum=39, release="39"))

+     handler.upload_disk_image.call_count == 0

+ 

+ 

+ @pytest.mark.vcr

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+                 "publish_amqp_messages": True,

+             }

+         },

+     },

+ )

+ def test_messages(fixtures_dir):

+     with open(os.path.join(fixtures_dir, "messages/rc_compose.json")) as fd:

+         msg = message.load_message(json.load(fd))

+     fake_images = [

+         mock.Mock(

+             family="fedora-cloud-40",

+             name="fedora-cloud-40-1-14-aarch64",

+             self_link="http://example.com/link",

+             storage_locations=["us"],

+         ),

+         mock.Mock(

+             family="fedora-cloud-40",

+             name="fedora-cloud-40-1-14-x86-64",

+             self_link="http://example.com/link",

+             storage_locations=["us"],

+         ),

+     ]

+     fake_images[0].name = "fedora-cloud-40-1-14-aarch64"

+     fake_images[1].name = "fedora-cloud-40-1-14-x86-64"

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.upload_disk_image = mock.Mock()

+     handler.import_image = mock.Mock(side_effect=fake_images)

+     handler.promote_image = mock.Mock()

+     handler.cleanup_old_images = mock.Mock()

+ 

+     expected_messages = [

+         GcpPublishedV1(

+             topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.aarch64",

+             body={

+                 "architecture": "aarch64",

+                 "compose_id": "Fedora-40-20240414.0",

+                 "release": 40,

+                 "subvariant": "Cloud_Base",

+                 "family": "fedora-cloud-40",

+                 "image_name": "fedora-cloud-40-1-14-aarch64",

+                 "image_url": "http://example.com/link",

+                 "storage_locations": ["us"],

+             },

+         ),

+         GcpPublishedV1(

+             topic="fedora_image_uploader.published.v1.gcp.rc.Cloud_Base.x86_64",

+             body={

+                 "architecture": "x86_64",

+                 "compose_id": "Fedora-40-20240414.0",

+                 "release": 40,

+                 "subvariant": "Cloud_Base",

+                 "family": "fedora-cloud-40",

+                 "image_name": "fedora-cloud-40-1-14-x86-64",

+                 "image_url": "http://example.com/link",

+                 "storage_locations": ["us"],

+             },

+         ),

+     ]

+     with fm_testing.mock_sends(*expected_messages):

+         consumer(msg)

+ 

+ 

+ @pytest.mark.vcr

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_import_image(fixtures_dir):

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.upload_disk_image = mock.Mock()

+     handler.promote_image = mock.Mock()

+     handler.cleanup_old_image = mock.Mock()

+     handler.images_client.list.return_value = []

+     handler.images_client.insert.return_value.warnings = []

+     handler.upload_disk_image.return_value = mock.Mock(

+         spec=storage.Blob, self_link="http://example.com/link"

+     )

+     expected_calls = [

+         mock.call(

+             compute_v1.InsertImageRequest(

+                 request_id="7153713f-2a44-6073-76b4-5786f609026c",

+                 project="fedora-cloud-devel",

+                 image_resource=compute_v1.Image(

+                     architecture="ARM64",

+                     family="fedora-cloud-40",

+                     description="Fedora Cloud base image version 40.1.14",

+                     deprecated=compute_v1.DeprecationStatus(state="ACTIVE"),

+                     guest_os_features=[

+                         compute_v1.GuestOsFeature(

+                             type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name

+                         ),

+                         compute_v1.GuestOsFeature(

+                             type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name

+                         ),

+                         compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),

+                         compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),

+                     ],

+                     labels={

+                         "fedora-compose-id": "fedora-40-20240414-0",

+                         "fedora-subvariant": "cloud_base",

+                         "fedora-release": "40",

+                         "fedora-version": "40",

+                         "end-of-life": "2025-05-13",

+                         "fedora-image-uploader-managed": "true",

+                     },

+                     name="fedora-cloud-40-1-14-aarch64",

+                     raw_disk=compute_v1.RawDisk(

+                         source=handler.upload_disk_image.return_value.self_link

+                     ),

+                     storage_locations=["us"],

+                 ),

+             ),

+             timeout=60,

+         ),

+         mock.call(

+             compute_v1.InsertImageRequest(

+                 request_id="dcc05425-e07b-87f6-f53f-142c693bff3d",

+                 project="fedora-cloud-devel",

+                 image_resource=compute_v1.Image(

+                     architecture="X86_64",

+                     family="fedora-cloud-40",

+                     description="Fedora Cloud base image version 40.1.14",

+                     deprecated=compute_v1.DeprecationStatus(state="ACTIVE"),

+                     guest_os_features=[

+                         compute_v1.GuestOsFeature(

+                             type_=compute_v1.GuestOsFeature.Type.UEFI_COMPATIBLE.name

+                         ),

+                         compute_v1.GuestOsFeature(

+                             type_=compute_v1.GuestOsFeature.Type.VIRTIO_SCSI_MULTIQUEUE.name

+                         ),

+                         compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.IDPF.name),

+                         compute_v1.GuestOsFeature(type_=compute_v1.GuestOsFeature.Type.GVNIC.name),

+                         compute_v1.GuestOsFeature(

+                             type_=compute_v1.GuestOsFeature.Type.SECURE_BOOT.name

+                         ),

+                     ],

+                     labels={

+                         "fedora-compose-id": "fedora-40-20240414-0",

+                         "fedora-subvariant": "cloud_base",

+                         "fedora-release": "40",

+                         "fedora-version": "40",

+                         "end-of-life": "2025-05-13",

+                         "fedora-image-uploader-managed": "true",

+                     },

+                     name="fedora-cloud-40-1-14-x86-64",

+                     raw_disk=compute_v1.RawDisk(

+                         source=handler.upload_disk_image.return_value.self_link

+                     ),

+                     storage_locations=["us"],

+                 ),

+             ),

+             timeout=60,

+         ),

+     ]

+ 

+     with open(os.path.join(fixtures_dir, "messages/rc_compose.json")) as fd:

+         consumer(message.load_message(json.load(fd)))

+ 

+     for call in handler.images_client.insert.call_args_list:

+         assert call in expected_calls

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_promote_image_eln_rawhide():

+     """Assert this is a no-op for eln and rawhide"""

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+ 

+     # rawhide

+     handler.promote_image(

+         compute_v1.Image(

+             architecture="X86_64",

+             family="fedora-40",

+             description="Fedora Cloud base image version",

+             labels={

+                 "fedora-release": "rawhide",

+                 "fedora-image-uploader-managed": "true",

+             },

+             name="fedora-40-1-14-x86-64",

+             raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+             storage_locations=["us"],

+             creation_timestamp="2024-10-01T00:00:00Z",

+         )

+     )

+     handler.images_client.list.call_count = 0

+ 

+     # eln

+     handler.promote_image(

+         compute_v1.Image(

+             architecture="X86_64",

+             family="fedora-40",

+             description="Fedora Cloud base image version",

+             labels={

+                 "fedora-release": "eln",

+                 "fedora-image-uploader-managed": "true",

+             },

+             name="fedora-40-1-14-x86-64",

+             raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+             storage_locations=["us"],

+             creation_timestamp="2024-10-01T00:00:00Z",

+         )

+     )

+     handler.images_client.list.call_count = 0

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_needs_promotion():

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.images_client.list.side_effect = (

+         [

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="olde",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-01T00:00:00Z",

+             )

+         ],

+     )

+     handler.images_client.deprecate.return_value.warnings = []

+ 

+     with freezegun.freeze_time("2024-10-16"):

+         handler.promote_image(

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="promoted",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-15T00:00:00Z",

+             )

+         )

+ 

+     handler.images_client.deprecate.assert_called_once_with(

+         request=compute_v1.DeprecateImageRequest(

+             project="fedora-cloud-devel",

+             image="promoted",

+             deprecation_status_resource=compute_v1.DeprecationStatus(

+                 state=compute_v1.DeprecationStatus.State.ACTIVE.name,

+             ),

+         ),

+         timeout=60,

+     )

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_no_promotion():

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.images_client.list.side_effect = (

+         [

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="olde",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-05T00:00:00Z",

+             )

+         ],

+     )

+     handler.images_client.deprecate.return_value.warnings = []

+ 

+     with freezegun.freeze_time("2024-10-16"):

+         handler.promote_image(

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="promoted",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-15T00:00:00Z",

+             )

+         )

+ 

+     assert handler.images_client.deprecate.call_count == 0

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_cleanup_skips_unmanaged_images():

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.images_client.list.return_value = [

+         compute_v1.Image(

+             architecture="X86_64",

+             family="fedora-40",

+             description="Fedora Cloud base image version",

+             name="fedora-40-1-14-aarch64",

+             raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+             storage_locations=["us"],

+         )

+     ]

+ 

+     # Test that cleanup_old_images is not called when the image is not managed

+     handler.cleanup_old_images()

+     assert handler.images_client.delete.call_count == 0

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_cleanup_rawhide():

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.images_client.list.side_effect = (

+         [

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "rawhide",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="gone-but-not-forgotten",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-01T00:00:00Z",

+             ),

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "rawhide",

+                     "fedora-image-uploader-managed": "true",

+                 },

+                 name="soon-to-go-but-not-yet",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-02T00:00:00Z",

+             ),

+         ],

+         [],

+     )

+     handler.images_client.delete.return_value.warnings = []

+ 

+     # Test that cleanup_old_images is not called when the image is not managed

+     with freezegun.freeze_time("2024-10-16"):

+         handler.cleanup_old_images()

+     assert handler.images_client.delete.call_count == 1

+     handler.images_client.delete.assert_called_once_with(

+         compute_v1.DeleteImageRequest(

+             project="fedora-cloud-devel",

+             image="gone-but-not-forgotten",

+             request_id="347bf095-cee1-dc0f-8b0b-287da4300d98",

+         ),

+         timeout=600,

+     )

+ 

+ 

+ @mock.patch.dict(

+     config.conf,

+     {

+         "consumer_config": {

+             "gcp": {

+                 "project": "fedora-cloud-devel",

+                 "bucket_name": "some-unique-bucket-name",

+                 "storage_locations": ["us"],

+             }

+         },

+     },

+ )

+ def test_cleanup_eol():

+     consumer = Uploader()

+     handler = consumer.handlers["gcp"]

+     handler.images_client = mock.Mock()

+     handler.images_client.list.side_effect = (

+         [],

+         [

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                     "end-of-life": "2024-10-01",

+                 },

+                 name="gone-but-not-forgotten",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-01T00:00:00Z",

+             ),

+             compute_v1.Image(

+                 architecture="X86_64",

+                 family="fedora-40",

+                 description="Fedora Cloud base image version",

+                 labels={

+                     "fedora-release": "40",

+                     "fedora-image-uploader-managed": "true",

+                     "end-of-life": "2024-10-17",

+                 },

+                 name="still-alive",

+                 raw_disk=compute_v1.RawDisk(source="http://example.com/link"),

+                 storage_locations=["us"],

+                 creation_timestamp="2024-10-01T00:00:00Z",

+             ),

+         ],

+     )

+     handler.images_client.delete.return_value.warnings = []

+ 

+     # Test that cleanup_old_images is not called when the image is not managed

+     with freezegun.freeze_time("2024-10-16"):

+         handler.cleanup_old_images()

+ 

+     handler.images_client.delete.assert_called_once_with(

+         compute_v1.DeleteImageRequest(

+             project="fedora-cloud-devel",

+             image="gone-but-not-forgotten",

+             request_id="347bf095-cee1-dc0f-8b0b-287da4300d98",

+         ),

+         timeout=600,

+     )

@@ -5,7 +5,7 @@ 

  [testenv]

  deps =

      ../fedora-image-uploader-messages/

-     .[test,azure,aws]

+     .[test,azure,aws,gcp]

  sitepackages = False

  skip_install = True

  commands_pre =

@@ -35,6 +35,7 @@ 

  exchange = "amq.topic"

  routing_keys = ["org.fedoraproject.*.pungi.compose.status.change"]

  

+ 

  [consumer_config.azure]

  location = "eastus"

  resource_group_name = "fedora-test"
@@ -108,6 +109,11 @@ 

  ]

  

  

+ [consumer_config.gcp]

+ project = "fedora-cloud-devel"

+ bucket_name = "fedora-cloud-image-upload-devel"

+ storage_locations = ["us"]

+ 

  [qos]

  prefetch_size = 0

  prefetch_count = 25