#9807 Update the container sync script to use podman
Merged a month ago by mohanboddu. Opened a month ago by cverna.
cverna/releng container_update  into  master

@@ -1,147 +0,0 @@ 

- #!/usr/bin/python3

- #

- # generate-manifest-list.py - A script used to create and push the container base image

- #                             manifest list used for mutli arch support

- #                             This is used by sync-latest-container-base-image.sh script

- #

- # Authors:

- #    Clement Verna <cverna@fedoraproject.org>

- # Copyright (C) 2018 Red Hat Inc,

- # SPDX-License-Identifier:	GPL-2.0+

- import argparse

- import json

- import os

- from functools import wraps

- from pathlib import Path

- 

- import requests

- 

- 

- MEDIA_TYPE_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json"

- 

- MANIFEST_LIST = {"schemaVersion": 2, "mediaType": MEDIA_TYPE_LIST_V2, "manifests": []}

- 

- ARCHES = {

-     "x86_64": "amd64",

-     "aarch64": "arm64",

-     "armhfp": "arm",

-     "ppc64le": "ppc64le",

-     "s390x": "s390x",

- }

- 

- 

- def certs(function):

-     """ Decorator that create the certificate path based

-     on the registry name """

- 

-     @wraps(function)

-     def get_certs_path(*args, **kwargs):

-         cert_path = f"/etc/docker/certs.d/{kwargs['registry']}/client.cert"

-         cert_key = f"/etc/docker/certs.d/{kwargs['registry']}/client.key"

-         kwargs["cert"] = (cert_path, cert_key)

-         return function(*args, **kwargs)

- 

-     return get_certs_path

- 

- 

- def get_auth_creds(registry):

-     """ Helper function to get the credentials

-     to authenticate to the candidate-registries

-     """

- 

-     config_path = os.path.join(Path.home(), ".docker/config.json")

-     if os.path.exists(config_path):

-         with open(config_path) as fp:

-             config = json.load(fp)

-         secrets = config.get("auths")

-         return secrets.get(registry)

- 

-     else:

-         print(f"ERROR: {config_path} does not exists")

-         return None

- 

- 

- @certs

- def create_image_manifest(tag, name, registry, cert):

-     """ Get the manifest for a specific image and returns the data

-     needed to build the manifest list """

- 

-     image = None

-     headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"}

-     res = requests.get(

-         f"https://{registry}/v2/{name}/manifests/{tag}", cert=cert, headers=headers

-     )

- 

-     if res.ok:

-         image = {

-             "mediaType": res.headers["Content-Type"],

-             "size": int(res.headers["Content-Length"]),

-             "digest": res.headers["Docker-Content-Digest"],

-             "platform": {"architecture": "", "os": "linux"},

-         }

- 

-     return image

- 

- 

- @certs

- def push_manifest_list(manifest_list, tags, name, registry, cert):

-     """ Push the manifest list to the correct tags """

- 

-     headers = {"Content-Type": MEDIA_TYPE_LIST_V2}

- 

-     print(f"Pushing the manifest list to {registry}")

-     print(f"Pushing the manifest list for tags : {tags}")

- 

-     # For candidate registries we need to use basic auth

-     if "candidate" in registry:

-         secrets = get_auth_creds(registry)

-         if secrets is not None:

-             auth = secrets.get("auth")

-             headers["Authorization"] = f"Basic {auth}"

- 

-     for tag in tags:

-         res = requests.put(

-             f"https://{registry}/v2/{name}/manifests/{tag}",

-             data=json.dumps(manifest_list),

-             headers=headers,

-             cert=cert,

-         )

- 

-         if not res.ok:

-             print(f"ERROR: Failed to push the manifest list : {res.text}")

- 

- 

- if __name__ == "__main__":

- 

-     parser = argparse.ArgumentParser()

-     parser.add_argument(

-         "--release", "-r", help="number of the fedora release", required=True

-     )

-     parser.add_argument(

-         "--registry",

-         help="name of the container registry",

-         default="registry.fedoraproject.org",

-     )

-     parser.add_argument("--tag", help="tag to apply to the container image")

-     parser.add_argument("--image", help="name of the container image", default="fedora")

-     args = parser.parse_args()

- 

-     tags = [f"{args.release}-" + arch for arch in ARCHES]

- 

-     for tag in tags:

-         image = create_image_manifest(tag=tag, name=args.image, registry=args.registry)

-         if image is not None:

-             manifest = image.copy()

-             manifest["platform"]["architecture"] = ARCHES[tag[3:]]

-             MANIFEST_LIST["manifests"].append(manifest)

-         else:

-             print(f"ERROR : Could not find the image manifest for fedora:{tag}")

- 

-     if args.tag:

-         tags = [args.release, args.tag]

-     else:

-         tags = [args.release]

- 

-     push_manifest_list(

-         manifest_list=MANIFEST_LIST, tags=tags, name=args.image, registry=args.registry

-     )

@@ -16,7 +16,7 @@ 

  SYNOPSIS

      ${0} FEDORA_RELEASE IMAGE_URL [-s]

  OPTIONS

-     -s  - Sync the stage registries instead of production

+     -s  - Sync the production and stage registries.

  DESCRIPTION

      This is a stop-gap solution to identify parent inheritance of container

      images until the container release automation[0] is in place and we have
@@ -58,91 +58,133 @@ 

  #   Need fXX-updates-canddiate to get actual latest nightly

  #

  build_name=$(koji -q latest-build --type=image f${1}-updates-candidate Fedora-Container-Base | awk '{print $1}')

- 

- if [[ ${1} -eq "$current_stable" ]]; then

-     tagname="latest"

- fi

- if [[ ${1} -eq "$current_rawhide" ]]; then

-     tagname="rawhide"

- fi

- 

  minimal_build_name=$(koji -q latest-build --type=image f${1}-updates-candidate Fedora-Container-Minimal-Base | awk '{print $1}')

+ 

+ #Push the base image

  if [[ -n ${build_name} ]]; then

+ 

      # Download the image

      work_dir=$(mktemp -d)

      pushd ${work_dir} &> /dev/null

      koji download-build --type=image  ${build_name}

+ 

+     # Create the manifest list for multi-arch support

+     podman manifest create registry.fedoraproject.org/fedora:${1}

+     podman manifest create candidate-registry.fedoraproject.org/fedora:${1}

+     podman manifest create quay.io/fedora/fedora:${1}

+ 

      # Import the image

      for arch in "${ARCHES[@]}"

      do

          xz -d ${build_name}.${arch}.tar.xz

-         # If ${stage} is a non-zero length string, then perform staging

-         if [[ -z "$stage" ]]; then

-             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")

-             skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora:${1}-${arch}

-             skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}

-             skopeo copy docker-archive:${build_name}.${arch}.tar docker://quay.io/fedora/fedora:${1}-${arch}

-         else

-             registries=("registry.stg.fedoraproject.org" "candidate-registry.stg.fedoraproject.org")

-             skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.stg.fedoraproject.org/fedora:${1}-${arch}

-             skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.stg.fedoraproject.org/fedora:${1}-${arch}

-         fi

+         skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora:${1}-${arch}

+         skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}

+         skopeo copy docker-archive:${build_name}.${arch}.tar docker://quay.io/fedora/fedora:${1}-${arch}

+         podman manifest add registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:${1}-${arch}

+         podman manifest add candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}

+         podman manifest add quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:${1}-${arch}

      done

- 

      popd &> /dev/null

  

-     for registry in "${registries[@]}"

-     do

-         printf "Push manifest to ${registry}\n"

-         if [ -z "$tagname" ]

-         then

-             printf "tag is not set: ${tagname}\n"

-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora

-         else

-             printf "tag is set: ${tagname}\n"

-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora

+     podman manifest push registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:${1}

+     podman manifest push candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:${1}

+     podman manifest push quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:${1}

+ 

+     # Create the latest tag

+     if [[ ${1} -eq "$current_stable" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:latest

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:latest

+         skopeo copy --all docker://quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:latest

+     fi

+     # Create the rawhide tag

+     if [[ ${1} -eq "$current_rawhide" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:rawhide

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:rawhide

+         skopeo copy --all docker://quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:rawhide

+     fi

+ 

+     # Copy the images in staging

+     if ! [[ -z "$stage" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:${1}

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://create candidate-registry.stg.fedoraproject.org/fedora:${1}

+ 

+         if [[ ${1} -eq "$current_stable" ]]; then

+             skopeo copy --all docker://registry.stg.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:latest

+             skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora:${1} docker://candidate-registry.stg.fedoraproject.org/fedora:latest

          fi

-     done

+         if [[ ${1} -eq "$current_rawhide" ]]; then

+             skopeo copy --all docker://registry.stg.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:rawhide

+             skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora:${1} docker://candidate-registry.stg.fedoraproject.org/fedora:latest

+         fi

+     fi

      printf "Removing temporary directory\n"

      rm -rf $work_dir

+     podman rmi -f registry.fedoraproject.org/fedora:${1}

+     podman rmi -f candidate-registry.fedoraproject.org/fedora:${1}

+     podman rmi -f quay.io/fedora/fedora:${1}

  fi

+ 

+ #Push the minimal base image

  if [[ -n ${minimal_build_name} ]]; then

      # Download the image

      work_dir=$(mktemp -d)

      pushd ${work_dir} &> /dev/null

      koji download-build --type=image  ${minimal_build_name}

+ 

+     # Create the manifest list for multi-arch support

+     podman manifest create registry.fedoraproject.org/fedora-minimal:${1}

+     podman manifest create candidate-registry.fedoraproject.org/fedora-minimal:${1}

+     podman manifest create quay.io/fedora/fedora-minimal:${1}

+ 

      # Import the image

      for arch in "${ARCHES[@]}"

      do

          xz -d ${minimal_build_name}.${arch}.tar.xz

-         # If ${stage} is a non-zero length string, then perform staging

-         if [[ -z "$stage" ]]; then

-             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")

-             skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}

-             skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}

-         else

-             registries=("registry.stg.fedoraproject.org" "candidate-registry.stg.fedoraproject.org")

-             skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.stg.fedoraproject.org/fedora-minimal:${1}-${arch}

-             skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1}-${arch}

-             skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://quay.io/fedora/fedora-minimal:${1}-${arch}

-         fi

+         skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}

+         skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}

+         skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://quay.io/fedora/fedora-minimal:${1}-${arch}

+         podman manifest add registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}

+         podman manifest add candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}

+         podman manifest add quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:${1}-${arch}

+ 

       done

       popd &> /dev/null

  

-      for registry in "${registries[@]}"

-      do

-          printf "Push manifest to ${registry}\n"

-          if [ -z "$tagname" ]

-          then

-              printf "tag is not set: ${tagname}\n"

-              python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora-minimal

-          else

-              printf "tag is set: ${tagname}\n"

-              python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora-minimal

-          fi

-      done

+     podman manifest push registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:${1}

+     podman manifest push candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}

+     podman manifest push quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:${1}

+ 

+     # Create the latest tag

+     if [[ ${1} -eq "$current_stable" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:latest

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:latest

+         skopeo copy --all docker://quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:latest

+     fi

+     # Create the rawhide tag

+     if [[ ${1} -eq "$current_rawhide" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:rawhide

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:rawhide

+         skopeo copy --all docker://quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:rawhide

+     fi

+ 

+     # Copy the images in staging

+     if ! [[ -z "$stage" ]]; then

+         skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:${1}

+         skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://create candidate-registry.stg.fedoraproject.org/fedora-minimal:${1}

+ 

+         if [[ ${1} -eq "$current_stable" ]]; then

+             skopeo copy --all docker://registry.stg.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:latest

+             skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:latest

+         fi

+         if [[ ${1} -eq "$current_rawhide" ]]; then

+             skopeo copy --all docker://registry.stg.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:rawhide

+             skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:latest

+         fi

+     fi

  

       printf "Removing temporary directory\n"

       rm -rf $work_dir

- 

+      podman rmi -f registry.fedoraproject.org/fedora-minimal:${1}

+      podman rmi -f candidate-registry.fedoraproject.org/fedora-minimal:${1}

+      podman rmi -f quay.io/fedora/fedora-minimal:${1}

  fi

We can now use podman to generate the manifest list instead of
a custom python script. This commit also changes the way pushing
changes to staging works. You can now push production and staging
at the same time instead of having to run the script twice.

Signed-off-by: Clement Verna cverna@tutanota.com

Do we want to wait for F33 release before merging this ? or are we ok with merging now ?

rebased onto 1c08c7e

a month ago

Yeah, lets wait for F33 release

rebased onto 0b3e9d5

a month ago

Pull-Request has been merged by mohanboddu

a month ago