#10249 Use buildah to generate the manifest list.
Merged 2 years ago by cverna. Opened 2 years ago by cverna.
cverna/releng releng_9880  into  main

@@ -1,147 +0,0 @@ 

- #!/usr/bin/python3

- #

- # generate-manifest-list.py - A script used to create and push the container base image

- #                             manifest list used for mutli arch support

- #                             This is used by sync-latest-container-base-image.sh script

- #

- # Authors:

- #    Clement Verna <cverna@fedoraproject.org>

- # Copyright (C) 2018 Red Hat Inc,

- # SPDX-License-Identifier:	GPL-2.0+

- import argparse

- import json

- import os

- from functools import wraps

- from pathlib import Path

- 

- import requests

- 

- 

- MEDIA_TYPE_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json"

- 

- MANIFEST_LIST = {"schemaVersion": 2, "mediaType": MEDIA_TYPE_LIST_V2, "manifests": []}

- 

- ARCHES = {

-     "x86_64": "amd64",

-     "aarch64": "arm64",

-     "armhfp": "arm",

-     "ppc64le": "ppc64le",

-     "s390x": "s390x",

- }

- 

- 

- def certs(function):

-     """ Decorator that create the certificate path based

-     on the registry name """

- 

-     @wraps(function)

-     def get_certs_path(*args, **kwargs):

-         cert_path = f"/etc/docker/certs.d/{kwargs['registry']}/client.cert"

-         cert_key = f"/etc/docker/certs.d/{kwargs['registry']}/client.key"

-         kwargs["cert"] = (cert_path, cert_key)

-         return function(*args, **kwargs)

- 

-     return get_certs_path

- 

- 

- def get_auth_creds(registry):

-     """ Helper function to get the credentials

-     to authenticate to the candidate-registries

-     """

- 

-     config_path = os.path.join(Path.home(), ".docker/config.json")

-     if os.path.exists(config_path):

-         with open(config_path) as fp:

-             config = json.load(fp)

-         secrets = config.get("auths")

-         return secrets.get(registry)

- 

-     else:

-         print(f"ERROR: {config_path} does not exists")

-         return None

- 

- 

- @certs

- def create_image_manifest(tag, name, registry, cert):

-     """ Get the manifest for a specific image and returns the data

-     needed to build the manifest list """

- 

-     image = None

-     headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"}

-     res = requests.get(

-         f"https://{registry}/v2/{name}/manifests/{tag}", cert=cert, headers=headers

-     )

- 

-     if res.ok:

-         image = {

-             "mediaType": res.headers["Content-Type"],

-             "size": int(res.headers["Content-Length"]),

-             "digest": res.headers["Docker-Content-Digest"],

-             "platform": {"architecture": "", "os": "linux"},

-         }

- 

-     return image

- 

- 

- @certs

- def push_manifest_list(manifest_list, tags, name, registry, cert):

-     """ Push the manifest list to the correct tags """

- 

-     headers = {"Content-Type": MEDIA_TYPE_LIST_V2}

- 

-     print(f"Pushing the manifest list to {registry}")

-     print(f"Pushing the manifest list for tags : {tags}")

- 

-     # For candidate registries we need to use basic auth

-     if "candidate" in registry:

-         secrets = get_auth_creds(registry)

-         if secrets is not None:

-             auth = secrets.get("auth")

-             headers["Authorization"] = f"Basic {auth}"

- 

-     for tag in tags:

-         res = requests.put(

-             f"https://{registry}/v2/{name}/manifests/{tag}",

-             data=json.dumps(manifest_list),

-             headers=headers,

-             cert=cert,

-         )

- 

-         if not res.ok:

-             print(f"ERROR: Failed to push the manifest list : {res.text}")

- 

- 

- if __name__ == "__main__":

- 

-     parser = argparse.ArgumentParser()

-     parser.add_argument(

-         "--release", "-r", help="number of the fedora release", required=True

-     )

-     parser.add_argument(

-         "--registry",

-         help="name of the container registry",

-         default="registry.fedoraproject.org",

-     )

-     parser.add_argument("--tag", help="tag to apply to the container image")

-     parser.add_argument("--image", help="name of the container image", default="fedora")

-     args = parser.parse_args()

- 

-     tags = [f"{args.release}-" + arch for arch in ARCHES]

- 

-     for tag in tags:

-         image = create_image_manifest(tag=tag, name=args.image, registry=args.registry)

-         if image is not None:

-             manifest = image.copy()

-             manifest["platform"]["architecture"] = ARCHES[tag[3:]]

-             MANIFEST_LIST["manifests"].append(manifest)

-         else:

-             print(f"ERROR : Could not find the image manifest for fedora:{tag}")

- 

-     if args.tag:

-         tags = [args.release, args.tag]

-     else:

-         tags = [args.release]

- 

-     push_manifest_list(

-         manifest_list=MANIFEST_LIST, tags=tags, name=args.image, registry=args.registry

-     )

@@ -78,7 +78,7 @@ 

          xz -d ${build_name}.${arch}.tar.xz

          # If ${stage} is a non-zero length string, then perform staging

          if [[ -z "$stage" ]]; then

-             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")

+             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org" "quay.io/fedora")

              skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora:${1}-${arch}

              skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}

              skopeo copy docker-archive:${build_name}.${arch}.tar docker://quay.io/fedora/fedora:${1}-${arch}
@@ -94,14 +94,17 @@ 

      for registry in "${registries[@]}"

      do

          printf "Push manifest to ${registry}\n"

-         if [ -z "$tagname" ]

+         if [ -n "$tagname" ]

          then

-             printf "tag is not set: ${tagname}\n"

-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora

-         else

              printf "tag is set: ${tagname}\n"

-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora

-         fi

+             buildah rmi "${registry}/fedora:${tagname}" || true

+             buildah manifest create "${registry}/fedora:${tagname}" "${ARCHES[@]/#/docker://${registry}/fedora:${1}-}"

+             buildah manifest push "${registry}/fedora:${tagname}" "docker://${registry}/fedora:${tagname}" --all

+ 

+         fi 

+         buildah rmi "${registry}/fedora:${1}" || true

+         buildah manifest create "${registry}/fedora:${1}" "${ARCHES[@]/#/docker://${registry}/fedora:${1}-}"

+         buildah manifest push "${registry}/fedora:${1}" "docker://${registry}/fedora:${1}" --all

      done

      printf "Removing temporary directory\n"

      rm -rf $work_dir
@@ -117,7 +120,7 @@ 

          xz -d ${minimal_build_name}.${arch}.tar.xz

          # If ${stage} is a non-zero length string, then perform staging

          if [[ -z "$stage" ]]; then

-             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")

+             registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org" "quay.io/fedora")

              skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}

              skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}

          else
@@ -132,14 +135,16 @@ 

       for registry in "${registries[@]}"

       do

           printf "Push manifest to ${registry}\n"

-          if [ -z "$tagname" ]

+          if [ -n "$tagname" ]

           then

-              printf "tag is not set: ${tagname}\n"

-              python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora-minimal

-          else

               printf "tag is set: ${tagname}\n"

-              python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora-minimal

-          fi

+              buildah rmi "${registry}/fedora-minimal:${tagname}" || true

+              buildah manifest create "${registry}/fedora-minimal:${tagname}" "${ARCHES[@]/#/docker://${registry}/fedora-minimal:${1}-}"

+              buildah manifest push "${registry}/fedora-minimal:${tagname}" "docker://${registry}/fedora-minimal:${tagname}" --all

+          fi 

+          buildah rmi "${registry}/fedora-minimal:${1}" || true

+          buildah manifest create "${registry}/fedora-minimal:${1}" "${ARCHES[@]/#/docker://${registry}/fedora-minimal:${1}-}"

+          buildah manifest push "${registry}/fedora-minimal:${1}" "docker://${registry}/fedora-minimal:${1}" --all

       done

  

       printf "Removing temporary directory\n"

This commit replaces how we build the manifest list for
the fedora base image container. This now uses buildah
manifest command instead of a custom script.

We use buildah since the manifest create command support
passing multiple argument compared to podman manifest create
which takes only 2 arguments.

Fixes #9880

Signed-off-by: Clement Verna cverna@tutanota.com

rebased onto ca97ff02be7d953b39fd0b18decb8cff87141950

2 years ago

rebased onto 05c98aef58c6a52a4cdc9d6af3af763d2b3b28f1

2 years ago

rebased onto 4409bb64de41077616128b7ea5cd7924f4e5382d

2 years ago

rebased onto 6fb8b3a

2 years ago

Pull-Request has been merged by cverna

2 years ago