From 0b3e9d5a53162245635e07b6f7e9b3a008da05cd Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Oct 27 2020 17:57:06 +0000
Subject: Update the container sync script to use podman


We can now use podman to generate the manifest list instead of
a custom python script. This commit also changes the way pushing
changes to staging works. You can now push production and staging
at the same time instead of having to run the script twice.

Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/scripts/generate-manifest-list.py b/scripts/generate-manifest-list.py
deleted file mode 100644
index 5dad2fc..0000000
--- a/scripts/generate-manifest-list.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/python3
-#
-# generate-manifest-list.py - A script used to create and push the container base image
-#                             manifest list used for mutli arch support
-#                             This is used by sync-latest-container-base-image.sh script
-#
-# Authors:
-#    Clement Verna <cverna@fedoraproject.org>
-# Copyright (C) 2018 Red Hat Inc,
-# SPDX-License-Identifier:	GPL-2.0+
-import argparse
-import json
-import os
-from functools import wraps
-from pathlib import Path
-
-import requests
-
-
-MEDIA_TYPE_LIST_V2 = "application/vnd.docker.distribution.manifest.list.v2+json"
-
-MANIFEST_LIST = {"schemaVersion": 2, "mediaType": MEDIA_TYPE_LIST_V2, "manifests": []}
-
-ARCHES = {
-    "x86_64": "amd64",
-    "aarch64": "arm64",
-    "armhfp": "arm",
-    "ppc64le": "ppc64le",
-    "s390x": "s390x",
-}
-
-
-def certs(function):
-    """ Decorator that create the certificate path based
-    on the registry name """
-
-    @wraps(function)
-    def get_certs_path(*args, **kwargs):
-        cert_path = f"/etc/docker/certs.d/{kwargs['registry']}/client.cert"
-        cert_key = f"/etc/docker/certs.d/{kwargs['registry']}/client.key"
-        kwargs["cert"] = (cert_path, cert_key)
-        return function(*args, **kwargs)
-
-    return get_certs_path
-
-
-def get_auth_creds(registry):
-    """ Helper function to get the credentials
-    to authenticate to the candidate-registries
-    """
-
-    config_path = os.path.join(Path.home(), ".docker/config.json")
-    if os.path.exists(config_path):
-        with open(config_path) as fp:
-            config = json.load(fp)
-        secrets = config.get("auths")
-        return secrets.get(registry)
-
-    else:
-        print(f"ERROR: {config_path} does not exists")
-        return None
-
-
-@certs
-def create_image_manifest(tag, name, registry, cert):
-    """ Get the manifest for a specific image and returns the data
-    needed to build the manifest list """
-
-    image = None
-    headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"}
-    res = requests.get(
-        f"https://{registry}/v2/{name}/manifests/{tag}", cert=cert, headers=headers
-    )
-
-    if res.ok:
-        image = {
-            "mediaType": res.headers["Content-Type"],
-            "size": int(res.headers["Content-Length"]),
-            "digest": res.headers["Docker-Content-Digest"],
-            "platform": {"architecture": "", "os": "linux"},
-        }
-
-    return image
-
-
-@certs
-def push_manifest_list(manifest_list, tags, name, registry, cert):
-    """ Push the manifest list to the correct tags """
-
-    headers = {"Content-Type": MEDIA_TYPE_LIST_V2}
-
-    print(f"Pushing the manifest list to {registry}")
-    print(f"Pushing the manifest list for tags : {tags}")
-
-    # For candidate registries we need to use basic auth
-    if "candidate" in registry:
-        secrets = get_auth_creds(registry)
-        if secrets is not None:
-            auth = secrets.get("auth")
-            headers["Authorization"] = f"Basic {auth}"
-
-    for tag in tags:
-        res = requests.put(
-            f"https://{registry}/v2/{name}/manifests/{tag}",
-            data=json.dumps(manifest_list),
-            headers=headers,
-            cert=cert,
-        )
-
-        if not res.ok:
-            print(f"ERROR: Failed to push the manifest list : {res.text}")
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--release", "-r", help="number of the fedora release", required=True
-    )
-    parser.add_argument(
-        "--registry",
-        help="name of the container registry",
-        default="registry.fedoraproject.org",
-    )
-    parser.add_argument("--tag", help="tag to apply to the container image")
-    parser.add_argument("--image", help="name of the container image", default="fedora")
-    args = parser.parse_args()
-
-    tags = [f"{args.release}-" + arch for arch in ARCHES]
-
-    for tag in tags:
-        image = create_image_manifest(tag=tag, name=args.image, registry=args.registry)
-        if image is not None:
-            manifest = image.copy()
-            manifest["platform"]["architecture"] = ARCHES[tag[3:]]
-            MANIFEST_LIST["manifests"].append(manifest)
-        else:
-            print(f"ERROR : Could not find the image manifest for fedora:{tag}")
-
-    if args.tag:
-        tags = [args.release, args.tag]
-    else:
-        tags = [args.release]
-
-    push_manifest_list(
-        manifest_list=MANIFEST_LIST, tags=tags, name=args.image, registry=args.registry
-    )
diff --git a/scripts/sync-latest-container-base-image.sh b/scripts/sync-latest-container-base-image.sh
index 72cd8ce..c7d966f 100755
--- a/scripts/sync-latest-container-base-image.sh
+++ b/scripts/sync-latest-container-base-image.sh
@@ -16,7 +16,7 @@ NAME
 SYNOPSIS
     ${0} FEDORA_RELEASE IMAGE_URL [-s]
 OPTIONS
-    -s  - Sync the stage registries instead of production
+    -s  - Sync the production and stage registries.
 DESCRIPTION
     This is a stop-gap solution to identify parent inheritance of container
     images until the container release automation[0] is in place and we have
@@ -58,91 +58,133 @@ fi
 #   Need fXX-updates-canddiate to get actual latest nightly
 #
 build_name=$(koji -q latest-build --type=image f${1}-updates-candidate Fedora-Container-Base | awk '{print $1}')
-
-if [[ ${1} -eq "$current_stable" ]]; then
-    tagname="latest"
-fi
-if [[ ${1} -eq "$current_rawhide" ]]; then
-    tagname="rawhide"
-fi
-
 minimal_build_name=$(koji -q latest-build --type=image f${1}-updates-candidate Fedora-Container-Minimal-Base | awk '{print $1}')
+
+#Push the base image
 if [[ -n ${build_name} ]]; then
+
     # Download the image
     work_dir=$(mktemp -d)
     pushd ${work_dir} &> /dev/null
     koji download-build --type=image  ${build_name}
+
+    # Create the manifest list for multi-arch support
+    podman manifest create registry.fedoraproject.org/fedora:${1}
+    podman manifest create candidate-registry.fedoraproject.org/fedora:${1}
+    podman manifest create quay.io/fedora/fedora:${1}
+
     # Import the image
     for arch in "${ARCHES[@]}"
     do
         xz -d ${build_name}.${arch}.tar.xz
-        # If ${stage} is a non-zero length string, then perform staging
-        if [[ -z "$stage" ]]; then
-            registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")
-            skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora:${1}-${arch}
-            skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}
-            skopeo copy docker-archive:${build_name}.${arch}.tar docker://quay.io/fedora/fedora:${1}-${arch}
-        else
-            registries=("registry.stg.fedoraproject.org" "candidate-registry.stg.fedoraproject.org")
-            skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.stg.fedoraproject.org/fedora:${1}-${arch}
-            skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.stg.fedoraproject.org/fedora:${1}-${arch}
-        fi
+        skopeo copy docker-archive:${build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora:${1}-${arch}
+        skopeo copy docker-archive:${build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}
+        skopeo copy docker-archive:${build_name}.${arch}.tar docker://quay.io/fedora/fedora:${1}-${arch}
+        podman manifest add registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:${1}-${arch}
+        podman manifest add candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:${1}-${arch}
+        podman manifest add quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:${1}-${arch}
     done
-
     popd &> /dev/null
 
-    for registry in "${registries[@]}"
-    do
-        printf "Push manifest to ${registry}\n"
-        if [ -z "$tagname" ]
-        then
-            printf "tag is not set: ${tagname}\n"
-            python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora
-        else
-            printf "tag is set: ${tagname}\n"
-            python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora
+    podman manifest push registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:${1}
+    podman manifest push candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:${1}
+    podman manifest push quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:${1}
+
+    # Create the latest tag
+    if [[ ${1} -eq "$current_stable" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:latest
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:latest
+        skopeo copy --all docker://quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:latest
+    fi
+    # Create the rawhide tag
+    if [[ ${1} -eq "$current_rawhide" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.fedoraproject.org/fedora:rawhide
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://candidate-registry.fedoraproject.org/fedora:rawhide
+        skopeo copy --all docker://quay.io/fedora/fedora:${1} docker://quay.io/fedora/fedora:rawhide
+    fi
+
+    # Copy the images in staging
+    if ! [[ -z "$stage" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:${1}
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora:${1} docker://create candidate-registry.stg.fedoraproject.org/fedora:${1}
+
+        if [[ ${1} -eq "$current_stable" ]]; then
+            skopeo copy --all docker://registry.stg.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:latest
+            skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora:${1} docker://candidate-registry.stg.fedoraproject.org/fedora:latest
         fi
-    done
+        if [[ ${1} -eq "$current_rawhide" ]]; then
+            skopeo copy --all docker://registry.stg.fedoraproject.org/fedora:${1} docker://registry.stg.fedoraproject.org/fedora:rawhide
+            skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora:${1} docker://candidate-registry.stg.fedoraproject.org/fedora:latest
+        fi
+    fi
     printf "Removing temporary directory\n"
     rm -rf $work_dir
+    podman rmi -f registry.fedoraproject.org/fedora:${1}
+    podman rmi -f candidate-registry.fedoraproject.org/fedora:${1}
+    podman rmi -f quay.io/fedora/fedora:${1}
 fi
+
+#Push the minimal base image
 if [[ -n ${minimal_build_name} ]]; then
     # Download the image
     work_dir=$(mktemp -d)
     pushd ${work_dir} &> /dev/null
     koji download-build --type=image  ${minimal_build_name}
+
+    # Create the manifest list for multi-arch support
+    podman manifest create registry.fedoraproject.org/fedora-minimal:${1}
+    podman manifest create candidate-registry.fedoraproject.org/fedora-minimal:${1}
+    podman manifest create quay.io/fedora/fedora-minimal:${1}
+
     # Import the image
     for arch in "${ARCHES[@]}"
     do
         xz -d ${minimal_build_name}.${arch}.tar.xz
-        # If ${stage} is a non-zero length string, then perform staging
-        if [[ -z "$stage" ]]; then
-            registries=("registry.fedoraproject.org" "candidate-registry.fedoraproject.org")
-            skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}
-            skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}
-        else
-            registries=("registry.stg.fedoraproject.org" "candidate-registry.stg.fedoraproject.org")
-            skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.stg.fedoraproject.org/fedora-minimal:${1}-${arch}
-            skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1}-${arch}
-            skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://quay.io/fedora/fedora-minimal:${1}-${arch}
-        fi
+        skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}
+        skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}
+        skopeo copy docker-archive:${minimal_build_name}.${arch}.tar docker://quay.io/fedora/fedora-minimal:${1}-${arch}
+        podman manifest add registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:${1}-${arch}
+        podman manifest add candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}-${arch}
+        podman manifest add quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:${1}-${arch}
+
      done
      popd &> /dev/null
 
-     for registry in "${registries[@]}"
-     do
-         printf "Push manifest to ${registry}\n"
-         if [ -z "$tagname" ]
-         then
-             printf "tag is not set: ${tagname}\n"
-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --image fedora-minimal
-         else
-             printf "tag is set: ${tagname}\n"
-             python3 generate-manifest-list.py -r ${1} --registry ${registry} --tag ${tagname} --image fedora-minimal
-         fi
-     done
+    podman manifest push registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:${1}
+    podman manifest push candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:${1}
+    podman manifest push quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:${1}
+
+    # Create the latest tag
+    if [[ ${1} -eq "$current_stable" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:latest
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:latest
+        skopeo copy --all docker://quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:latest
+    fi
+    # Create the rawhide tag
+    if [[ ${1} -eq "$current_rawhide" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.fedoraproject.org/fedora-minimal:rawhide
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.fedoraproject.org/fedora-minimal:rawhide
+        skopeo copy --all docker://quay.io/fedora/fedora-minimal:${1} docker://quay.io/fedora/fedora-minimal:rawhide
+    fi
+
+    # Copy the images in staging
+    if ! [[ -z "$stage" ]]; then
+        skopeo copy --all docker://registry.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:${1}
+        skopeo copy --all docker://candidate-registry.fedoraproject.org/fedora-minimal:${1} docker://create candidate-registry.stg.fedoraproject.org/fedora-minimal:${1}
+
+        if [[ ${1} -eq "$current_stable" ]]; then
+            skopeo copy --all docker://registry.stg.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:latest
+            skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:latest
+        fi
+        if [[ ${1} -eq "$current_rawhide" ]]; then
+            skopeo copy --all docker://registry.stg.fedoraproject.org/fedora-minimal:${1} docker://registry.stg.fedoraproject.org/fedora-minimal:rawhide
+            skopeo copy --all docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:${1} docker://candidate-registry.stg.fedoraproject.org/fedora-minimal:latest
+        fi
+    fi
 
      printf "Removing temporary directory\n"
      rm -rf $work_dir
-
+     podman rmi -f registry.fedoraproject.org/fedora-minimal:${1}
+     podman rmi -f candidate-registry.fedoraproject.org/fedora-minimal:${1}
+     podman rmi -f quay.io/fedora/fedora-minimal:${1}
 fi