src/cmd-push-container-manifest: support pushing build artifacts

This commit adds an --artifact option to `cosa push-container-manifest`
which allows for pushing artifacts that were built as part of a
pipeline build and are referenced in the `meta.json` file to a remote
registry.

It is multi-arch aware and defaults to pushing a manifest list with
all the requested architectures available for that build.

Example usage:

cosa push-container-manifest \
    --repo quay.io/dustymabe/fedora-coreos --tag stable --artifact=ostree \
    --metajsonname=base-oscontainer --build=latest --arch=x86_64 --arch=aarch64

(cherry picked from commit a8259fe)
(cherry picked from commit cc925ec)
(cherry picked from commit 6f5edcf)

src/cmd-push-container-manifest

@@ -6,7 +6,11 @@
 import argparse
 import os
 import sys
+import tempfile
 from cosalib.container_manifest import create_and_push_container_manifest
+from cosalib.builds import Builds
+from cosalib.meta import GenericBuildMeta
+from cosalib.cmdlib import sha256sum_file
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -15,8 +19,57 @@ def main():
     args = parse_args()
     if args.authfile:
         os.environ["REGISTRY_AUTH_FILE"] = args.authfile
-    create_and_push_container_manifest(
-        args.repo, args.tag, args.images, args.v2s2)
+    if args.images:
+        # User provided images directly
+        create_and_push_container_manifest(
+            args.repo, args.tag, args.images, args.v2s2, None)
+    else:
+        # Picking up images from artifacts in meta.json
+        builds = Builds()
+        if args.build == 'latest':
+            args.build = builds.get_latest()
+        print(f"Targeting build: {args.build}")
+        build_arches = builds.get_build_arches(args.build)
+        if not args.arches:
+            args.arches = build_arches
+        # Iterate over the requested architectures and:
+        #   - Make sure the container images exist and are on disk
+        #   - Store the buildmeta for the build/arch in the buildmetas dict
+        #   - Store the path to the container image in the container_images list
+        images = []
+        buildmetas = dict()
+        for arch in args.arches:
+            if arch not in build_arches:
+                print(f"Requested architecture {arch} is not in {args.build}")
+                raise Exception
+            builddir = builds.get_build_dir(build_id=args.build, basearch=arch)
+            buildmeta = GenericBuildMeta(build=args.build, basearch=arch,
+                                         workdir=os.path.abspath(os.getcwd()))
+            buildmetas[arch] = buildmeta
+            if not buildmeta['images'][args.artifact]:
+                print(f"No artifact {args.artifact} in {args.build}/{arch}")
+                raise Exception
+            ociarchive = os.path.join(builddir, buildmeta['images'][args.artifact]['path'])
+            ocisha256sum = buildmeta['images'][args.artifact]['sha256']
+            if not os.path.exists(ociarchive):
+                print(f"The file does not exist on disk: {ociarchive}")
+                raise Exception
+            if sha256sum_file(ociarchive) != ocisha256sum:
+                print(f"The file on disk {ociarchive} has an incorrect checksum")
+                raise Exception
+            images.append(f"oci-archive:{ociarchive}")
+
+        # Create/Upload the manifest list to the container registry
+        with tempfile.NamedTemporaryFile() as digestfile:
+            create_and_push_container_manifest(
+                args.repo, args.tag, images, args.v2s2, digestfile.name)
+            digestfile.seek(0)
+            digest = digestfile.read().decode('utf-8').strip()
+
+        # Update the meta.json in each build/arch metadata
+        for _, buildmeta in buildmetas.items():
+            buildmeta[args.metajsonname] = {'image': f"{args.repo}@{digest}"}
+        buildmeta.write(artifact_name=args.metajsonname)
 
 
 def parse_args():
@@ -36,17 +89,32 @@ Examples:
         --repo quay.io/dustymabe/fedora-coreos --tag stable \\
         --image oci-archive://builds/36.20220716.3.1/x86_64/fedora-coreos-37.20220725.91.0-ostree.x86_64.ociarchive \\
         --image oci-archive://builds/36.20220716.3.1/aarch64/fedora-coreos-37.20220725.91.0-ostree.aarch64.ociarchive \\
-        --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive""")
+        --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive
+
+    cosa push-container-manifest \\
+        --repo quay.io/dustymabe/fedora-coreos --tag stable --artifact=ostree \\
+        --metajsonname=base-oscontainer --build=latest --arch=x86_64 --arch=aarch64""")
     parser.add_argument("--repo", required=True, help="The registry repo to target for the manifest")
     parser.add_argument("--tag", required=True, help="The tag of the manifest to use")
     parser.add_argument("--authfile", help="A file to use for registry auth")
     parser.add_argument('--v2s2', action='store_true',
                         help='Use old image manifest version2 schema 2 format')
-    parser.add_argument("--images", required=True, action='append', default=[],
-                        help="""The images to add to the manifest. Can be specified multiple times like
-                        --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456
-                        --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""")
 
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--image", dest='images', action='append', default=[],
+                       help="""The images to add to the manifest. Can be specified multiple times like
+                       --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456
+                       --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""")
+    group.add_argument("--artifact", help="""The artifact""")
+
+    # A few more arguments that are used for `--artifact`
+    parser.add_argument("--build", default="latest", help="Build ID")
+    parser.add_argument("--arch", dest='arches', action='append', default=[],
+                        help="""Limit the architectures to upload to the specificed set
+                        (otherwise it defaults to all available for that build). Can be
+                        specificed multiple times like: --arch x86_64 --arch aarch64""")
+    parser.add_argument("--metajsonname",
+                        help="The name under which to store the container information in meta.json")
     return parser.parse_args()
 
 

src/cosalib/container_manifest.py

@@ -25,11 +25,12 @@ def delete_local_container_manifest(repo, tag):
     runcmd(cmd)
 
 
-def push_container_manifest(repo, tag, v2s2=False):
+def push_container_manifest(repo, tag, digestfile, v2s2=False):
     '''
     Push manifest to registry
     @param repo str registry repository
     @param tag str manifest tag
+    @param digestfile str write container digest to file
     @param v2s2 boolean use to force v2s2 format
     '''
     cmd = ["podman", "manifest", "push",
@@ -39,17 +40,20 @@ def push_container_manifest(repo, tag, v2s2=False):
         # to create a manifest with 2 different mediaType. It seems to be
         # a Quay issue.
         cmd.extend(["--remove-signatures", "-f", "v2s2"])
+    if digestfile:
+        cmd.extend([f"--digestfile={digestfile}"])
     runcmd(cmd)
 
 
-def create_and_push_container_manifest(repo, tag, images, v2s2):
+def create_and_push_container_manifest(repo, tag, images, v2s2, digestfile):
     '''
     Do it all! Create, Push, Cleanup
     @param repo str registry repository
     @param tag str manifest tag
     @param images list of image specifications (including transport)
     @param v2s2 boolean use to force v2s2 format
+    @param digestfile str write container digest to file
     '''
     create_local_container_manifest(repo, tag, images)
-    push_container_manifest(repo, tag, v2s2)
+    push_container_manifest(repo, tag, digestfile, v2s2)
     delete_local_container_manifest(repo, tag)