From cc925ec723c596cdefb3799aa9cf38dce7a9e8b5 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 13 Sep 2022 13:45:35 -0400 Subject: [PATCH] src/cmd-push-container-manifest: support pushing build artifacts This commit adds an --artifact option to `cosa push-container-manifest` which allows for pushing artifacts that were built as part of a pipeline build and are referenced in the `meta.json` file to a remote registry. It is multi-arch aware and defaults to pushing a manifest list with all the requested architectures available for that build. Example usage: cosa push-container-manifest \ --repo quay.io/dustymabe/fedora-coreos --tag stable --artifact=ostree \ --metajsonname=base-oscontainer --build=latest --arch=x86_64 --arch=aarch64 (cherry picked from commit a8259fe4ffcb7e497a96e7847c20d238f239727b) --- src/cmd-push-container-manifest | 82 ++++++++++++++++++++++++++++--- src/cosalib/container_manifest.py | 10 ++-- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/cmd-push-container-manifest b/src/cmd-push-container-manifest index 27cfc84058..e095c0ee3f 100755 --- a/src/cmd-push-container-manifest +++ b/src/cmd-push-container-manifest @@ -6,7 +6,11 @@ import argparse import os import sys +import tempfile from cosalib.container_manifest import create_and_push_container_manifest +from cosalib.builds import Builds +from cosalib.meta import GenericBuildMeta +from cosalib.cmdlib import sha256sum_file sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -15,8 +19,57 @@ def main(): args = parse_args() if args.authfile: os.environ["REGISTRY_AUTH_FILE"] = args.authfile - create_and_push_container_manifest( - args.repo, args.tag, args.images, args.v2s2) + if args.images: + # User provided images directly + create_and_push_container_manifest( + args.repo, args.tag, args.images, args.v2s2, None) + else: + # Picking up images from artifacts in meta.json + builds = Builds() + if args.build == 'latest': + args.build = builds.get_latest() + print(f"Targeting build: {args.build}") + build_arches = builds.get_build_arches(args.build) + if not args.arches: + args.arches = build_arches + # Iterate over the requested architectures and: + # - Make sure the container images exist and are on disk + # - Store the buildmeta for the build/arch in the buildmetas dict + # - Store the path to the container image in the container_images list + images = [] + buildmetas = dict() + for arch in args.arches: + if arch not in build_arches: + print(f"Requested architecture {arch} is not in {args.build}") + raise Exception + builddir = builds.get_build_dir(build_id=args.build, basearch=arch) + buildmeta = GenericBuildMeta(build=args.build, basearch=arch, + workdir=os.path.abspath(os.getcwd())) + buildmetas[arch] = buildmeta + if not buildmeta['images'][args.artifact]: + print(f"No artifact {args.artifact} in {args.build}/{arch}") + raise Exception + ociarchive = os.path.join(builddir, buildmeta['images'][args.artifact]['path']) + ocisha256sum = buildmeta['images'][args.artifact]['sha256'] + if not os.path.exists(ociarchive): + print(f"The file does not exist on disk: {ociarchive}") + raise Exception + if sha256sum_file(ociarchive) != ocisha256sum: + print(f"The file on disk {ociarchive} has an incorrect checksum") + raise Exception + images.append(f"oci-archive:{ociarchive}") + + # Create/Upload the manifest list to the container registry + with tempfile.NamedTemporaryFile() as digestfile: + create_and_push_container_manifest( + args.repo, args.tag, images, args.v2s2, digestfile.name) + digestfile.seek(0) + digest = digestfile.read().decode('utf-8').strip() + + # Update the meta.json in each build/arch metadata + for _, buildmeta in buildmetas.items(): + buildmeta[args.metajsonname] = {'image': f"{args.repo}@{digest}"} + buildmeta.write(artifact_name=args.metajsonname) def parse_args(): @@ -36,17 +89,32 @@ Examples: --repo quay.io/dustymabe/fedora-coreos --tag stable \\ --image oci-archive://builds/36.20220716.3.1/x86_64/fedora-coreos-37.20220725.91.0-ostree.x86_64.ociarchive \\ --image oci-archive://builds/36.20220716.3.1/aarch64/fedora-coreos-37.20220725.91.0-ostree.aarch64.ociarchive \\ - --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive""") + --image oci-archive://builds/36.20220716.3.1/s390x/fedora-coreos-37.20220725.91.0-ostree.s390x.ociarchive + + cosa push-container-manifest \\ + --repo quay.io/dustymabe/fedora-coreos --tag stable --artifact=ostree \\ + --metajsonname=base-oscontainer --build=latest --arch=x86_64 --arch=aarch64""") parser.add_argument("--repo", required=True, help="The registry repo to target for the manifest") parser.add_argument("--tag", required=True, help="The tag of the manifest to use") parser.add_argument("--authfile", help="A file to use for registry auth") parser.add_argument('--v2s2', action='store_true', help='Use old image manifest version2 schema 2 format') - parser.add_argument("--images", required=True, action='append', default=[], - help="""The images to add to the manifest. Can be specified multiple times like - --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456 - --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--image", dest='images', action='append', default=[], + help="""The images to add to the manifest. Can be specified multiple times like + --image docker://quay.io/dustymabe/coreos-assembler:s390x-686456 + --image oci-archive://path/to/cosa-aarch64-686456.ociarchive""") + group.add_argument("--artifact", help="""The artifact""") + + # A few more arguments that are used for `--artifact` + parser.add_argument("--build", default="latest", help="Build ID") + parser.add_argument("--arch", dest='arches', action='append', default=[], + help="""Limit the architectures to upload to the specificed set + (otherwise it defaults to all available for that build). Can be + specificed multiple times like: --arch x86_64 --arch aarch64""") + parser.add_argument("--metajsonname", + help="The name under which to store the container information in meta.json") return parser.parse_args() diff --git a/src/cosalib/container_manifest.py b/src/cosalib/container_manifest.py index 74aaaaf9f8..e407a00e67 100644 --- a/src/cosalib/container_manifest.py +++ b/src/cosalib/container_manifest.py @@ -25,11 +25,12 @@ def delete_local_container_manifest(repo, tag): runcmd(cmd) -def push_container_manifest(repo, tag, v2s2=False): +def push_container_manifest(repo, tag, digestfile, v2s2=False): ''' Push manifest to registry @param repo str registry repository @param tag str manifest tag + @param digestfile str write container digest to file @param v2s2 boolean use to force v2s2 format ''' cmd = ["podman", "manifest", "push", @@ -39,17 +40,20 @@ def push_container_manifest(repo, tag, v2s2=False): # to create a manifest with 2 different mediaType. It seems to be # a Quay issue. cmd.extend(["--remove-signatures", "-f", "v2s2"]) + if digestfile: + cmd.extend([f"--digestfile={digestfile}"]) runcmd(cmd) -def create_and_push_container_manifest(repo, tag, images, v2s2): +def create_and_push_container_manifest(repo, tag, images, v2s2, digestfile): ''' Do it all! Create, Push, Cleanup @param repo str registry repository @param tag str manifest tag @param images list of image specifications (including transport) @param v2s2 boolean use to force v2s2 format + @param digestfile str write container digest to file ''' create_local_container_manifest(repo, tag, images) - push_container_manifest(repo, tag, v2s2) + push_container_manifest(repo, tag, digestfile, v2s2) delete_local_container_manifest(repo, tag)