fetch-images command (#768)

Implementation of a command to fetch pre-built images from a remote registry, complementing the --push-images option already present on build-containers.

The two subcommands used together allow a stack to be deployed without needing to built its images, provided they have been already built and pushed to the specified container image registry.

This implementation simply picks the newest image with the right name and platform (matches against the platform Python is running on, so watch out for scenarios where Python is an x86 binary on M1 macs).

Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/768
Co-authored-by: David Boreham <[email protected]>
Co-committed-by: David Boreham <[email protected]>

stack_orchestrator/build/build_containers.py

@@ -25,13 +25,13 @@
 from decouple import config
 import subprocess
 import click
-import importlib.resources
 from pathlib import Path
 from stack_orchestrator.opts import opts
-from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, stack_is_external, error_exit, warn_exit
+from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit
 from stack_orchestrator.base import get_npm_registry_url
 from stack_orchestrator.build.build_types import BuildContext
 from stack_orchestrator.build.publish import publish_image
+from stack_orchestrator.build.build_util import get_containers_in_scope
 
 # TODO: find a place for this
 #    epilog="Config provided either in .env or settings.ini or env vars: CERC_REPO_BASE_DIR (defaults to ~/cerc)"
@@ -149,24 +149,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args, publish_imag
         if not image_registry:
             error_exit("--image-registry must be supplied with --publish-images")
 
-    # See: https://stackoverflow.com/a/20885799/1701505
-    from stack_orchestrator import data
-    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
-        all_containers = container_list_file.read().splitlines()
-
-    containers_in_scope = []
-    if stack:
-        stack_config = get_parsed_stack_config(stack)
-        if "containers" not in stack_config or stack_config["containers"] is None:
-            warn_exit(f"stack {stack} does not define any containers")
-        containers_in_scope = stack_config['containers']
-    else:
-        containers_in_scope = all_containers
-
-    if opts.o.verbose:
-        print(f'Containers: {containers_in_scope}')
-        if stack:
-            print(f"Stack: {stack}")
+    containers_in_scope = get_containers_in_scope(stack)
 
     container_build_env = make_container_build_env(dev_root_path,
                                                    container_build_dir,

stack_orchestrator/build/build_util.py

@@ -0,0 +1,43 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import importlib.resources
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import get_parsed_stack_config, warn_exit
+
+
+def get_containers_in_scope(stack: str):
+
+    # See: https://stackoverflow.com/a/20885799/1701505
+    from stack_orchestrator import data
+    with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
+        all_containers = container_list_file.read().splitlines()
+
+    containers_in_scope = []
+    if stack:
+        stack_config = get_parsed_stack_config(stack)
+        if "containers" not in stack_config or stack_config["containers"] is None:
+            warn_exit(f"stack {stack} does not define any containers")
+        containers_in_scope = stack_config['containers']
+    else:
+        containers_in_scope = all_containers
+
+    if opts.o.verbose:
+        print(f'Containers: {containers_in_scope}')
+        if stack:
+            print(f"Stack: {stack}")
+
+    return containers_in_scope

stack_orchestrator/build/fetch_containers.py

@@ -0,0 +1,180 @@
+# Copyright © 2024 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+from dataclasses import dataclass
+import json
+import platform
+from python_on_whales import DockerClient
+from python_on_whales.components.manifest.cli_wrapper import ManifestCLI, ManifestList
+from python_on_whales.utils import run
+import requests
+from typing import List
+
+from stack_orchestrator.opts import opts
+from stack_orchestrator.util import include_exclude_check, error_exit
+from stack_orchestrator.build.build_util import get_containers_in_scope
+
+# Experimental fetch-container command
+
+
+@dataclass
+class RegistryInfo:
+    registry: str
+    registry_username: str
+    registry_token: str
+
+
+# Extending this code to support the --verbose option, cnosider contributing upstream
+# https://github.com/gabrieldemarmiesse/python-on-whales/blob/master/python_on_whales/components/manifest/cli_wrapper.py#L129
+class ExtendedManifestCLI(ManifestCLI):
+    def inspect_verbose(self, x: str) -> ManifestList:
+        """Returns a Docker manifest list object."""
+        json_str = run(self.docker_cmd + ["manifest", "inspect", "--verbose", x])
+        return json.loads(json_str)
+
+
+def _local_tag_for(container: str):
+    return f"{container}:local"
+
+
+# See: https://docker-docs.uclv.cu/registry/spec/api/
+# Emulate this:
+# $ curl -u "my-username:my-token" -X GET "https://<container-registry-hostname>/v2/cerc-io/cerc/test-container/tags/list"
+# {"name":"cerc-io/cerc/test-container","tags":["202402232130","202402232208"]}
+def _get_tags_for_container(container: str, registry_info: RegistryInfo) -> List[str]:
+    # registry looks like: git.vdb.to/cerc-io
+    registry_parts = registry_info.registry.split("/")
+    url = f"https://{registry_parts[0]}/v2/{registry_parts[1]}/{container}/tags/list"
+    if opts.o.debug:
+        print(f"Fetching tags from: {url}")
+    response = requests.get(url, auth=(registry_info.registry_username, registry_info.registry_token))
+    if response.status_code == 200:
+        tag_info = response.json()
+        if opts.o.debug:
+            print(f"container tags list: {tag_info}")
+        tags_array = tag_info["tags"]
+        return tags_array
+    else:
+        error_exit(f"failed to fetch tags from image registry, status code: {response.status_code}")
+
+
+def _find_latest(candidate_tags: List[str]):
+    # Lex sort should give us the latest first
+    sorted_candidates = sorted(candidate_tags)
+    if opts.o.debug:
+        print(f"sorted candidates: {sorted_candidates}")
+    return sorted_candidates[0]
+
+
+def _filter_for_platform(container: str, 
+                         registry_info: RegistryInfo,
+                         tag_list: List[str]) -> List[str] :
+    filtered_tags = []
+    this_machine = platform.machine()
+    # Translate between Python platform names and docker
+    if this_machine == "x86_64":
+        this_machine = "amd64"
+    if opts.o.debug:
+        print(f"Python says the architecture is: {this_machine}")
+    docker = DockerClient()
+    docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+    for tag in tag_list:
+        remote_tag = f"{registry_info.registry}/{container}:{tag}"
+        manifest_cmd = ExtendedManifestCLI(docker.client_config)
+        manifest = manifest_cmd.inspect_verbose(remote_tag)
+        if opts.o.debug:
+            print(f"manifest: {manifest}")
+        image_architecture =  manifest["Descriptor"]["platform"]["architecture"]
+        if opts.o.debug:
+            print(f"image_architecture: {image_architecture}")
+        if this_machine == image_architecture:
+            filtered_tags.append(tag)
+    if opts.o.debug:
+        print(f"Tags filtered for platform: {filtered_tags}")
+    return filtered_tags
+
+
+def _get_latest_image(container: str, registry_info: RegistryInfo):
+    all_tags = _get_tags_for_container(container, registry_info)
+    tags_for_platform = _filter_for_platform(container, registry_info, all_tags)
+    latest_tag = _find_latest(tags_for_platform)
+    return f"{container}:{latest_tag}"
+
+
+def _fetch_image(tag: str, registry_info: RegistryInfo):
+    docker = DockerClient()
+    docker.login(registry_info.registry, registry_info.registry_username, registry_info.registry_token)
+    remote_tag = f"{registry_info.registry}/{tag}"
+    if opts.o.debug:
+        print(f"Attempting to pull this image: {remote_tag}")
+    docker.image.pull(remote_tag)
+
+
+def _exists_locally(container: str):
+    docker = DockerClient()
+    return docker.image.exists(_local_tag_for(container))
+
+
+def _add_local_tag(remote_tag: str, registry: str, local_tag: str):
+    docker = DockerClient()
+    docker.image.tag(f"{registry}/{remote_tag}", local_tag)
+
+
+@click.command()
+@click.option('--include', help="only fetch these containers")
+@click.option('--exclude', help="don\'t fetch these containers")
+@click.option("--force-local-overwrite", is_flag=True, default=False, help="Overwrite a locally built image, if present")
+@click.option("--image-registry", required=True, help="Specify the image registry to fetch from")
+@click.option("--registry-username", required=True, help="Specify the image registry username")
+@click.option("--registry-token", required=True, help="Specify the image registry access token")
+@click.pass_context
+def command(ctx, include, exclude, force_local_overwrite, image_registry, registry_username, registry_token):
+    '''EXPERIMENTAL: fetch the images for a stack from remote registry'''
+
+    registry_info = RegistryInfo(image_registry, registry_username, registry_token)
+    # Generate list of target containers
+    stack = ctx.obj.stack
+    containers_in_scope = get_containers_in_scope(stack)
+    for container in containers_in_scope:
+        local_tag = _local_tag_for(container)
+        if include_exclude_check(container, include, exclude):
+            if opts.o.debug:
+                print(f"Processing: {container}")
+            # For each container, attempt to find the latest of a set of
+            # images with the correct name and platform in the specified registry
+            image_to_fetch = _get_latest_image(container, registry_info)
+            if opts.o.debug:
+                print(f"Fetching: {image_to_fetch}")
+            _fetch_image(image_to_fetch, registry_info)
+            # Now check if the target container already exists exists locally already
+            if (_exists_locally(container)):
+                if not opts.o.quiet:
+                    print(f"Container image {container} already exists locally")
+                # if so, fail unless the user specified force-local-overwrite
+                if (force_local_overwrite):
+                    # In that case remove the existing :local tag
+                    if not opts.o.quiet:
+                        print(f"Warning: overwriting local tag from this image: {container} because "
+                              "--force-local-overwrite was specified")
+                else:
+                    if not opts.o.quiet:
+                        print(f"Skipping local tagging for this image: {container} because that would "
+                              "overwrite an existing :local tagged image, use --force-local-overwrite to do so.")
+            # Tag the fetched image with the :local tag
+            _add_local_tag(image_to_fetch, image_registry, local_tag)
+        else:
+            if opts.o.verbose:
+                print(f"Excluding: {container}")

stack_orchestrator/main.py

@@ -17,7 +17,7 @@
 
 from stack_orchestrator.command_types import CommandOptions
 from stack_orchestrator.repos import setup_repositories
-from stack_orchestrator.build import build_containers
+from stack_orchestrator.build import build_containers, fetch_containers
 from stack_orchestrator.build import build_npms
 from stack_orchestrator.build import build_webapp
 from stack_orchestrator.deploy.webapp import (run_webapp,
@@ -52,6 +52,7 @@ def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_err
 
 cli.add_command(setup_repositories.command, "setup-repositories")
 cli.add_command(build_containers.command, "build-containers")
+cli.add_command(fetch_containers.command, "fetch-containers")
 cli.add_command(build_npms.command, "build-npms")
 cli.add_command(build_webapp.command, "build-webapp")
 cli.add_command(run_webapp.command, "run-webapp")