From b27ecc40b24474f36783ee96fe85aa86a84a4659 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Tue, 18 Oct 2022 21:16:49 +1100 Subject: [PATCH 1/8] Initial sketch of exporting metadata about a packaged docker image [ci skip-rust] [ci skip-build-wheels] --- .../backend/docker/goals/package_image.py | 143 +++++++++++++++--- .../docker/goals/package_image_test.py | 9 +- .../backend/docker/goals/publish_test.py | 11 +- .../backend/helm/util_rules/post_renderer.py | 14 +- 4 files changed, 140 insertions(+), 37 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 2efe6cec524..4bd25644561 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -2,13 +2,14 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations +import json import logging import os import re -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import partial from itertools import chain -from typing import Iterator, cast +from typing import Iterator, Literal, cast # Re-exporting BuiltDockerImage here, as it has its natural home here, but has moved out to resolve # a dependency cycle from docker_build_context. @@ -114,15 +115,18 @@ def format_repository( repository_text, source=source, error_cls=DockerRepositoryNameError ).lower() - def format_names( + def format_image_ref_tags( self, repository: str, tags: tuple[str, ...], interpolation_context: InterpolationContext, - ) -> Iterator[str]: + ) -> Iterator[ImageRefTag]: for tag in tags: - yield ":".join( - s for s in [repository, self.format_tag(tag, interpolation_context)] if s + formatted = self.format_tag(tag, interpolation_context) + yield ImageRefTag( + template=tag, + formatted=formatted, + full_name=":".join(s for s in [repository, formatted] if s), ) def image_refs( @@ -131,7 +135,8 @@ def image_refs( registries: DockerRegistries, interpolation_context: InterpolationContext, additional_tags: tuple[str, ...] = (), - ) -> tuple[str, ...]: + ) -> Iterator[ImageRefRegistry]: + # FIXME: update this doc string """The image refs are the full image name, including any registry and version tag. In the Docker world, the term `tag` is used both for what we here prefer to call the image @@ -154,17 +159,30 @@ def image_refs( if not registries_options: # The image name is also valid as image ref without registry. repository = self.format_repository(default_repository, interpolation_context) - return tuple(self.format_names(repository, image_tags, interpolation_context)) - - return tuple( - "/".join([registry.address, image_name]) - for registry in registries_options - for image_name in self.format_names( - self.format_repository(default_repository, interpolation_context, registry), - image_tags + registry.extra_image_tags, - interpolation_context, + yield ImageRefRegistry( + registry=None, + repository=repository, + tags=tuple( + self.format_image_ref_tags(repository, image_tags, interpolation_context) + ), + ) + return + + for registry in registries_options: + repository = self.format_repository(default_repository, interpolation_context, registry) + full_repository = "/".join([registry.address, repository]) + + yield ImageRefRegistry( + registry=registry, + repository=repository, + tags=tuple( + self.format_image_ref_tags( + full_repository, + image_tags + registry.extra_image_tags, + interpolation_context, + ) + ), ) - ) def get_context_root(self, default_context_root: str) -> str: """Examines `default_context_root` and `self.context_root.value` and translates that to a @@ -187,6 +205,80 @@ def get_context_root(self, default_context_root: str) -> str: return os.path.normpath(context_root) +@dataclass(frozen=True) +class ImageRefRegistry: + registry: DockerRegistryOptions | None + repository: str + tags: tuple[ImageRefTag, ...] + + +@dataclass(frozen=True) +class ImageRefTag: + template: str + formatted: str + full_name: str + + +@dataclass(frozen=True) +class DockerInfoV1: + """The format of the .docker-info.json file.""" + + version: Literal[1] + image_id: str + # FIXME: it'd be good to include the digest here (i.e. to allow 'docker run + # registry/repository@digest'), but that is only known after pushing to a V2 registry + + # registry alias or address -> registry (using a dict rather than just a list[registry] for more + # convenient look-ups when multiple values exist) + registries: dict[str, DockerInfoV1Registry] + + @staticmethod + def from_image_refs(image_refs: tuple[ImageRefRegistry, ...], image_id: str) -> DockerInfoV1: + def registry_key(reg: DockerRegistryOptions | None) -> str: + if reg is None: + return "" + if reg.alias: + return f"@{reg.alias}" + return reg.address + + return DockerInfoV1( + version=1, + image_id=image_id, + registries={ + registry_key(r.registry): DockerInfoV1Registry( + alias=r.registry.alias if r.registry and r.registry.alias else None, + address=r.registry.address if r.registry else None, + repository=r.repository, + tags={ + t.template: DockerInfoV1ImageTag( + template=t.template, tag=t.formatted, name=t.full_name + ) + for t in r.tags + }, + ) + for r in image_refs + }, + ) + + +@dataclass(frozen=True) +class DockerInfoV1Registry: + # set if registry was specified as `@something` + alias: str | None + address: str | None + repository: str + # tag template -> Tag + tags: dict[str, DockerInfoV1ImageTag] + + +@dataclass(frozen=True) +class DockerInfoV1ImageTag: + template: str + tag: str + # for convenience, include the concatenated registry/repository:tag name (using this tag) + name: str + + def get_build_options( context: DockerBuildContext, field_set: DockerFieldSet, @@ -263,12 +355,15 @@ async def build_docker_image( if image_tags_request_cls.is_applicable(wrapped_target.target) ) - tags = field_set.image_refs( - default_repository=options.default_repository, - registries=options.registries(), - interpolation_context=context.interpolation_context, - additional_tags=tuple(chain.from_iterable(additional_image_tags)), + image_refs = tuple( + field_set.image_refs( + default_repository=options.default_repository, + registries=options.registries(), + interpolation_context=context.interpolation_context, + additional_tags=tuple(chain.from_iterable(additional_image_tags)), + ) ) + tags = tuple(tag.full_name for registry in image_refs for tag in registry.tags) # Mix the upstream image ids into the env to ensure that Pants invalidates this # image-building process correctly when an upstream image changes, even though the @@ -330,6 +425,10 @@ async def build_docker_image( else: logger.debug(docker_build_output_msg) + # FIXME: how to write this to an appropriate file for inclusion in dist/ and use as a dependency + metadata = DockerInfoV1.from_image_refs(image_refs, image_id=image_id) + logger.info(json.dumps(asdict(metadata))) + return BuiltPackage( result.output_digest, (BuiltDockerImage.create(image_id, tags),), diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index 9d9bfd5c81e..c3ea2d65c7e 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -390,11 +390,12 @@ def test_dynamic_image_version(rule_runner: RuleRunner) -> None: def assert_tags(name: str, *expect_tags: str) -> None: tgt = rule_runner.get_target(Address("docker/test", target_name=name)) fs = DockerFieldSet.create(tgt) - tags = fs.image_refs( + image_refs = fs.image_refs( "image", DockerRegistries.from_dict({}), interpolation_context, ) + tags = tuple(t.full_name for r in image_refs for t in r.tags) assert expect_tags == tags rule_runner.write_files( @@ -1138,10 +1139,10 @@ def test_image_ref_formatting(test: ImageRefTest) -> None: registries = DockerRegistries.from_dict(test.registries) interpolation_context = InterpolationContext.from_dict({}) with test.expect_error or no_exception(): - assert ( - field_set.image_refs(test.default_repository, registries, interpolation_context) - == test.expect_refs + image_refs = field_set.image_refs( + test.default_repository, registries, interpolation_context ) + assert tuple(t.full_name for r in image_refs for t in r.tags) == test.expect_refs def test_docker_image_tags_from_plugin_hook(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/docker/goals/publish_test.py b/src/python/pants/backend/docker/goals/publish_test.py index 2f4501929ca..6e3cb00f9d1 100644 --- a/src/python/pants/backend/docker/goals/publish_test.py +++ b/src/python/pants/backend/docker/goals/publish_test.py @@ -56,17 +56,18 @@ def rule_runner() -> RuleRunner: def build(tgt: DockerImageTarget, options: DockerOptions): fs = DockerFieldSet.create(tgt) + image_refs = fs.image_refs( + options.default_repository, + options.registries(), + InterpolationContext(), + ) return ( BuiltPackage( EMPTY_DIGEST, ( BuiltDockerImage.create( "sha256:made-up", - fs.image_refs( - options.default_repository, - options.registries(), - InterpolationContext(), - ), + tuple(t.full_name for r in image_refs for t in r.tags), ), ), ), diff --git a/src/python/pants/backend/helm/util_rules/post_renderer.py b/src/python/pants/backend/helm/util_rules/post_renderer.py index d7b848d54be..59fb9026213 100644 --- a/src/python/pants/backend/helm/util_rules/post_renderer.py +++ b/src/python/pants/backend/helm/util_rules/post_renderer.py @@ -119,12 +119,14 @@ async def resolve_docker_image_ref(address: Address, context: DockerBuildContext # Choose first non-latest image reference found, or fallback to 'latest'. found_ref: str | None = None fallback_ref: str | None = None - for ref in image_refs: - if ref.endswith(":latest"): - fallback_ref = ref - else: - found_ref = ref - break + for registry in image_refs: + for tag in registry.tags: + ref = tag.full_name + if ref.endswith(":latest"): + fallback_ref = ref + else: + found_ref = ref + break resolved_ref = found_ref or fallback_ref if resolved_ref: From 03b348a2047ffe8895cb6317faa072d33678f519 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 21 Oct 2022 09:53:06 +1100 Subject: [PATCH 2/8] Write to metadata to file [ci skip-rust] --- .../pants/backend/docker/goals/package_image.py | 13 +++++++++---- .../pants/backend/docker/goals/publish_test.py | 1 + src/python/pants/backend/docker/package_types.py | 6 ++++-- src/python/pants/backend/docker/target_types.py | 2 ++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 4bd25644561..36207d446e9 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -35,9 +35,10 @@ DockerBuildContextRequest, ) from pants.backend.docker.utils import format_rename_suggestion -from pants.core.goals.package import BuiltPackage, PackageFieldSet +from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet from pants.core.goals.run import RunFieldSet from pants.engine.addresses import Address +from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import FallibleProcessResult, Process, ProcessExecutionFailure from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import Target, WrappedTarget, WrappedTargetRequest @@ -75,6 +76,7 @@ class DockerFieldSet(PackageFieldSet, RunFieldSet): source: DockerImageSourceField tags: DockerImageTagsField target_stage: DockerImageTargetStageField + output_path: OutputPathField def format_tag(self, tag: str, interpolation_context: InterpolationContext) -> str: source = InterpolationContext.TextSource( @@ -425,13 +427,16 @@ async def build_docker_image( else: logger.debug(docker_build_output_msg) + metadata_filename = field_set.output_path.value_or_default(file_ending="docker-info.json") # FIXME: how to write this to an appropriate file for inclusion in dist/ and use as a dependency metadata = DockerInfoV1.from_image_refs(image_refs, image_id=image_id) - logger.info(json.dumps(asdict(metadata))) + metadata_bytes = json.dumps(asdict(metadata)).encode() + + digest = await Get(Digest, CreateDigest([FileContent(metadata_filename, metadata_bytes)])) return BuiltPackage( - result.output_digest, - (BuiltDockerImage.create(image_id, tags),), + digest, + (BuiltDockerImage.create(image_id, tags, metadata_filename),), ) diff --git a/src/python/pants/backend/docker/goals/publish_test.py b/src/python/pants/backend/docker/goals/publish_test.py index 6e3cb00f9d1..941ffb3f638 100644 --- a/src/python/pants/backend/docker/goals/publish_test.py +++ b/src/python/pants/backend/docker/goals/publish_test.py @@ -68,6 +68,7 @@ def build(tgt: DockerImageTarget, options: DockerOptions): BuiltDockerImage.create( "sha256:made-up", tuple(t.full_name for r in image_refs for t in r.tags), + "made-up.json", ), ), ), diff --git a/src/python/pants/backend/docker/package_types.py b/src/python/pants/backend/docker/package_types.py index e978f9d4900..bcce33b6654 100644 --- a/src/python/pants/backend/docker/package_types.py +++ b/src/python/pants/backend/docker/package_types.py @@ -18,12 +18,14 @@ class BuiltDockerImage(BuiltPackageArtifact): tags: tuple[str, ...] = () @classmethod - def create(cls, image_id: str, tags: tuple[str, ...]) -> BuiltDockerImage: + def create( + cls, image_id: str, tags: tuple[str, ...], metadata_filename: str + ) -> BuiltDockerImage: tags_string = tags[0] if len(tags) == 1 else f"\n{bullet_list(tags)}" return cls( image_id=image_id, tags=tags, - relpath=None, + relpath=metadata_filename, extra_log_lines=( f"Built docker {pluralize(len(tags), 'image', False)}: {tags_string}", f"Docker image ID: {image_id}", diff --git a/src/python/pants/backend/docker/target_types.py b/src/python/pants/backend/docker/target_types.py index 28653c0958d..539a5b94cdf 100644 --- a/src/python/pants/backend/docker/target_types.py +++ b/src/python/pants/backend/docker/target_types.py @@ -13,6 +13,7 @@ from pants.backend.docker.registries import ALL_DEFAULT_REGISTRIES from pants.base.build_environment import get_buildroot +from pants.core.goals.package import OutputPathField from pants.core.goals.run import RestartableField from pants.engine.addresses import Address from pants.engine.collection import Collection @@ -403,6 +404,7 @@ class DockerImageTarget(Target): DockerImageTargetStageField, DockerImageBuildPullOptionField, DockerImageBuildSquashOptionField, + OutputPathField, RestartableField, ) help = softwrap( From aad97565a1a9eaeddf17cdb6c56827aa377b0bbc Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 21 Oct 2022 11:09:29 +1100 Subject: [PATCH 3/8] Write tests, resolve FIXMEs [ci skip-rust] --- .../backend/docker/goals/package_image.py | 24 +- .../docker/goals/package_image_test.py | 463 +++++++++++++++++- .../util_rules/docker_build_context_test.py | 2 +- 3 files changed, 465 insertions(+), 24 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 36207d446e9..b232a3b0bd9 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -138,15 +138,15 @@ def image_refs( interpolation_context: InterpolationContext, additional_tags: tuple[str, ...] = (), ) -> Iterator[ImageRefRegistry]: - # FIXME: update this doc string - """The image refs are the full image name, including any registry and version tag. + """The per-registry image refs: the full image name including any registry and version tag. In the Docker world, the term `tag` is used both for what we here prefer to call the image `ref`, as well as for the image version, or tag, that is at the end of the image name separated with a colon. By introducing the image `ref` we can retain the use of `tag` for the version part of the image name. - Returns all image refs to apply to the Docker image, on the form: + Returns all image refs to apply to the Docker image, accessible within the `full_name` + attribute of each element of the `tags` field: [/][:] @@ -154,7 +154,8 @@ def image_refs( the `default_repository` from configuration or the `repository` field on the target `docker_image`. - This method will always return a non-empty tuple. + This method will always return at least one `ImageRefRegistry`, and there will be at least + one tag. """ image_tags = (self.tags.value or ()) + additional_tags registries_options = tuple(registries.get(*(self.registries.value or []))) @@ -227,7 +228,7 @@ class DockerInfoV1: version: Literal[1] image_id: str - # FIXME: it'd be good to include the digest here (i.e. to allow 'docker run + # It'd be good to include the digest here (e.g. to allow 'docker run # registry/repository@digest'), but that is only known after pushing to a V2 registry # registry alias or address -> registry (using a dict rather than just a list[registry] for more @@ -235,7 +236,7 @@ class DockerInfoV1: registries: dict[str, DockerInfoV1Registry] @staticmethod - def from_image_refs(image_refs: tuple[ImageRefRegistry, ...], image_id: str) -> DockerInfoV1: + def serialize(image_refs: tuple[ImageRefRegistry, ...], image_id: str) -> bytes: def registry_key(reg: DockerRegistryOptions | None) -> str: if reg is None: return "" @@ -243,7 +244,7 @@ def registry_key(reg: DockerRegistryOptions | None) -> str: return f"@{reg.alias}" return reg.address - return DockerInfoV1( + info = DockerInfoV1( version=1, image_id=image_id, registries={ @@ -262,6 +263,8 @@ def registry_key(reg: DockerRegistryOptions | None) -> str: }, ) + return json.dumps(asdict(info)).encode() + @dataclass(frozen=True) class DockerInfoV1Registry: @@ -428,11 +431,8 @@ async def build_docker_image( logger.debug(docker_build_output_msg) metadata_filename = field_set.output_path.value_or_default(file_ending="docker-info.json") - # FIXME: how to write this to an appropriate file for inclusion in dist/ and use as a dependency - metadata = DockerInfoV1.from_image_refs(image_refs, image_id=image_id) - metadata_bytes = json.dumps(asdict(metadata)).encode() - - digest = await Get(Digest, CreateDigest([FileContent(metadata_filename, metadata_bytes)])) + metadata = DockerInfoV1.serialize(image_refs, image_id=image_id) + digest = await Get(Digest, CreateDigest([FileContent(metadata_filename, metadata)])) return BuiltPackage( digest, diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index c3ea2d65c7e..ac7e41fa3d8 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import logging import os.path from collections import namedtuple @@ -15,12 +16,15 @@ DockerBuildTargetStageError, DockerFieldSet, DockerImageTagValueError, + DockerInfoV1, DockerRepositoryNameError, + ImageRefRegistry, + ImageRefTag, build_docker_image, parse_image_id_from_docker_build_output, rules, ) -from pants.backend.docker.registries import DockerRegistries +from pants.backend.docker.registries import DockerRegistries, DockerRegistryOptions from pants.backend.docker.subsystems.docker_options import DockerOptions from pants.backend.docker.subsystems.dockerfile_parser import DockerfileInfo from pants.backend.docker.target_types import ( @@ -44,7 +48,15 @@ ) from pants.backend.docker.util_rules.docker_build_env import rules as build_env_rules from pants.engine.addresses import Address -from pants.engine.fs import EMPTY_DIGEST, EMPTY_FILE_DIGEST, EMPTY_SNAPSHOT, Snapshot +from pants.engine.fs import ( + EMPTY_DIGEST, + EMPTY_FILE_DIGEST, + EMPTY_SNAPSHOT, + CreateDigest, + Digest, + FileContent, + Snapshot, +) from pants.engine.platform import Platform from pants.engine.process import ( FallibleProcessResult, @@ -93,8 +105,11 @@ def assert_build( build_context_snapshot: Snapshot = EMPTY_SNAPSHOT, version_tags: tuple[str, ...] = (), plugin_tags: tuple[str, ...] = (), + expected_registries_metadata: None | dict = None, ) -> None: tgt = rule_runner.get_target(address) + metadata_file_path: list[str] = [] + metadata_file_contents: list[bytes] = [] def build_context_mock(request: DockerBuildContextRequest) -> DockerBuildContext: return DockerBuildContext.create( @@ -128,6 +143,13 @@ def run_process_mock(process: Process) -> FallibleProcessResult: metadata=ProcessResultMetadata(0, "ran_locally", 0), ) + def mock_get_info_file(request: CreateDigest) -> Digest: + assert len(request) == 1 + assert isinstance(request[0], FileContent) + metadata_file_path.append(request[0].path) + metadata_file_contents.append(request[0].content) + return EMPTY_DIGEST + if options: opts = options or {} opts.setdefault("registries", {}) @@ -180,12 +202,27 @@ def run_process_mock(process: Process) -> FallibleProcessResult: input_types=(Process,), mock=run_process_mock, ), + MockGet( + output_type=Digest, + input_types=(CreateDigest,), + mock=mock_get_info_file, + ), ], ) assert result.digest == EMPTY_DIGEST assert len(result.artifacts) == 1 - assert result.artifacts[0].relpath is None + assert len(metadata_file_path) == len(metadata_file_contents) == 1 + assert result.artifacts[0].relpath == metadata_file_path[0] + + metadata = json.loads(metadata_file_contents[0]) + # basic checks that we can always do + assert metadata["version"] == 1 + assert metadata["image_id"] == "" + assert isinstance(metadata["registries"], dict) + # detailed checks, if the test opts in + if expected_registries_metadata is not None: + assert metadata["registries"] == expected_registries_metadata for log_line in extra_log_lines: assert log_line in result.artifacts[0].extra_log_lines @@ -234,11 +271,27 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="test1"), "Built docker image: test/test1:1.2.3", + expected_registries_metadata={ + "": dict( + alias=None, + address=None, + repository="test/test1", + tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="test/test1:1.2.3")}, + ) + }, ) assert_build( rule_runner, Address("docker/test", target_name="test2"), "Built docker image: test2:1.2.3", + expected_registries_metadata={ + "": dict( + alias=None, + address=None, + repository="test2", + tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="test2:1.2.3")}, + ) + }, ) assert_build( rule_runner, @@ -260,6 +313,20 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: " * test/test5:alpha-1" ), options=dict(default_repository="{directory}/{name}"), + expected_registries_metadata={ + "": dict( + alias=None, + address=None, + repository="test/test5", + tags={ + "latest": dict(template="latest", tag="latest", name="test/test5:latest"), + "alpha-1.0": dict( + template="alpha-1.0", tag="alpha-1.0", name="test/test5:alpha-1.0" + ), + "alpha-1": dict(template="alpha-1", tag="alpha-1", name="test/test5:alpha-1"), + }, + ) + }, ) err1 = ( @@ -310,6 +377,18 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr1"), "Built docker image: myregistry1domain:port/addr1:1.2.3", options=options, + expected_registries_metadata={ + "@reg1": dict( + alias="reg1", + address="myregistry1domain:port", + repository="addr1", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry1domain:port/addr1:1.2.3" + ) + }, + ) + }, ) assert_build( rule_runner, @@ -322,12 +401,36 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr3"), "Built docker image: myregistry3domain:port/addr3:1.2.3", options=options, + expected_registries_metadata={ + "myregistry3domain:port": dict( + alias=None, + address="myregistry3domain:port", + repository="addr3", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry3domain:port/addr3:1.2.3" + ) + }, + ) + }, ) assert_build( rule_runner, Address("docker/test", target_name="alias1"), "Built docker image: myregistry1domain:port/alias1:1.2.3", options=options, + expected_registries_metadata={ + "@reg1": dict( + alias="reg1", + address="myregistry1domain:port", + repository="alias1", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry1domain:port/alias1:1.2.3" + ) + }, + ) + }, ) assert_build( rule_runner, @@ -346,12 +449,32 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="unreg"), "Built docker image: unreg:1.2.3", options=options, + expected_registries_metadata={ + "": dict( + alias=None, + address=None, + repository="unreg", + tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="unreg:1.2.3")}, + ) + }, ) assert_build( rule_runner, Address("docker/test", target_name="def"), "Built docker image: myregistry2domain:port/def:1.2.3", options=options, + expected_registries_metadata={ + "@reg2": dict( + alias="reg2", + address="myregistry2domain:port", + repository="def", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry2domain:port/def:1.2.3" + ) + }, + ) + }, ) assert_build( rule_runner, @@ -362,6 +485,28 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * myregistry1domain:port/multi:1.2.3" ), options=options, + expected_registries_metadata={ + "@reg1": dict( + alias="reg1", + address="myregistry1domain:port", + repository="multi", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry1domain:port/multi:1.2.3" + ) + }, + ), + "@reg2": dict( + alias="reg2", + address="myregistry2domain:port", + repository="multi", + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", name="myregistry2domain:port/multi:1.2.3" + ) + }, + ), + }, ) assert_build( rule_runner, @@ -373,6 +518,29 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * extra/extra_tags:latest" ), options=options, + expected_registries_metadata={ + "@reg1": dict( + alias="reg1", + address="myregistry1domain:port", + repository="extra_tags", + tags={ + "1.2.3": dict( + template="1.2.3", + tag="1.2.3", + name="myregistry1domain:port/extra_tags:1.2.3", + ) + }, + ), + "@extra": dict( + alias="extra", + address="extra", + repository="extra_tags", + tags={ + "1.2.3": dict(template="1.2.3", tag="1.2.3", name="extra/extra_tags:1.2.3"), + "latest": dict(template="latest", tag="latest", name="extra/extra_tags:latest"), + }, + ), + }, ) @@ -602,6 +770,18 @@ def test_docker_image_version_from_build_arg(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="ver1"), "Built docker image: ver1:1.2.3", + expected_registries_metadata={ + "": dict( + alias=None, + address=None, + repository="ver1", + tags={ + "{build_args.VERSION}": dict( + template="{build_args.VERSION}", tag="1.2.3", name="ver1:1.2.3" + ) + }, + ) + }, ) @@ -949,6 +1129,14 @@ def check_docker_proc(process: Process): options=options, process_assertions=check_docker_proc, version_tags=("build latest", "dev latest", "prod latest"), + expected_registries_metadata={ + "": dict( + address=None, + alias=None, + repository="image", + tags={"latest": dict(template="latest", tag="latest", name="image:latest")}, + ) + }, ) @@ -1094,17 +1282,114 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std @pytest.mark.parametrize( "test", [ - ImageRefTest(docker_image=dict(name="lowercase"), expect_refs=("lowercase:latest",)), - ImageRefTest(docker_image=dict(name="CamelCase"), expect_refs=("camelcase:latest",)), - ImageRefTest(docker_image=dict(image_tags=["CamelCase"]), expect_refs=("image:CamelCase",)), + ImageRefTest( + docker_image=dict(name="lowercase"), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="lowercase", + tags=( + ImageRefTag( + template="latest", formatted="latest", full_name="lowercase:latest" + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(name="CamelCase"), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="camelcase", + tags=( + ImageRefTag( + template="latest", formatted="latest", full_name="camelcase:latest" + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(image_tags=["CamelCase"]), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="image", + tags=( + ImageRefTag( + template="CamelCase", formatted="CamelCase", full_name="image:CamelCase" + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(image_tags=["{val1}", "prefix-{val2}"]), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="image", + tags=( + ImageRefTag( + template="{val1}", + formatted="first-value", + full_name="image:first-value", + ), + ImageRefTag( + template="prefix-{val2}", + formatted="prefix-second-value", + full_name="image:prefix-second-value", + ), + ), + ), + ), + ), ImageRefTest( docker_image=dict(registries=["REG1.example.net"]), - expect_refs=("REG1.example.net/image:latest",), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="REG1.example.net"), + repository="image", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + full_name="REG1.example.net/image:latest", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict(registries=["docker.io", "@private"], repository="our-the/pkg"), registries=dict(private={"address": "our.registry", "repository": "the/pkg"}), - expect_refs=("docker.io/our-the/pkg:latest", "our.registry/the/pkg:latest"), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="our-the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + full_name="docker.io/our-the/pkg:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", address="our.registry", repository="the/pkg" + ), + repository="the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + full_name="our.registry/the/pkg:latest", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict( @@ -1114,7 +1399,62 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std registries=dict( private={"address": "our.registry", "repository": "{target_repository}/the/pkg"} ), - expect_refs=("docker.io/test/image:latest", "our.registry/test/image/the/pkg:latest"), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="test/image", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + full_name="docker.io/test/image:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + repository="{target_repository}/the/pkg", + ), + repository="test/image/the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + full_name="our.registry/test/image/the/pkg:latest", + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(registries=["@private"], image_tags=["prefix-{val1}"]), + registries=dict( + private={"address": "our.registry", "extra_image_tags": ["{val2}-suffix"]} + ), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + extra_image_tags=("{val2}-suffix",), + ), + repository="image", + tags=( + ImageRefTag( + template="prefix-{val1}", + formatted="prefix-first-value", + full_name="our.registry/image:prefix-first-value", + ), + ImageRefTag( + template="{val2}-suffix", + formatted="second-value-suffix", + full_name="our.registry/image:second-value-suffix", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict(repository="{default_repository}/a"), @@ -1137,12 +1477,14 @@ def test_image_ref_formatting(test: ImageRefTest) -> None: tgt = DockerImageTarget(test.docker_image, address) field_set = DockerFieldSet.create(tgt) registries = DockerRegistries.from_dict(test.registries) - interpolation_context = InterpolationContext.from_dict({}) + interpolation_context = InterpolationContext.from_dict( + {"val1": "first-value", "val2": "second-value"} + ) with test.expect_error or no_exception(): image_refs = field_set.image_refs( test.default_repository, registries, interpolation_context ) - assert tuple(t.full_name for r in image_refs for t in r.tags) == test.expect_refs + assert tuple(image_refs) == test.expect_refs def test_docker_image_tags_from_plugin_hook(rule_runner: RuleRunner) -> None: @@ -1168,3 +1510,102 @@ def check_docker_proc(process: Process): process_assertions=check_docker_proc, plugin_tags=("1.2.3",), ) + + +def test_docker_info_serialize() -> None: + image_id = "abc123" + # image refs with unique strings (i.e. not actual templates/names etc.), to make sure they're + # ending up in the right place in the JSON + image_refs = ( + ImageRefRegistry( + registry=None, + repository="repo", + tags=( + ImageRefTag( + template="repo tag1 template", + formatted="repo tag1 formatted", + full_name="repo tag1 full name", + ), + ImageRefTag( + template="repo tag2 template", + formatted="repo tag2 formatted", + full_name="repo tag2 full name", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions(address="address"), + repository="address repo", + tags=( + ImageRefTag( + template="address tag template", + formatted="address tag formatted", + full_name="address tag full name", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + address="alias address", alias="alias", repository="alias registry repo" + ), + repository="alias repo", + tags=( + ImageRefTag( + template="alias tag template", + formatted="alias tag formatted", + full_name="alias tag full name", + ), + ), + ), + ) + + expected = dict( + version=1, + image_id=image_id, + registries={ + "": dict( + alias=None, + address=None, + repository="repo", + tags={ + "repo tag1 template": dict( + template="repo tag1 template", + tag="repo tag1 formatted", + name="repo tag1 full name", + ), + "repo tag2 template": dict( + template="repo tag2 template", + tag="repo tag2 formatted", + name="repo tag2 full name", + ), + }, + ), + "address": dict( + alias=None, + address="address", + repository="address repo", + tags={ + "address tag template": dict( + template="address tag template", + tag="address tag formatted", + name="address tag full name", + ) + }, + ), + "@alias": dict( + alias="alias", + address="alias address", + repository="alias repo", + tags={ + "alias tag template": dict( + template="alias tag template", + tag="alias tag formatted", + name="alias tag full name", + ) + }, + ), + }, + ) + + result = DockerInfoV1.serialize(image_refs, image_id) + assert json.loads(result) == expected diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py index ded48c2280c..ffca1bcc247 100644 --- a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py +++ b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py @@ -241,7 +241,7 @@ def test_from_image_build_arg_dependency(rule_runner: RuleRunner) -> None: assert_build_context( rule_runner, Address("src/downstream", target_name="image"), - expected_files=["src/downstream/Dockerfile"], + expected_files=["src/downstream/Dockerfile", "src.upstream/image.docker-info.json"], build_upstream_images=True, expected_interpolation_context={ "tags": { From 980fb68ec383f1625495f5b93d9dd03b5791ce59 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 21 Oct 2022 11:34:16 +1100 Subject: [PATCH 4/8] Use typing_extensions.Literal, for Python 3.7 [ci skip-rust] --- src/python/pants/backend/docker/goals/package_image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index b232a3b0bd9..5e531f8facc 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -9,7 +9,9 @@ from dataclasses import asdict, dataclass from functools import partial from itertools import chain -from typing import Iterator, Literal, cast +from typing import Iterator, cast + +from typing_extensions import Literal # Re-exporting BuiltDockerImage here, as it has its natural home here, but has moved out to resolve # a dependency cycle from docker_build_context. From 1ed0d63f7bd4d7fe5485ad31ac0829f9ec04dc31 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Thu, 10 Nov 2022 13:24:56 +1100 Subject: [PATCH 5/8] bring in a concept of alias tags --- .../backend/docker/goals/package_image.py | 67 +++--- .../docker/goals/package_image_test.py | 194 +++++++++++++++--- 2 files changed, 205 insertions(+), 56 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 8a3b47c0e86..9b9d67c5119 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -123,6 +123,7 @@ def format_image_ref_tags( repository: str, tags: tuple[str, ...], interpolation_context: InterpolationContext, + uses_local_alias, ) -> Iterator[ImageRefTag]: for tag in tags: formatted = self.format_tag(tag, interpolation_context) @@ -130,6 +131,7 @@ def format_image_ref_tags( template=tag, formatted=formatted, full_name=":".join(s for s in [repository, formatted] if s), + uses_local_alias=uses_local_alias, ) def image_refs( @@ -158,22 +160,6 @@ def image_refs( This method will always return at least one `ImageRefRegistry`, and there will be at least one tag. """ - return tuple( - self._image_refs_generator( - default_repository=default_repository, - registries=registries, - interpolation_context=interpolation_context, - additional_tags=additional_tags, - ) - ) - - def _image_refs_generator( - self, - default_repository: str, - registries: DockerRegistries, - interpolation_context: InterpolationContext, - additional_tags: tuple[str, ...] = (), - ) -> Iterator[str]: image_tags = (self.tags.value or ()) + additional_tags registries_options = tuple(registries.get(*(self.registries.value or []))) if not registries_options: @@ -183,24 +169,41 @@ def _image_refs_generator( registry=None, repository=repository, tags=tuple( - self.format_image_ref_tags(repository, image_tags, interpolation_context) + self.format_image_ref_tags( + repository, image_tags, interpolation_context, uses_local_alias=False + ) ), ) return for registry in registries_options: repository = self.format_repository(default_repository, interpolation_context, registry) - full_repository = "/".join([registry.address, repository]) + address_repository = "/".join([registry.address, repository]) + if registry.use_local_alias and registry.alias: + alias_repository = "/".join([registry.alias, repository]) + else: + alias_repository = None yield ImageRefRegistry( registry=registry, repository=repository, - tags=tuple( - self.format_image_ref_tags( - full_repository, + tags=( + *self.format_image_ref_tags( + address_repository, image_tags + registry.extra_image_tags, interpolation_context, - ) + uses_local_alias=False, + ), + *( + self.format_image_ref_tags( + alias_repository, + image_tags + registry.extra_image_tags, + interpolation_context, + uses_local_alias=True, + ) + if alias_repository + else [] + ), ), ) @@ -237,6 +240,7 @@ class ImageRefTag: template: str formatted: str full_name: str + uses_local_alias: bool @dataclass(frozen=True) @@ -261,6 +265,11 @@ def registry_key(reg: DockerRegistryOptions | None) -> str: return f"@{reg.alias}" return reg.address + def tag_key(tag: ImageRefTag) -> str: + if tag.uses_local_alias: + return f"{tag.template}:local alias" + return tag.template + info = DockerInfoV1( version=1, image_id=image_id, @@ -270,10 +279,15 @@ def registry_key(reg: DockerRegistryOptions | None) -> str: address=r.registry.address if r.registry else None, repository=r.repository, tags={ - t.template: DockerInfoV1ImageTag( - template=t.template, tag=t.formatted, name=t.full_name - ) - for t in r.tags + **{ + tag_key(t): DockerInfoV1ImageTag( + template=t.template, + tag=t.formatted, + uses_local_alias=t.uses_local_alias, + name=t.full_name, + ) + for t in r.tags + }, }, ) for r in image_refs @@ -297,6 +311,7 @@ class DockerInfoV1Registry: class DockerInfoV1ImageTag: template: str tag: str + uses_local_alias: bool # for convenience, include the concatenated registry/repository:tag name (using this tag) name: str diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index 77d9e20473a..2870466116d 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -276,7 +276,14 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: alias=None, address=None, repository="test/test1", - tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="test/test1:1.2.3")}, + tags={ + "1.2.3": dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="test/test1:1.2.3", + ) + }, ) }, ) @@ -289,7 +296,11 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: alias=None, address=None, repository="test2", - tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="test2:1.2.3")}, + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", uses_local_alias=False, name="test2:1.2.3" + ) + }, ) }, ) @@ -319,11 +330,24 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: address=None, repository="test/test5", tags={ - "latest": dict(template="latest", tag="latest", name="test/test5:latest"), + "latest": dict( + template="latest", + tag="latest", + uses_local_alias=False, + name="test/test5:latest", + ), "alpha-1.0": dict( - template="alpha-1.0", tag="alpha-1.0", name="test/test5:alpha-1.0" + template="alpha-1.0", + tag="alpha-1.0", + uses_local_alias=False, + name="test/test5:alpha-1.0", + ), + "alpha-1": dict( + template="alpha-1", + tag="alpha-1", + uses_local_alias=False, + name="test/test5:alpha-1", ), - "alpha-1": dict(template="alpha-1", tag="alpha-1", name="test/test5:alpha-1"), }, ) }, @@ -384,7 +408,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="addr1", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry1domain:port/addr1:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/addr1:1.2.3", ) }, ) @@ -408,7 +435,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="addr3", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry3domain:port/addr3:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry3domain:port/addr3:1.2.3", ) }, ) @@ -426,7 +456,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="alias1", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry1domain:port/alias1:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/alias1:1.2.3", ) }, ) @@ -454,7 +487,11 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: alias=None, address=None, repository="unreg", - tags={"1.2.3": dict(template="1.2.3", tag="1.2.3", name="unreg:1.2.3")}, + tags={ + "1.2.3": dict( + template="1.2.3", tag="1.2.3", uses_local_alias=False, name="unreg:1.2.3" + ) + }, ) }, ) @@ -470,7 +507,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="def", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry2domain:port/def:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry2domain:port/def:1.2.3", ) }, ) @@ -492,7 +532,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="multi", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry1domain:port/multi:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/multi:1.2.3", ) }, ), @@ -502,7 +545,10 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: repository="multi", tags={ "1.2.3": dict( - template="1.2.3", tag="1.2.3", name="myregistry2domain:port/multi:1.2.3" + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry2domain:port/multi:1.2.3", ) }, ), @@ -527,6 +573,7 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: "1.2.3": dict( template="1.2.3", tag="1.2.3", + uses_local_alias=False, name="myregistry1domain:port/extra_tags:1.2.3", ) }, @@ -536,8 +583,18 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: address="extra", repository="extra_tags", tags={ - "1.2.3": dict(template="1.2.3", tag="1.2.3", name="extra/extra_tags:1.2.3"), - "latest": dict(template="latest", tag="latest", name="extra/extra_tags:latest"), + "1.2.3": dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="extra/extra_tags:1.2.3", + ), + "latest": dict( + template="latest", + tag="latest", + uses_local_alias=False, + name="extra/extra_tags:latest", + ), }, ), }, @@ -777,7 +834,10 @@ def test_docker_image_version_from_build_arg(rule_runner: RuleRunner) -> None: repository="ver1", tags={ "{build_args.VERSION}": dict( - template="{build_args.VERSION}", tag="1.2.3", name="ver1:1.2.3" + template="{build_args.VERSION}", + tag="1.2.3", + uses_local_alias=False, + name="ver1:1.2.3", ) }, ) @@ -1134,7 +1194,11 @@ def check_docker_proc(process: Process): address=None, alias=None, repository="image", - tags={"latest": dict(template="latest", tag="latest", name="image:latest")}, + tags={ + "latest": dict( + template="latest", tag="latest", uses_local_alias=False, name="image:latest" + ) + }, ) }, ) @@ -1290,7 +1354,10 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std repository="lowercase", tags=( ImageRefTag( - template="latest", formatted="latest", full_name="lowercase:latest" + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="lowercase:latest", ), ), ), @@ -1304,7 +1371,10 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std repository="camelcase", tags=( ImageRefTag( - template="latest", formatted="latest", full_name="camelcase:latest" + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="camelcase:latest", ), ), ), @@ -1318,7 +1388,10 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std repository="image", tags=( ImageRefTag( - template="CamelCase", formatted="CamelCase", full_name="image:CamelCase" + template="CamelCase", + formatted="CamelCase", + uses_local_alias=False, + full_name="image:CamelCase", ), ), ), @@ -1334,11 +1407,13 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="{val1}", formatted="first-value", + uses_local_alias=False, full_name="image:first-value", ), ImageRefTag( template="prefix-{val2}", formatted="prefix-second-value", + uses_local_alias=False, full_name="image:prefix-second-value", ), ), @@ -1355,6 +1430,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="latest", formatted="latest", + uses_local_alias=False, full_name="REG1.example.net/image:latest", ), ), @@ -1372,6 +1448,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="latest", formatted="latest", + uses_local_alias=False, full_name="docker.io/our-the/pkg:latest", ), ), @@ -1385,6 +1462,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="latest", formatted="latest", + uses_local_alias=False, full_name="our.registry/the/pkg:latest", ), ), @@ -1407,6 +1485,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="latest", formatted="latest", + uses_local_alias=False, full_name="docker.io/test/image:latest", ), ), @@ -1422,6 +1501,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="latest", formatted="latest", + uses_local_alias=False, full_name="our.registry/test/image/the/pkg:latest", ), ), @@ -1445,11 +1525,13 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std ImageRefTag( template="prefix-{val1}", formatted="prefix-first-value", + uses_local_alias=False, full_name="our.registry/image:prefix-first-value", ), ImageRefTag( template="{val2}-suffix", formatted="second-value-suffix", + uses_local_alias=False, full_name="our.registry/image:second-value-suffix", ), ), @@ -1481,9 +1563,41 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std } ), expect_refs=( - "docker.io/our-the/pkg:latest", - "private/the/pkg:latest", - "our.registry/the/pkg:latest", + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="our-the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="docker.io/our-the/pkg:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + repository="the/pkg", + use_local_alias=True, + ), + repository="the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="our.registry/the/pkg:latest", + ), + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=True, + full_name="private/the/pkg:latest", + ), + ), + ), ), ), ], @@ -1540,11 +1654,13 @@ def test_docker_info_serialize() -> None: ImageRefTag( template="repo tag1 template", formatted="repo tag1 formatted", + uses_local_alias=False, full_name="repo tag1 full name", ), ImageRefTag( template="repo tag2 template", formatted="repo tag2 formatted", + uses_local_alias=False, full_name="repo tag2 full name", ), ), @@ -1556,6 +1672,7 @@ def test_docker_info_serialize() -> None: ImageRefTag( template="address tag template", formatted="address tag formatted", + uses_local_alias=False, full_name="address tag full name", ), ), @@ -1567,9 +1684,16 @@ def test_docker_info_serialize() -> None: repository="alias repo", tags=( ImageRefTag( - template="alias tag template", - formatted="alias tag formatted", - full_name="alias tag full name", + template="alias tag (address) template", + formatted="alias tag (address) formatted", + uses_local_alias=False, + full_name="alias tag (address) full name", + ), + ImageRefTag( + template="alias tag (local alias) template", + formatted="alias tag (local alias) formatted", + uses_local_alias=True, + full_name="alias tag (local alias) full name", ), ), ), @@ -1587,11 +1711,13 @@ def test_docker_info_serialize() -> None: "repo tag1 template": dict( template="repo tag1 template", tag="repo tag1 formatted", + uses_local_alias=False, name="repo tag1 full name", ), "repo tag2 template": dict( template="repo tag2 template", tag="repo tag2 formatted", + uses_local_alias=False, name="repo tag2 full name", ), }, @@ -1604,6 +1730,7 @@ def test_docker_info_serialize() -> None: "address tag template": dict( template="address tag template", tag="address tag formatted", + uses_local_alias=False, name="address tag full name", ) }, @@ -1613,11 +1740,18 @@ def test_docker_info_serialize() -> None: address="alias address", repository="alias repo", tags={ - "alias tag template": dict( - template="alias tag template", - tag="alias tag formatted", - name="alias tag full name", - ) + "alias tag (address) template": dict( + template="alias tag (address) template", + tag="alias tag (address) formatted", + uses_local_alias=False, + name="alias tag (address) full name", + ), + "alias tag (local alias) template:local alias": dict( + template="alias tag (local alias) template", + tag="alias tag (local alias) formatted", + uses_local_alias=True, + name="alias tag (local alias) full name", + ), }, ), }, From 8566c73fc99bee14cc112edaefac3ec30260ae0a Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Thu, 10 Nov 2022 15:22:14 +1100 Subject: [PATCH 6/8] tweak docs --- .../pants/backend/docker/goals/package_image.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 9b9d67c5119..aef2c16b595 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -141,15 +141,18 @@ def image_refs( interpolation_context: InterpolationContext, additional_tags: tuple[str, ...] = (), ) -> Iterator[ImageRefRegistry]: - """The per-registry image refs: the full image name including any registry and version tag. + """The per-registry image refs: each returned element is a collection of the tags applied to + the image in a single registry. In the Docker world, the term `tag` is used both for what we here prefer to call the image `ref`, as well as for the image version, or tag, that is at the end of the image name separated with a colon. By introducing the image `ref` we can retain the use of `tag` for the version part of the image name. - Returns all image refs to apply to the Docker image, accessible within the `full_name` - attribute of each element of the `tags` field: + This function returns all image refs to apply to the Docker image, grouped by + registry. Within each registry, the `tags` attribute contains a metadata about each tag in + the context of that registry, and the `full_name` attribute of each `ImageRefTag` provides + the image ref, of the following form: [/][:] @@ -245,7 +248,7 @@ class ImageRefTag: @dataclass(frozen=True) class DockerInfoV1: - """The format of the .docker-info.json file.""" + """The format of the `$target_name.docker-info.json` file.""" version: Literal[1] image_id: str From fe7507021a5469c4be8ee1a35dcb9c61306a37d4 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 11 Nov 2022 10:04:28 +1100 Subject: [PATCH 7/8] switch to using lists, not dicts --- .../backend/docker/goals/package_image.py | 50 ++-- .../docker/goals/package_image_test.py | 236 +++++++++--------- 2 files changed, 135 insertions(+), 151 deletions(-) diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index aef2c16b595..2c0830d5d7a 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -255,46 +255,35 @@ class DockerInfoV1: # It'd be good to include the digest here (e.g. to allow 'docker run # registry/repository@digest'), but that is only known after pushing to a V2 registry - # registry alias or address -> registry (using a dict rather than just a list[registry] for more - # convenient look-ups when multiple values exist) - registries: dict[str, DockerInfoV1Registry] + registries: list[DockerInfoV1Registry] @staticmethod def serialize(image_refs: tuple[ImageRefRegistry, ...], image_id: str) -> bytes: - def registry_key(reg: DockerRegistryOptions | None) -> str: - if reg is None: - return "" - if reg.alias: - return f"@{reg.alias}" - return reg.address - - def tag_key(tag: ImageRefTag) -> str: - if tag.uses_local_alias: - return f"{tag.template}:local alias" - return tag.template + # make sure these are in a consistent order (the exact order doesn't matter + # so much), no matter how they were configured + sorted_refs = sorted(image_refs, key=lambda r: r.registry.address if r.registry else "") info = DockerInfoV1( version=1, image_id=image_id, - registries={ - registry_key(r.registry): DockerInfoV1Registry( + registries=[ + DockerInfoV1Registry( alias=r.registry.alias if r.registry and r.registry.alias else None, address=r.registry.address if r.registry else None, repository=r.repository, - tags={ - **{ - tag_key(t): DockerInfoV1ImageTag( - template=t.template, - tag=t.formatted, - uses_local_alias=t.uses_local_alias, - name=t.full_name, - ) - for t in r.tags - }, - }, + tags=[ + DockerInfoV1ImageTag( + template=t.template, + tag=t.formatted, + uses_local_alias=t.uses_local_alias, + name=t.full_name, + ) + # consistent order, as above + for t in sorted(r.tags, key=lambda t: t.full_name) + ], ) - for r in image_refs - }, + for r in sorted_refs + ], ) return json.dumps(asdict(info)).encode() @@ -306,8 +295,7 @@ class DockerInfoV1Registry: alias: str | None address: str | None repository: str - # tag template -> Tag - tags: dict[str, DockerInfoV1ImageTag] + tags: list[DockerInfoV1ImageTag] @dataclass(frozen=True) diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index 2870466116d..903bcacf346 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -105,7 +105,7 @@ def assert_build( build_context_snapshot: Snapshot = EMPTY_SNAPSHOT, version_tags: tuple[str, ...] = (), plugin_tags: tuple[str, ...] = (), - expected_registries_metadata: None | dict = None, + expected_registries_metadata: None | list = None, ) -> None: tgt = rule_runner.get_target(address) metadata_file_path: list[str] = [] @@ -219,7 +219,7 @@ def mock_get_info_file(request: CreateDigest) -> Digest: # basic checks that we can always do assert metadata["version"] == 1 assert metadata["image_id"] == "" - assert isinstance(metadata["registries"], dict) + assert isinstance(metadata["registries"], list) # detailed checks, if the test opts in if expected_registries_metadata is not None: assert metadata["registries"] == expected_registries_metadata @@ -271,38 +271,36 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="test1"), "Built docker image: test/test1:1.2.3", - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( alias=None, address=None, repository="test/test1", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="test/test1:1.2.3", ) - }, + ], ) - }, + ], ) assert_build( rule_runner, Address("docker/test", target_name="test2"), "Built docker image: test2:1.2.3", - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( alias=None, address=None, repository="test2", - tags={ - "1.2.3": dict( - template="1.2.3", tag="1.2.3", uses_local_alias=False, name="test2:1.2.3" - ) - }, + tags=[ + dict(template="1.2.3", tag="1.2.3", uses_local_alias=False, name="test2:1.2.3") + ], ) - }, + ], ) assert_build( rule_runner, @@ -324,33 +322,33 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: " * test/test5:alpha-1" ), options=dict(default_repository="{directory}/{name}"), - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( alias=None, address=None, repository="test/test5", - tags={ - "latest": dict( - template="latest", - tag="latest", + tags=[ + dict( + template="alpha-1", + tag="alpha-1", uses_local_alias=False, - name="test/test5:latest", + name="test/test5:alpha-1", ), - "alpha-1.0": dict( + dict( template="alpha-1.0", tag="alpha-1.0", uses_local_alias=False, name="test/test5:alpha-1.0", ), - "alpha-1": dict( - template="alpha-1", - tag="alpha-1", + dict( + template="latest", + tag="latest", uses_local_alias=False, - name="test/test5:alpha-1", + name="test/test5:latest", ), - }, + ], ) - }, + ], ) err1 = ( @@ -401,21 +399,21 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr1"), "Built docker image: myregistry1domain:port/addr1:1.2.3", options=options, - expected_registries_metadata={ - "@reg1": dict( + expected_registries_metadata=[ + dict( alias="reg1", address="myregistry1domain:port", repository="addr1", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry1domain:port/addr1:1.2.3", ) - }, + ], ) - }, + ], ) assert_build( rule_runner, @@ -428,42 +426,42 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr3"), "Built docker image: myregistry3domain:port/addr3:1.2.3", options=options, - expected_registries_metadata={ - "myregistry3domain:port": dict( + expected_registries_metadata=[ + dict( alias=None, address="myregistry3domain:port", repository="addr3", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry3domain:port/addr3:1.2.3", ) - }, + ], ) - }, + ], ) assert_build( rule_runner, Address("docker/test", target_name="alias1"), "Built docker image: myregistry1domain:port/alias1:1.2.3", options=options, - expected_registries_metadata={ - "@reg1": dict( + expected_registries_metadata=[ + dict( alias="reg1", address="myregistry1domain:port", repository="alias1", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry1domain:port/alias1:1.2.3", ) - }, + ], ) - }, + ], ) assert_build( rule_runner, @@ -482,39 +480,37 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="unreg"), "Built docker image: unreg:1.2.3", options=options, - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( alias=None, address=None, repository="unreg", - tags={ - "1.2.3": dict( - template="1.2.3", tag="1.2.3", uses_local_alias=False, name="unreg:1.2.3" - ) - }, + tags=[ + dict(template="1.2.3", tag="1.2.3", uses_local_alias=False, name="unreg:1.2.3") + ], ) - }, + ], ) assert_build( rule_runner, Address("docker/test", target_name="def"), "Built docker image: myregistry2domain:port/def:1.2.3", options=options, - expected_registries_metadata={ - "@reg2": dict( + expected_registries_metadata=[ + dict( alias="reg2", address="myregistry2domain:port", repository="def", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry2domain:port/def:1.2.3", ) - }, + ], ) - }, + ], ) assert_build( rule_runner, @@ -525,34 +521,34 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * myregistry1domain:port/multi:1.2.3" ), options=options, - expected_registries_metadata={ - "@reg1": dict( + expected_registries_metadata=[ + dict( alias="reg1", address="myregistry1domain:port", repository="multi", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry1domain:port/multi:1.2.3", ) - }, + ], ), - "@reg2": dict( + dict( alias="reg2", address="myregistry2domain:port", repository="multi", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="myregistry2domain:port/multi:1.2.3", ) - }, + ], ), - }, + ], ) assert_build( rule_runner, @@ -564,40 +560,40 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * extra/extra_tags:latest" ), options=options, - expected_registries_metadata={ - "@reg1": dict( - alias="reg1", - address="myregistry1domain:port", - repository="extra_tags", - tags={ - "1.2.3": dict( - template="1.2.3", - tag="1.2.3", - uses_local_alias=False, - name="myregistry1domain:port/extra_tags:1.2.3", - ) - }, - ), - "@extra": dict( + expected_registries_metadata=[ + dict( alias="extra", address="extra", repository="extra_tags", - tags={ - "1.2.3": dict( + tags=[ + dict( template="1.2.3", tag="1.2.3", uses_local_alias=False, name="extra/extra_tags:1.2.3", ), - "latest": dict( + dict( template="latest", tag="latest", uses_local_alias=False, name="extra/extra_tags:latest", ), - }, + ], ), - }, + dict( + alias="reg1", + address="myregistry1domain:port", + repository="extra_tags", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/extra_tags:1.2.3", + ) + ], + ), + ], ) @@ -827,21 +823,21 @@ def test_docker_image_version_from_build_arg(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="ver1"), "Built docker image: ver1:1.2.3", - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( alias=None, address=None, repository="ver1", - tags={ - "{build_args.VERSION}": dict( + tags=[ + dict( template="{build_args.VERSION}", tag="1.2.3", uses_local_alias=False, name="ver1:1.2.3", ) - }, + ], ) - }, + ], ) @@ -1189,18 +1185,18 @@ def check_docker_proc(process: Process): options=options, process_assertions=check_docker_proc, version_tags=("build latest", "dev latest", "prod latest"), - expected_registries_metadata={ - "": dict( + expected_registries_metadata=[ + dict( address=None, alias=None, repository="image", - tags={ - "latest": dict( + tags=[ + dict( template="latest", tag="latest", uses_local_alias=False, name="image:latest" ) - }, + ], ) - }, + ], ) @@ -1702,59 +1698,59 @@ def test_docker_info_serialize() -> None: expected = dict( version=1, image_id=image_id, - registries={ - "": dict( + registries=[ + dict( alias=None, address=None, repository="repo", - tags={ - "repo tag1 template": dict( + tags=[ + dict( template="repo tag1 template", tag="repo tag1 formatted", uses_local_alias=False, name="repo tag1 full name", ), - "repo tag2 template": dict( + dict( template="repo tag2 template", tag="repo tag2 formatted", uses_local_alias=False, name="repo tag2 full name", ), - }, + ], ), - "address": dict( + dict( alias=None, address="address", repository="address repo", - tags={ - "address tag template": dict( + tags=[ + dict( template="address tag template", tag="address tag formatted", uses_local_alias=False, name="address tag full name", ) - }, + ], ), - "@alias": dict( + dict( alias="alias", address="alias address", repository="alias repo", - tags={ - "alias tag (address) template": dict( + tags=[ + dict( template="alias tag (address) template", tag="alias tag (address) formatted", uses_local_alias=False, name="alias tag (address) full name", ), - "alias tag (local alias) template:local alias": dict( + dict( template="alias tag (local alias) template", tag="alias tag (local alias) formatted", uses_local_alias=True, name="alias tag (local alias) full name", ), - }, + ], ), - }, + ], ) result = DockerInfoV1.serialize(image_refs, image_id) From 19cb523c66a104e57aa9aa2d0bcd44fe7ce575cc Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Mon, 14 Nov 2022 11:46:10 +1100 Subject: [PATCH 8/8] Add some documentation --- docs/markdown/Docker/tagging-docker-images.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/markdown/Docker/tagging-docker-images.md b/docs/markdown/Docker/tagging-docker-images.md index b09b55d2254..8cce5e97677 100644 --- a/docs/markdown/Docker/tagging-docker-images.md +++ b/docs/markdown/Docker/tagging-docker-images.md @@ -312,3 +312,34 @@ See [Setting a repository name](doc:tagging-docker-images#setting-a-repository-n > 📘 The `{pants.hash}` stability guarantee > > The calculated hash value _may_ change between stable versions of Pants for the otherwise same input sources. + +Retrieving the tags of an packaged image +---------------------------------------- + +When a docker image is packaged, metadata about the resulting image is output to a JSON file artefact. This includes the image ID, as well as the full names that the image was tagged with. This file is written in the same manner as outputs of other packageable targets and available for later steps (for example, a test with `runtime_package_dependencies` including the docker image target) or in `dist/` after `./pants package`. By default, this is available at `path.to.target/target_name.docker-info.json`. + +The structure of this JSON file is: + +``` javascript +{ + "version": 1, // always 1, until a breaking change is made to this schema + "image_id": "sha256:..." // the local Image ID of the computed image + "registries": [ // info about each registry used for this image + { + "alias": "name", // set if the registry is configured in pants.toml, or null if not + "address": "reg.invalid", // the address of the registry itself + "repository": "the/repo", // the repository used for the image within the registry + "tags": [ + { + "template": "tag-{...}", // the tag before substituting any placeholders + "tag": "tag-some-value", // the fully-substituted tag, actually used to tag the image + "uses_local_alias": false, // if this tag used the local alias for the registry or not + "name": "reg.invalid/the/repo:tag-some-value", // the full name that the image was tagged with + } + ] + } + ] +} +``` + +This JSON file can be used to retrieve the exact name to place into cloud deploy templates or to use for running locally, especially when using tags with placeholders.