diff --git a/docs/markdown/Docker/tagging-docker-images.md b/docs/markdown/Docker/tagging-docker-images.md index b09b55d2254..8cce5e97677 100644 --- a/docs/markdown/Docker/tagging-docker-images.md +++ b/docs/markdown/Docker/tagging-docker-images.md @@ -312,3 +312,34 @@ See [Setting a repository name](doc:tagging-docker-images#setting-a-repository-n > 📘 The `{pants.hash}` stability guarantee > > The calculated hash value _may_ change between stable versions of Pants for the otherwise same input sources. + +Retrieving the tags of an packaged image +---------------------------------------- + +When a docker image is packaged, metadata about the resulting image is output to a JSON file artefact. This includes the image ID, as well as the full names that the image was tagged with. This file is written in the same manner as outputs of other packageable targets and available for later steps (for example, a test with `runtime_package_dependencies` including the docker image target) or in `dist/` after `./pants package`. By default, this is available at `path.to.target/target_name.docker-info.json`. + +The structure of this JSON file is: + +``` javascript +{ + "version": 1, // always 1, until a breaking change is made to this schema + "image_id": "sha256:..." // the local Image ID of the computed image + "registries": [ // info about each registry used for this image + { + "alias": "name", // set if the registry is configured in pants.toml, or null if not + "address": "reg.invalid", // the address of the registry itself + "repository": "the/repo", // the repository used for the image within the registry + "tags": [ + { + "template": "tag-{...}", // the tag before substituting any placeholders + "tag": "tag-some-value", // the fully-substituted tag, actually used to tag the image + "uses_local_alias": false, // if this tag used the local alias for the registry or not + "name": "reg.invalid/the/repo:tag-some-value", // the full name that the image was tagged with + } + ] + } + ] +} +``` + +This JSON file can be used to retrieve the exact name to place into cloud deploy templates or to use for running locally, especially when using tags with placeholders. diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index a9d75d7f8af..2c0830d5d7a 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -2,14 +2,17 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations +import json import logging import os import re -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import partial from itertools import chain from typing import Iterator, cast +from typing_extensions import Literal + # Re-exporting BuiltDockerImage here, as it has its natural home here, but has moved out to resolve # a dependency cycle from docker_build_context. from pants.backend.docker.package_types import BuiltDockerImage as BuiltDockerImage @@ -34,8 +37,9 @@ DockerBuildContextRequest, ) from pants.backend.docker.utils import format_rename_suggestion -from pants.core.goals.package import BuiltPackage, PackageFieldSet +from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet from pants.engine.addresses import Address +from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import FallibleProcessResult, Process, ProcessExecutionFailure from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import Target, WrappedTarget, WrappedTargetRequest @@ -73,6 +77,7 @@ class DockerPackageFieldSet(PackageFieldSet): source: DockerImageSourceField tags: DockerImageTagsField target_stage: DockerImageTargetStageField + output_path: OutputPathField def format_tag(self, tag: str, interpolation_context: InterpolationContext) -> str: source = InterpolationContext.TextSource( @@ -113,15 +118,20 @@ def format_repository( repository_text, source=source, error_cls=DockerRepositoryNameError ).lower() - def format_names( + def format_image_ref_tags( self, repository: str, tags: tuple[str, ...], interpolation_context: InterpolationContext, - ) -> Iterator[str]: + uses_local_alias, + ) -> Iterator[ImageRefTag]: for tag in tags: - yield ":".join( - s for s in [repository, self.format_tag(tag, interpolation_context)] if s + formatted = self.format_tag(tag, interpolation_context) + yield ImageRefTag( + template=tag, + formatted=formatted, + full_name=":".join(s for s in [repository, formatted] if s), + uses_local_alias=uses_local_alias, ) def image_refs( @@ -130,15 +140,19 @@ def image_refs( registries: DockerRegistries, interpolation_context: InterpolationContext, additional_tags: tuple[str, ...] = (), - ) -> tuple[str, ...]: - """The image refs are the full image name, including any registry and version tag. + ) -> Iterator[ImageRefRegistry]: + """The per-registry image refs: each returned element is a collection of the tags applied to + the image in a single registry. In the Docker world, the term `tag` is used both for what we here prefer to call the image `ref`, as well as for the image version, or tag, that is at the end of the image name separated with a colon. By introducing the image `ref` we can retain the use of `tag` for the version part of the image name. - Returns all image refs to apply to the Docker image, on the form: + This function returns all image refs to apply to the Docker image, grouped by + registry. Within each registry, the `tags` attribute contains a metadata about each tag in + the context of that registry, and the `full_name` attribute of each `ImageRefTag` provides + the image ref, of the following form: [/][:] @@ -146,42 +160,55 @@ def image_refs( the `default_repository` from configuration or the `repository` field on the target `docker_image`. - This method will always return a non-empty tuple. + This method will always return at least one `ImageRefRegistry`, and there will be at least + one tag. """ - return tuple( - self._image_refs_generator( - default_repository=default_repository, - registries=registries, - interpolation_context=interpolation_context, - additional_tags=additional_tags, - ) - ) - - def _image_refs_generator( - self, - default_repository: str, - registries: DockerRegistries, - interpolation_context: InterpolationContext, - additional_tags: tuple[str, ...] = (), - ) -> Iterator[str]: image_tags = (self.tags.value or ()) + additional_tags registries_options = tuple(registries.get(*(self.registries.value or []))) if not registries_options: # The image name is also valid as image ref without registry. repository = self.format_repository(default_repository, interpolation_context) - yield from self.format_names(repository, image_tags, interpolation_context) + yield ImageRefRegistry( + registry=None, + repository=repository, + tags=tuple( + self.format_image_ref_tags( + repository, image_tags, interpolation_context, uses_local_alias=False + ) + ), + ) return for registry in registries_options: - image_names = self.format_names( - self.format_repository(default_repository, interpolation_context, registry), - image_tags + registry.extra_image_tags, - interpolation_context, + repository = self.format_repository(default_repository, interpolation_context, registry) + address_repository = "/".join([registry.address, repository]) + if registry.use_local_alias and registry.alias: + alias_repository = "/".join([registry.alias, repository]) + else: + alias_repository = None + + yield ImageRefRegistry( + registry=registry, + repository=repository, + tags=( + *self.format_image_ref_tags( + address_repository, + image_tags + registry.extra_image_tags, + interpolation_context, + uses_local_alias=False, + ), + *( + self.format_image_ref_tags( + alias_repository, + image_tags + registry.extra_image_tags, + interpolation_context, + uses_local_alias=True, + ) + if alias_repository + else [] + ), + ), ) - for image_name in image_names: - if registry.use_local_alias and registry.alias: - yield "/".join([registry.alias, image_name]) - yield "/".join([registry.address, image_name]) def get_context_root(self, default_context_root: str) -> str: """Examines `default_context_root` and `self.context_root.value` and translates that to a @@ -204,6 +231,82 @@ def get_context_root(self, default_context_root: str) -> str: return os.path.normpath(context_root) +@dataclass(frozen=True) +class ImageRefRegistry: + registry: DockerRegistryOptions | None + repository: str + tags: tuple[ImageRefTag, ...] + + +@dataclass(frozen=True) +class ImageRefTag: + template: str + formatted: str + full_name: str + uses_local_alias: bool + + +@dataclass(frozen=True) +class DockerInfoV1: + """The format of the `$target_name.docker-info.json` file.""" + + version: Literal[1] + image_id: str + # It'd be good to include the digest here (e.g. to allow 'docker run + # registry/repository@digest'), but that is only known after pushing to a V2 registry + + registries: list[DockerInfoV1Registry] + + @staticmethod + def serialize(image_refs: tuple[ImageRefRegistry, ...], image_id: str) -> bytes: + # make sure these are in a consistent order (the exact order doesn't matter + # so much), no matter how they were configured + sorted_refs = sorted(image_refs, key=lambda r: r.registry.address if r.registry else "") + + info = DockerInfoV1( + version=1, + image_id=image_id, + registries=[ + DockerInfoV1Registry( + alias=r.registry.alias if r.registry and r.registry.alias else None, + address=r.registry.address if r.registry else None, + repository=r.repository, + tags=[ + DockerInfoV1ImageTag( + template=t.template, + tag=t.formatted, + uses_local_alias=t.uses_local_alias, + name=t.full_name, + ) + # consistent order, as above + for t in sorted(r.tags, key=lambda t: t.full_name) + ], + ) + for r in sorted_refs + ], + ) + + return json.dumps(asdict(info)).encode() + + +@dataclass(frozen=True) +class DockerInfoV1Registry: + # set if registry was specified as `@something` + alias: str | None + address: str | None + repository: str + tags: list[DockerInfoV1ImageTag] + + +@dataclass(frozen=True) +class DockerInfoV1ImageTag: + template: str + tag: str + uses_local_alias: bool + # for convenience, include the concatenated registry/repository:tag name (using this tag) + name: str + + def get_build_options( context: DockerBuildContext, field_set: DockerPackageFieldSet, @@ -280,12 +383,15 @@ async def build_docker_image( if image_tags_request_cls.is_applicable(wrapped_target.target) ) - tags = field_set.image_refs( - default_repository=options.default_repository, - registries=options.registries(), - interpolation_context=context.interpolation_context, - additional_tags=tuple(chain.from_iterable(additional_image_tags)), + image_refs = tuple( + field_set.image_refs( + default_repository=options.default_repository, + registries=options.registries(), + interpolation_context=context.interpolation_context, + additional_tags=tuple(chain.from_iterable(additional_image_tags)), + ) ) + tags = tuple(tag.full_name for registry in image_refs for tag in registry.tags) # Mix the upstream image ids into the env to ensure that Pants invalidates this # image-building process correctly when an upstream image changes, even though the @@ -347,9 +453,13 @@ async def build_docker_image( else: logger.debug(docker_build_output_msg) + metadata_filename = field_set.output_path.value_or_default(file_ending="docker-info.json") + metadata = DockerInfoV1.serialize(image_refs, image_id=image_id) + digest = await Get(Digest, CreateDigest([FileContent(metadata_filename, metadata)])) + return BuiltPackage( - result.output_digest, - (BuiltDockerImage.create(image_id, tags),), + digest, + (BuiltDockerImage.create(image_id, tags, metadata_filename),), ) diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index d74bc3e1ad2..903bcacf346 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import logging import os.path from collections import namedtuple @@ -14,13 +15,16 @@ from pants.backend.docker.goals.package_image import ( DockerBuildTargetStageError, DockerImageTagValueError, + DockerInfoV1, DockerPackageFieldSet, DockerRepositoryNameError, + ImageRefRegistry, + ImageRefTag, build_docker_image, parse_image_id_from_docker_build_output, rules, ) -from pants.backend.docker.registries import DockerRegistries +from pants.backend.docker.registries import DockerRegistries, DockerRegistryOptions from pants.backend.docker.subsystems.docker_options import DockerOptions from pants.backend.docker.subsystems.dockerfile_parser import DockerfileInfo from pants.backend.docker.target_types import ( @@ -44,7 +48,15 @@ ) from pants.backend.docker.util_rules.docker_build_env import rules as build_env_rules from pants.engine.addresses import Address -from pants.engine.fs import EMPTY_DIGEST, EMPTY_FILE_DIGEST, EMPTY_SNAPSHOT, Snapshot +from pants.engine.fs import ( + EMPTY_DIGEST, + EMPTY_FILE_DIGEST, + EMPTY_SNAPSHOT, + CreateDigest, + Digest, + FileContent, + Snapshot, +) from pants.engine.platform import Platform from pants.engine.process import ( FallibleProcessResult, @@ -93,8 +105,11 @@ def assert_build( build_context_snapshot: Snapshot = EMPTY_SNAPSHOT, version_tags: tuple[str, ...] = (), plugin_tags: tuple[str, ...] = (), + expected_registries_metadata: None | list = None, ) -> None: tgt = rule_runner.get_target(address) + metadata_file_path: list[str] = [] + metadata_file_contents: list[bytes] = [] def build_context_mock(request: DockerBuildContextRequest) -> DockerBuildContext: return DockerBuildContext.create( @@ -128,6 +143,13 @@ def run_process_mock(process: Process) -> FallibleProcessResult: metadata=ProcessResultMetadata(0, "ran_locally", 0), ) + def mock_get_info_file(request: CreateDigest) -> Digest: + assert len(request) == 1 + assert isinstance(request[0], FileContent) + metadata_file_path.append(request[0].path) + metadata_file_contents.append(request[0].content) + return EMPTY_DIGEST + if options: opts = options or {} opts.setdefault("registries", {}) @@ -180,12 +202,27 @@ def run_process_mock(process: Process) -> FallibleProcessResult: input_types=(Process,), mock=run_process_mock, ), + MockGet( + output_type=Digest, + input_types=(CreateDigest,), + mock=mock_get_info_file, + ), ], ) assert result.digest == EMPTY_DIGEST assert len(result.artifacts) == 1 - assert result.artifacts[0].relpath is None + assert len(metadata_file_path) == len(metadata_file_contents) == 1 + assert result.artifacts[0].relpath == metadata_file_path[0] + + metadata = json.loads(metadata_file_contents[0]) + # basic checks that we can always do + assert metadata["version"] == 1 + assert metadata["image_id"] == "" + assert isinstance(metadata["registries"], list) + # detailed checks, if the test opts in + if expected_registries_metadata is not None: + assert metadata["registries"] == expected_registries_metadata for log_line in extra_log_lines: assert log_line in result.artifacts[0].extra_log_lines @@ -234,11 +271,36 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="test1"), "Built docker image: test/test1:1.2.3", + expected_registries_metadata=[ + dict( + alias=None, + address=None, + repository="test/test1", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="test/test1:1.2.3", + ) + ], + ) + ], ) assert_build( rule_runner, Address("docker/test", target_name="test2"), "Built docker image: test2:1.2.3", + expected_registries_metadata=[ + dict( + alias=None, + address=None, + repository="test2", + tags=[ + dict(template="1.2.3", tag="1.2.3", uses_local_alias=False, name="test2:1.2.3") + ], + ) + ], ) assert_build( rule_runner, @@ -260,6 +322,33 @@ def test_build_docker_image(rule_runner: RuleRunner) -> None: " * test/test5:alpha-1" ), options=dict(default_repository="{directory}/{name}"), + expected_registries_metadata=[ + dict( + alias=None, + address=None, + repository="test/test5", + tags=[ + dict( + template="alpha-1", + tag="alpha-1", + uses_local_alias=False, + name="test/test5:alpha-1", + ), + dict( + template="alpha-1.0", + tag="alpha-1.0", + uses_local_alias=False, + name="test/test5:alpha-1.0", + ), + dict( + template="latest", + tag="latest", + uses_local_alias=False, + name="test/test5:latest", + ), + ], + ) + ], ) err1 = ( @@ -310,6 +399,21 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr1"), "Built docker image: myregistry1domain:port/addr1:1.2.3", options=options, + expected_registries_metadata=[ + dict( + alias="reg1", + address="myregistry1domain:port", + repository="addr1", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/addr1:1.2.3", + ) + ], + ) + ], ) assert_build( rule_runner, @@ -322,12 +426,42 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="addr3"), "Built docker image: myregistry3domain:port/addr3:1.2.3", options=options, + expected_registries_metadata=[ + dict( + alias=None, + address="myregistry3domain:port", + repository="addr3", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry3domain:port/addr3:1.2.3", + ) + ], + ) + ], ) assert_build( rule_runner, Address("docker/test", target_name="alias1"), "Built docker image: myregistry1domain:port/alias1:1.2.3", options=options, + expected_registries_metadata=[ + dict( + alias="reg1", + address="myregistry1domain:port", + repository="alias1", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/alias1:1.2.3", + ) + ], + ) + ], ) assert_build( rule_runner, @@ -346,12 +480,37 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: Address("docker/test", target_name="unreg"), "Built docker image: unreg:1.2.3", options=options, + expected_registries_metadata=[ + dict( + alias=None, + address=None, + repository="unreg", + tags=[ + dict(template="1.2.3", tag="1.2.3", uses_local_alias=False, name="unreg:1.2.3") + ], + ) + ], ) assert_build( rule_runner, Address("docker/test", target_name="def"), "Built docker image: myregistry2domain:port/def:1.2.3", options=options, + expected_registries_metadata=[ + dict( + alias="reg2", + address="myregistry2domain:port", + repository="def", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry2domain:port/def:1.2.3", + ) + ], + ) + ], ) assert_build( rule_runner, @@ -362,6 +521,34 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * myregistry1domain:port/multi:1.2.3" ), options=options, + expected_registries_metadata=[ + dict( + alias="reg1", + address="myregistry1domain:port", + repository="multi", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/multi:1.2.3", + ) + ], + ), + dict( + alias="reg2", + address="myregistry2domain:port", + repository="multi", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry2domain:port/multi:1.2.3", + ) + ], + ), + ], ) assert_build( rule_runner, @@ -373,6 +560,40 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: " * extra/extra_tags:latest" ), options=options, + expected_registries_metadata=[ + dict( + alias="extra", + address="extra", + repository="extra_tags", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="extra/extra_tags:1.2.3", + ), + dict( + template="latest", + tag="latest", + uses_local_alias=False, + name="extra/extra_tags:latest", + ), + ], + ), + dict( + alias="reg1", + address="myregistry1domain:port", + repository="extra_tags", + tags=[ + dict( + template="1.2.3", + tag="1.2.3", + uses_local_alias=False, + name="myregistry1domain:port/extra_tags:1.2.3", + ) + ], + ), + ], ) @@ -390,11 +611,12 @@ def test_dynamic_image_version(rule_runner: RuleRunner) -> None: def assert_tags(name: str, *expect_tags: str) -> None: tgt = rule_runner.get_target(Address("docker/test", target_name=name)) fs = DockerPackageFieldSet.create(tgt) - tags = fs.image_refs( + image_refs = fs.image_refs( "image", DockerRegistries.from_dict({}), interpolation_context, ) + tags = tuple(t.full_name for r in image_refs for t in r.tags) assert expect_tags == tags rule_runner.write_files( @@ -601,6 +823,21 @@ def test_docker_image_version_from_build_arg(rule_runner: RuleRunner) -> None: rule_runner, Address("docker/test", target_name="ver1"), "Built docker image: ver1:1.2.3", + expected_registries_metadata=[ + dict( + alias=None, + address=None, + repository="ver1", + tags=[ + dict( + template="{build_args.VERSION}", + tag="1.2.3", + uses_local_alias=False, + name="ver1:1.2.3", + ) + ], + ) + ], ) @@ -948,6 +1185,18 @@ def check_docker_proc(process: Process): options=options, process_assertions=check_docker_proc, version_tags=("build latest", "dev latest", "prod latest"), + expected_registries_metadata=[ + dict( + address=None, + alias=None, + repository="image", + tags=[ + dict( + template="latest", tag="latest", uses_local_alias=False, name="image:latest" + ) + ], + ) + ], ) @@ -1093,17 +1342,128 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std @pytest.mark.parametrize( "test", [ - ImageRefTest(docker_image=dict(name="lowercase"), expect_refs=("lowercase:latest",)), - ImageRefTest(docker_image=dict(name="CamelCase"), expect_refs=("camelcase:latest",)), - ImageRefTest(docker_image=dict(image_tags=["CamelCase"]), expect_refs=("image:CamelCase",)), + ImageRefTest( + docker_image=dict(name="lowercase"), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="lowercase", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="lowercase:latest", + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(name="CamelCase"), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="camelcase", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="camelcase:latest", + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(image_tags=["CamelCase"]), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="image", + tags=( + ImageRefTag( + template="CamelCase", + formatted="CamelCase", + uses_local_alias=False, + full_name="image:CamelCase", + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(image_tags=["{val1}", "prefix-{val2}"]), + expect_refs=( + ImageRefRegistry( + registry=None, + repository="image", + tags=( + ImageRefTag( + template="{val1}", + formatted="first-value", + uses_local_alias=False, + full_name="image:first-value", + ), + ImageRefTag( + template="prefix-{val2}", + formatted="prefix-second-value", + uses_local_alias=False, + full_name="image:prefix-second-value", + ), + ), + ), + ), + ), ImageRefTest( docker_image=dict(registries=["REG1.example.net"]), - expect_refs=("REG1.example.net/image:latest",), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="REG1.example.net"), + repository="image", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="REG1.example.net/image:latest", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict(registries=["docker.io", "@private"], repository="our-the/pkg"), registries=dict(private={"address": "our.registry", "repository": "the/pkg"}), - expect_refs=("docker.io/our-the/pkg:latest", "our.registry/the/pkg:latest"), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="our-the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="docker.io/our-the/pkg:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", address="our.registry", repository="the/pkg" + ), + repository="the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="our.registry/the/pkg:latest", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict( @@ -1113,7 +1473,66 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std registries=dict( private={"address": "our.registry", "repository": "{target_repository}/the/pkg"} ), - expect_refs=("docker.io/test/image:latest", "our.registry/test/image/the/pkg:latest"), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="test/image", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="docker.io/test/image:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + repository="{target_repository}/the/pkg", + ), + repository="test/image/the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="our.registry/test/image/the/pkg:latest", + ), + ), + ), + ), + ), + ImageRefTest( + docker_image=dict(registries=["@private"], image_tags=["prefix-{val1}"]), + registries=dict( + private={"address": "our.registry", "extra_image_tags": ["{val2}-suffix"]} + ), + expect_refs=( + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + extra_image_tags=("{val2}-suffix",), + ), + repository="image", + tags=( + ImageRefTag( + template="prefix-{val1}", + formatted="prefix-first-value", + uses_local_alias=False, + full_name="our.registry/image:prefix-first-value", + ), + ImageRefTag( + template="{val2}-suffix", + formatted="second-value-suffix", + uses_local_alias=False, + full_name="our.registry/image:second-value-suffix", + ), + ), + ), + ), ), ImageRefTest( docker_image=dict(repository="{default_repository}/a"), @@ -1140,9 +1559,41 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std } ), expect_refs=( - "docker.io/our-the/pkg:latest", - "private/the/pkg:latest", - "our.registry/the/pkg:latest", + ImageRefRegistry( + registry=DockerRegistryOptions(address="docker.io"), + repository="our-the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="docker.io/our-the/pkg:latest", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + alias="private", + address="our.registry", + repository="the/pkg", + use_local_alias=True, + ), + repository="the/pkg", + tags=( + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=False, + full_name="our.registry/the/pkg:latest", + ), + ImageRefTag( + template="latest", + formatted="latest", + uses_local_alias=True, + full_name="private/the/pkg:latest", + ), + ), + ), ), ), ], @@ -1152,12 +1603,14 @@ def test_image_ref_formatting(test: ImageRefTest) -> None: tgt = DockerImageTarget(test.docker_image, address) field_set = DockerPackageFieldSet.create(tgt) registries = DockerRegistries.from_dict(test.registries) - interpolation_context = InterpolationContext.from_dict({}) + interpolation_context = InterpolationContext.from_dict( + {"val1": "first-value", "val2": "second-value"} + ) with test.expect_error or no_exception(): - assert ( - field_set.image_refs(test.default_repository, registries, interpolation_context) - == test.expect_refs + image_refs = field_set.image_refs( + test.default_repository, registries, interpolation_context ) + assert tuple(image_refs) == test.expect_refs def test_docker_image_tags_from_plugin_hook(rule_runner: RuleRunner) -> None: @@ -1183,3 +1636,122 @@ def check_docker_proc(process: Process): process_assertions=check_docker_proc, plugin_tags=("1.2.3",), ) + + +def test_docker_info_serialize() -> None: + image_id = "abc123" + # image refs with unique strings (i.e. not actual templates/names etc.), to make sure they're + # ending up in the right place in the JSON + image_refs = ( + ImageRefRegistry( + registry=None, + repository="repo", + tags=( + ImageRefTag( + template="repo tag1 template", + formatted="repo tag1 formatted", + uses_local_alias=False, + full_name="repo tag1 full name", + ), + ImageRefTag( + template="repo tag2 template", + formatted="repo tag2 formatted", + uses_local_alias=False, + full_name="repo tag2 full name", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions(address="address"), + repository="address repo", + tags=( + ImageRefTag( + template="address tag template", + formatted="address tag formatted", + uses_local_alias=False, + full_name="address tag full name", + ), + ), + ), + ImageRefRegistry( + registry=DockerRegistryOptions( + address="alias address", alias="alias", repository="alias registry repo" + ), + repository="alias repo", + tags=( + ImageRefTag( + template="alias tag (address) template", + formatted="alias tag (address) formatted", + uses_local_alias=False, + full_name="alias tag (address) full name", + ), + ImageRefTag( + template="alias tag (local alias) template", + formatted="alias tag (local alias) formatted", + uses_local_alias=True, + full_name="alias tag (local alias) full name", + ), + ), + ), + ) + + expected = dict( + version=1, + image_id=image_id, + registries=[ + dict( + alias=None, + address=None, + repository="repo", + tags=[ + dict( + template="repo tag1 template", + tag="repo tag1 formatted", + uses_local_alias=False, + name="repo tag1 full name", + ), + dict( + template="repo tag2 template", + tag="repo tag2 formatted", + uses_local_alias=False, + name="repo tag2 full name", + ), + ], + ), + dict( + alias=None, + address="address", + repository="address repo", + tags=[ + dict( + template="address tag template", + tag="address tag formatted", + uses_local_alias=False, + name="address tag full name", + ) + ], + ), + dict( + alias="alias", + address="alias address", + repository="alias repo", + tags=[ + dict( + template="alias tag (address) template", + tag="alias tag (address) formatted", + uses_local_alias=False, + name="alias tag (address) full name", + ), + dict( + template="alias tag (local alias) template", + tag="alias tag (local alias) formatted", + uses_local_alias=True, + name="alias tag (local alias) full name", + ), + ], + ), + ], + ) + + result = DockerInfoV1.serialize(image_refs, image_id) + assert json.loads(result) == expected diff --git a/src/python/pants/backend/docker/goals/publish_test.py b/src/python/pants/backend/docker/goals/publish_test.py index b2c153177b1..3ed8fcccbfb 100644 --- a/src/python/pants/backend/docker/goals/publish_test.py +++ b/src/python/pants/backend/docker/goals/publish_test.py @@ -56,17 +56,19 @@ def rule_runner() -> RuleRunner: def build(tgt: DockerImageTarget, options: DockerOptions): fs = DockerPackageFieldSet.create(tgt) + image_refs = fs.image_refs( + options.default_repository, + options.registries(), + InterpolationContext(), + ) return ( BuiltPackage( EMPTY_DIGEST, ( BuiltDockerImage.create( "sha256:made-up", - fs.image_refs( - options.default_repository, - options.registries(), - InterpolationContext(), - ), + tuple(t.full_name for r in image_refs for t in r.tags), + "made-up.json", ), ), ), diff --git a/src/python/pants/backend/docker/package_types.py b/src/python/pants/backend/docker/package_types.py index e978f9d4900..bcce33b6654 100644 --- a/src/python/pants/backend/docker/package_types.py +++ b/src/python/pants/backend/docker/package_types.py @@ -18,12 +18,14 @@ class BuiltDockerImage(BuiltPackageArtifact): tags: tuple[str, ...] = () @classmethod - def create(cls, image_id: str, tags: tuple[str, ...]) -> BuiltDockerImage: + def create( + cls, image_id: str, tags: tuple[str, ...], metadata_filename: str + ) -> BuiltDockerImage: tags_string = tags[0] if len(tags) == 1 else f"\n{bullet_list(tags)}" return cls( image_id=image_id, tags=tags, - relpath=None, + relpath=metadata_filename, extra_log_lines=( f"Built docker {pluralize(len(tags), 'image', False)}: {tags_string}", f"Docker image ID: {image_id}", diff --git a/src/python/pants/backend/docker/target_types.py b/src/python/pants/backend/docker/target_types.py index 80fd8758779..009c2061810 100644 --- a/src/python/pants/backend/docker/target_types.py +++ b/src/python/pants/backend/docker/target_types.py @@ -13,6 +13,7 @@ from pants.backend.docker.registries import ALL_DEFAULT_REGISTRIES from pants.base.build_environment import get_buildroot +from pants.core.goals.package import OutputPathField from pants.core.goals.run import RestartableField from pants.engine.addresses import Address from pants.engine.collection import Collection @@ -404,6 +405,7 @@ class DockerImageTarget(Target): DockerImageTargetStageField, DockerImageBuildPullOptionField, DockerImageBuildSquashOptionField, + OutputPathField, RestartableField, ) help = softwrap( diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py index ded48c2280c..ffca1bcc247 100644 --- a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py +++ b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py @@ -241,7 +241,7 @@ def test_from_image_build_arg_dependency(rule_runner: RuleRunner) -> None: assert_build_context( rule_runner, Address("src/downstream", target_name="image"), - expected_files=["src/downstream/Dockerfile"], + expected_files=["src/downstream/Dockerfile", "src.upstream/image.docker-info.json"], build_upstream_images=True, expected_interpolation_context={ "tags": { diff --git a/src/python/pants/backend/helm/util_rules/post_renderer.py b/src/python/pants/backend/helm/util_rules/post_renderer.py index 042b6bf0238..a9b7b2c60f8 100644 --- a/src/python/pants/backend/helm/util_rules/post_renderer.py +++ b/src/python/pants/backend/helm/util_rules/post_renderer.py @@ -119,12 +119,14 @@ async def resolve_docker_image_ref(address: Address, context: DockerBuildContext # Choose first non-latest image reference found, or fallback to 'latest'. found_ref: str | None = None fallback_ref: str | None = None - for ref in image_refs: - if ref.endswith(":latest"): - fallback_ref = ref - else: - found_ref = ref - break + for registry in image_refs: + for tag in registry.tags: + ref = tag.full_name + if ref.endswith(":latest"): + fallback_ref = ref + else: + found_ref = ref + break resolved_ref = found_ref or fallback_ref if resolved_ref: