From a1aa9bfd9cf36fc6c576b73c24446d7ad48e3eb1 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Fri, 19 Aug 2022 08:24:31 +0200 Subject: [PATCH 1/9] Initial implementation of interpolation values in helm_deployment --- .../backend/docker/goals/package_image.py | 31 ++--- .../docker/goals/package_image_test.py | 11 +- .../backend/docker/goals/publish_test.py | 4 +- .../docker/util_rules/docker_build_context.py | 13 +- .../util_rules/docker_build_context_test.py | 11 +- .../backend/docker/value_interpolation.py | 119 +----------------- .../pants/backend/helm/subsystems/helm.py | 17 ++- src/python/pants/backend/helm/target_types.py | 28 +++++ .../pants/backend/helm/util_rules/renderer.py | 20 ++- .../pants/backend/helm/util_rules/tool.py | 10 +- .../pants/backend/helm/value_interpolation.py | 24 ++++ src/python/pants/util/value_interpolation.py | 109 ++++++++++++++++ 12 files changed, 237 insertions(+), 160 deletions(-) create mode 100644 src/python/pants/backend/helm/value_interpolation.py create mode 100644 src/python/pants/util/value_interpolation.py diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index dab165183f0..e818a06f44f 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -32,10 +32,6 @@ DockerBuildContextRequest, ) from pants.backend.docker.utils import format_rename_suggestion -from pants.backend.docker.value_interpolation import ( - DockerInterpolationContext, - DockerInterpolationError, -) from pants.core.goals.package import BuiltPackage, PackageFieldSet from pants.core.goals.run import RunFieldSet from pants.engine.addresses import Address @@ -45,15 +41,16 @@ from pants.engine.unions import UnionMembership, UnionRule from pants.option.global_options import GlobalOptions, KeepSandboxes from pants.util.strutil import bullet_list +from pants.util.value_interpolation import InterpolationContext, InterpolationError logger = logging.getLogger(__name__) -class DockerImageTagValueError(DockerInterpolationError): +class DockerImageTagValueError(InterpolationError): pass -class DockerRepositoryNameError(DockerInterpolationError): +class DockerRepositoryNameError(InterpolationError): pass @@ -76,8 +73,8 @@ class DockerFieldSet(PackageFieldSet, RunFieldSet): tags: DockerImageTagsField target_stage: DockerImageTargetStageField - def format_tag(self, tag: str, interpolation_context: DockerInterpolationContext) -> str: - source = DockerInterpolationContext.TextSource( + def format_tag(self, tag: str, interpolation_context: InterpolationContext) -> str: + source = InterpolationContext.TextSource( address=self.address, target_alias="docker_image", field_alias=self.tags.alias ) return interpolation_context.format(tag, source=source, error_cls=DockerImageTagValueError) @@ -85,10 +82,10 @@ def format_tag(self, tag: str, interpolation_context: DockerInterpolationContext def format_repository( self, default_repository: str, - interpolation_context: DockerInterpolationContext, + interpolation_context: InterpolationContext, registry: DockerRegistryOptions | None = None, ) -> str: - repository_context = DockerInterpolationContext.from_dict( + repository_context = InterpolationContext.from_dict( { "directory": os.path.basename(self.address.spec_path), "name": self.address.target_name, @@ -100,19 +97,17 @@ def format_repository( ) if registry and registry.repository: repository_text = registry.repository - source = DockerInterpolationContext.TextSource( + source = InterpolationContext.TextSource( options_scope=f"[docker.registries.{registry.alias or registry.address}].repository" ) elif self.repository.value: repository_text = self.repository.value - source = DockerInterpolationContext.TextSource( + source = InterpolationContext.TextSource( address=self.address, target_alias="docker_image", field_alias=self.repository.alias ) else: repository_text = default_repository - source = DockerInterpolationContext.TextSource( - options_scope="[docker].default_repository" - ) + source = InterpolationContext.TextSource(options_scope="[docker].default_repository") return repository_context.format( repository_text, source=source, error_cls=DockerRepositoryNameError ).lower() @@ -121,7 +116,7 @@ def format_names( self, repository: str, tags: tuple[str, ...], - interpolation_context: DockerInterpolationContext, + interpolation_context: InterpolationContext, ) -> Iterator[str]: for tag in tags: yield ":".join( @@ -132,7 +127,7 @@ def image_refs( self, default_repository: str, registries: DockerRegistries, - interpolation_context: DockerInterpolationContext, + interpolation_context: InterpolationContext, additional_tags: tuple[str, ...] = (), ) -> tuple[str, ...]: """The image refs are the full image name, including any registry and version tag. @@ -199,7 +194,7 @@ def get_build_options( # Build options from target fields inheriting from DockerBuildOptionFieldMixin for field_type in target.field_types: if issubclass(field_type, DockerBuildOptionFieldMixin): - source = DockerInterpolationContext.TextSource( + source = InterpolationContext.TextSource( address=target.address, target_alias=target.alias, field_alias=field_type.alias ) format = partial( diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index 38bd8da9e73..9df66c86ad2 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -43,10 +43,6 @@ DockerBuildEnvironmentRequest, ) from pants.backend.docker.util_rules.docker_build_env import rules as build_env_rules -from pants.backend.docker.value_interpolation import ( - DockerInterpolationContext, - DockerInterpolationError, -) from pants.engine.addresses import Address from pants.engine.fs import EMPTY_DIGEST, EMPTY_FILE_DIGEST, EMPTY_SNAPSHOT, Snapshot from pants.engine.platform import Platform @@ -63,6 +59,7 @@ from pants.testutil.pytest_util import assert_logged, no_exception from pants.testutil.rule_runner import MockGet, QueryRule, RuleRunner, run_rule_with_mocks from pants.util.frozendict import FrozenDict +from pants.util.value_interpolation import InterpolationContext, InterpolationError @pytest.fixture @@ -380,7 +377,7 @@ def test_build_image_with_registries(rule_runner: RuleRunner) -> None: def test_dynamic_image_version(rule_runner: RuleRunner) -> None: - interpolation_context = DockerInterpolationContext.from_dict( + interpolation_context = InterpolationContext.from_dict( { "baseimage": {"tag": "3.8"}, "stage0": {"tag": "3.8"}, @@ -1044,7 +1041,7 @@ def test_parse_image_id_from_docker_build_output(expected: str, stdout: str, std docker_image=dict(repository="{default_repository}/a"), default_repository="{target_repository}/b", expect_error=pytest.raises( - DockerInterpolationError, + InterpolationError, match=( r"Invalid value for the `repository` field of the `docker_image` target at " r"src/test/docker:image: '\{default_repository\}/a'\.\n\n" @@ -1061,7 +1058,7 @@ def test_image_ref_formatting(test: ImageRefTest) -> None: tgt = DockerImageTarget(test.docker_image, address) field_set = DockerFieldSet.create(tgt) registries = DockerRegistries.from_dict(test.registries) - interpolation_context = DockerInterpolationContext.from_dict({}) + interpolation_context = InterpolationContext.from_dict({}) with test.expect_error or no_exception(): assert ( field_set.image_refs(test.default_repository, registries, interpolation_context) diff --git a/src/python/pants/backend/docker/goals/publish_test.py b/src/python/pants/backend/docker/goals/publish_test.py index 5b9c42a2162..2f4501929ca 100644 --- a/src/python/pants/backend/docker/goals/publish_test.py +++ b/src/python/pants/backend/docker/goals/publish_test.py @@ -17,7 +17,6 @@ from pants.backend.docker.target_types import DockerImageTarget from pants.backend.docker.util_rules import docker_binary from pants.backend.docker.util_rules.docker_binary import DockerBinary -from pants.backend.docker.value_interpolation import DockerInterpolationContext from pants.core.goals.package import BuiltPackage from pants.core.goals.publish import PublishPackages, PublishProcesses from pants.engine.addresses import Address @@ -27,6 +26,7 @@ from pants.testutil.process_util import process_assertion from pants.testutil.rule_runner import QueryRule, RuleRunner from pants.util.frozendict import FrozenDict +from pants.util.value_interpolation import InterpolationContext @pytest.fixture @@ -65,7 +65,7 @@ def build(tgt: DockerImageTarget, options: DockerOptions): fs.image_refs( options.default_repository, options.registries(), - DockerInterpolationContext(), + InterpolationContext(), ), ), ), diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context.py b/src/python/pants/backend/docker/util_rules/docker_build_context.py index 95429704422..c1d91703e98 100644 --- a/src/python/pants/backend/docker/util_rules/docker_build_context.py +++ b/src/python/pants/backend/docker/util_rules/docker_build_context.py @@ -22,11 +22,7 @@ DockerBuildEnvironmentRequest, ) from pants.backend.docker.utils import get_hash, suggest_renames -from pants.backend.docker.value_interpolation import ( - DockerBuildArgsInterpolationValue, - DockerInterpolationContext, - DockerInterpolationValue, -) +from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue from pants.backend.shell.target_types import ShellSourceField from pants.core.goals.package import BuiltPackage, PackageFieldSet from pants.core.target_types import FileSourceField @@ -49,6 +45,7 @@ from pants.engine.unions import UnionRule from pants.util.meta import classproperty from pants.util.strutil import softwrap +from pants.util.value_interpolation import InterpolationContext, InterpolationValue logger = logging.getLogger(__name__) @@ -105,7 +102,7 @@ class DockerBuildContext: build_env: DockerBuildEnvironment upstream_image_ids: tuple[str, ...] dockerfile: str - interpolation_context: DockerInterpolationContext + interpolation_context: InterpolationContext copy_source_vs_context_source: tuple[tuple[str, str], ...] stages: tuple[str, ...] @@ -118,7 +115,7 @@ def create( upstream_image_ids: Iterable[str], dockerfile_info: DockerfileInfo, ) -> DockerBuildContext: - interpolation_context: dict[str, dict[str, str] | DockerInterpolationValue] = {} + interpolation_context: dict[str, dict[str, str] | InterpolationValue] = {} if build_args: interpolation_context["build_args"] = cls._merge_build_args( @@ -148,7 +145,7 @@ def create( dockerfile=dockerfile_info.source, build_env=build_env, upstream_image_ids=tuple(sorted(upstream_image_ids)), - interpolation_context=DockerInterpolationContext.from_dict(interpolation_context), + interpolation_context=InterpolationContext.from_dict(interpolation_context), copy_source_vs_context_source=tuple( suggest_renames( tentative_paths=( diff --git a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py index 723818e25aa..ded48c2280c 100644 --- a/src/python/pants/backend/docker/util_rules/docker_build_context_test.py +++ b/src/python/pants/backend/docker/util_rules/docker_build_context_test.py @@ -26,11 +26,7 @@ DockerBuildContextRequest, ) from pants.backend.docker.util_rules.docker_build_env import DockerBuildEnvironment -from pants.backend.docker.value_interpolation import ( - DockerBuildArgsInterpolationValue, - DockerInterpolationContext, - DockerInterpolationValue, -) +from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue from pants.backend.python import target_types_rules from pants.backend.python.goals import package_pex_binary from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet @@ -47,6 +43,7 @@ from pants.engine.internals.scheduler import ExecutionError from pants.testutil.pytest_util import no_exception from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.value_interpolation import InterpolationContext, InterpolationValue def create_rule_runner() -> RuleRunner: @@ -91,7 +88,7 @@ def assert_build_context( *, build_upstream_images: bool = False, expected_files: list[str], - expected_interpolation_context: dict[str, str | dict[str, str] | DockerInterpolationValue] + expected_interpolation_context: dict[str, str | dict[str, str] | InterpolationValue] | None = None, expected_num_upstream_images: int = 0, pants_args: list[str] | None = None, @@ -125,7 +122,7 @@ def assert_build_context( # Converting to `dict` to avoid the fact that FrozenDict is sensitive to the order of the keys. assert dict(context.interpolation_context) == dict( - DockerInterpolationContext.from_dict(expected_interpolation_context) + InterpolationContext.from_dict(expected_interpolation_context) ) if build_upstream_images: diff --git a/src/python/pants/backend/docker/value_interpolation.py b/src/python/pants/backend/docker/value_interpolation.py index 9a99dab6776..9966f1aee3d 100644 --- a/src/python/pants/backend/docker/value_interpolation.py +++ b/src/python/pants/backend/docker/value_interpolation.py @@ -3,46 +3,16 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import ClassVar, Mapping, TypeVar, Union +from pants.util.value_interpolation import InterpolationError, InterpolationValue -from pants.engine.addresses import Address -from pants.util.frozendict import FrozenDict - -class DockerInterpolationError(ValueError): - @classmethod - def attribute_error( - cls, value: str | DockerInterpolationValue, attribute: str - ) -> DockerInterpolationError: - msg = f"The placeholder {attribute!r} is unknown." - if value and isinstance(value, DockerInterpolationValue): - msg += f' Try with one of: {", ".join(value.keys())}.' - return cls(msg) - - -ErrorT = TypeVar("ErrorT", bound=DockerInterpolationError) - - -class DockerInterpolationValue(FrozenDict[str, str]): - """Dict class suitable for use as a format string context object, as it allows to use attribute - access rather than item access.""" - - _attribute_error_type: ClassVar[type[DockerInterpolationError]] = DockerInterpolationError - - def __getattr__(self, attribute: str) -> str: - if attribute not in self: - raise self._attribute_error_type.attribute_error(self, attribute) - return self[attribute] - - -class DockerBuildArgInterpolationError(DockerInterpolationError): +class DockerBuildArgInterpolationError(InterpolationError): @classmethod def attribute_error( - cls, value: str | DockerInterpolationValue, attribute: str - ) -> DockerInterpolationError: + cls, value: str | DockerBuildArgsInterpolationValue, attribute: str + ) -> DockerBuildArgInterpolationError: msg = f"The build arg {attribute!r} is undefined." - if value and isinstance(value, DockerInterpolationValue): + if value and isinstance(value, DockerBuildArgsInterpolationValue): msg += f' Defined build args are: {", ".join(value.keys())}.' msg += ( "\n\nThis build arg may be defined with the `[docker].build_args` option or directly " @@ -51,84 +21,7 @@ def attribute_error( return cls(msg) -class DockerBuildArgsInterpolationValue(DockerInterpolationValue): +class DockerBuildArgsInterpolationValue(InterpolationValue): """Interpolation context value with specific error handling for build args.""" _attribute_error_type = DockerBuildArgInterpolationError - - -class DockerInterpolationContext(FrozenDict[str, Union[str, DockerInterpolationValue]]): - @classmethod - def from_dict(cls, data: Mapping[str, str | Mapping[str, str]]) -> DockerInterpolationContext: - return DockerInterpolationContext( - {key: cls.create_value(value) for key, value in data.items()} - ) - - @staticmethod - def create_value(value: str | Mapping[str, str]) -> str | DockerInterpolationValue: - """Ensure that `value` satisfies the type `DockerInterpolationValue`.""" - if isinstance(value, (str, DockerInterpolationValue)): - return value - return DockerInterpolationValue(value) - - def merge(self, other: Mapping[str, str | Mapping[str, str]]) -> DockerInterpolationContext: - return DockerInterpolationContext.from_dict({**self, **other}) - - def format( - self, text: str, *, source: TextSource, error_cls: type[ErrorT] | None = None - ) -> str: - stack = [text] - try: - while "{" in stack[-1] and "}" in stack[-1]: - if len(stack) >= 5: - raise DockerInterpolationError( - "The formatted placeholders recurse too deep.\n" - + " => ".join(map(repr, stack)) - ) - stack.append(stack[-1].format(**self)) - if stack[-1] == stack[-2]: - break - return stack[-1] - except (KeyError, DockerInterpolationError) as e: - default_error_cls = DockerInterpolationError - msg = f"Invalid value for the {source}: {text!r}.\n\n" - if isinstance(e, DockerInterpolationError): - default_error_cls = type(e) - msg += str(e) - else: - # KeyError - msg += f"The placeholder {e} is unknown." - if self: - msg += f" Try with one of: {', '.join(sorted(self.keys()))}." - else: - msg += ( - " There are currently no known placeholders to use. These placeholders " - "can come from `[docker].build_args` or parsed from instructions in your " - "`Dockerfile`." - ) - raise (error_cls or default_error_cls)(msg) from e - - @dataclass(frozen=True) - class TextSource: - address: Address | None = None - target_alias: str | None = None - field_alias: str | None = None - options_scope: str | None = None - - def __post_init__(self): - field_infos_is_none = ( - x is None for x in [self.address, self.target_alias, self.field_alias] - ) - if self.options_scope is None: - assert not any(field_infos_is_none), f"Missing target field details in {self!r}." - else: - assert all( - field_infos_is_none - ), f"Must not refer to both configuration option and target field in {self!r}." - - def __str__(self) -> str: - if self.options_scope: - return f"`{self.options_scope}` configuration option" - return ( - f"`{self.field_alias}` field of the `{self.target_alias}` target at {self.address}" - ) diff --git a/src/python/pants/backend/helm/subsystems/helm.py b/src/python/pants/backend/helm/subsystems/helm.py index 679ba3ad81f..1e70665dbab 100644 --- a/src/python/pants/backend/helm/subsystems/helm.py +++ b/src/python/pants/backend/helm/subsystems/helm.py @@ -10,7 +10,13 @@ from pants.backend.helm.target_types import HelmChartTarget, HelmRegistriesField from pants.core.util_rules.external_tool import TemplatedExternalTool from pants.engine.platform import Platform -from pants.option.option_types import ArgsListOption, BoolOption, DictOption, StrOption +from pants.option.option_types import ( + ArgsListOption, + BoolOption, + DictOption, + StrListOption, + StrOption, +) from pants.util.memo import memoized_method from pants.util.strutil import bullet_list, softwrap @@ -111,6 +117,15 @@ class HelmSubsystem(TemplatedExternalTool): """ ), ) + extra_env_vars = StrListOption( + help=softwrap( + """ + Additional environment variables that would be made available to all Helm processes + or during value interpolation. + """ + ), + advanced=True, + ) tailor = BoolOption( default=True, help="If true, add `helm_chart` targets with the `tailor` goal.", diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index e90b6871ea7..d20fcb08896 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -3,9 +3,11 @@ from __future__ import annotations +import logging from dataclasses import dataclass from pants.backend.helm.resolve.remotes import ALL_DEFAULT_HELM_REGISTRIES +from pants.backend.helm.value_interpolation import HelmEnvironmentInterpolationError from pants.core.goals.package import OutputPathField from pants.core.goals.test import TestTimeoutField from pants.engine.rules import collect_rules, rule @@ -33,6 +35,9 @@ ) from pants.util.docutil import bin_name from pants.util.strutil import softwrap +from pants.util.value_interpolation import InterpolationContext + +logger = logging.getLogger(__name__) # ----------------------------------------------------------------------------------------------- # Generic commonly used fields @@ -465,6 +470,29 @@ class HelmDeploymentFieldSet(FieldSet): dependencies: HelmDeploymentDependenciesField values: HelmDeploymentValuesField + def format_values(self, interpolation_context: InterpolationContext) -> dict[str, str]: + def format_value(text: str) -> str | None: + source = InterpolationContext.TextSource( + self.address, + target_alias=HelmDeploymentTarget.alias, + field_alias=HelmDeploymentValuesField.alias, + ) + + try: + return interpolation_context.format( + text, source=source, error_cls=HelmEnvironmentInterpolationError + ) + except HelmEnvironmentInterpolationError as err: + logger.warning(err) + return None + + result = {} + for key, value in (self.values.value or {}).items(): + interpolated_value = format_value(value) + if interpolated_value is not None: + result[key] = interpolated_value + return result + class AllHelmDeploymentTargets(Targets): pass diff --git a/src/python/pants/backend/helm/util_rules/renderer.py b/src/python/pants/backend/helm/util_rules/renderer.py index b6e5d9f19d2..77aeae5fd82 100644 --- a/src/python/pants/backend/helm/util_rules/renderer.py +++ b/src/python/pants/backend/helm/util_rules/renderer.py @@ -14,14 +14,17 @@ from typing import Any, Iterable, Mapping from pants.backend.helm.subsystems import post_renderer +from pants.backend.helm.subsystems.helm import HelmSubsystem from pants.backend.helm.subsystems.post_renderer import HelmPostRenderer from pants.backend.helm.target_types import HelmDeploymentFieldSet, HelmDeploymentSourcesField from pants.backend.helm.util_rules import chart, tool from pants.backend.helm.util_rules.chart import FindHelmDeploymentChart, HelmChart from pants.backend.helm.util_rules.tool import HelmProcess +from pants.backend.helm.value_interpolation import HelmEnvironmentInterpolationValue from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType +from pants.engine.environment import Environment, EnvironmentRequest from pants.engine.fs import ( EMPTY_DIGEST, EMPTY_SNAPSHOT, @@ -41,6 +44,7 @@ from pants.util.logging import LogLevel from pants.util.meta import frozen_after_init from pants.util.strutil import pluralize, softwrap +from pants.util.value_interpolation import InterpolationContext, InterpolationValue logger = logging.getLogger(__name__) @@ -169,6 +173,16 @@ def metadata(self) -> dict[str, Any] | None: return {"address": self.address, "chart": self.chart, "post_processed": self.post_processed} +@rule_helper +async def _build_interpolation_context(helm_subsystem: HelmSubsystem) -> InterpolationContext: + interpolation_context: dict[str, dict[str, str] | InterpolationValue] = {} + + env = await Get(Environment, EnvironmentRequest(helm_subsystem.extra_env_vars)) + interpolation_context["env"] = HelmEnvironmentInterpolationValue(env) + + return InterpolationContext.from_dict(interpolation_context) + + @rule_helper async def _sort_value_file_names_for_evaluation( address: Address, @@ -230,7 +244,7 @@ def minimise_and_sort_subset(input_subset: set[str]) -> list[str]: @rule(desc="Prepare Helm deployment renderer") async def setup_render_helm_deployment_process( - request: HelmDeploymentRequest, + request: HelmDeploymentRequest, helm_subsystem: HelmSubsystem ) -> _HelmDeploymentProcessWrapper: value_files_prefix = "__values" chart, value_files = await MultiGet( @@ -282,11 +296,13 @@ async def setup_render_helm_deployment_process( merged_digests = await Get(Digest, MergeDigests(input_digests)) + interpolation_context = await _build_interpolation_context(helm_subsystem) + release_name = ( request.field_set.release_name.value or request.field_set.address.target_name.replace("_", "-") ) - inline_values = request.field_set.values.value + inline_values = request.field_set.format_values(interpolation_context) def maybe_escape_string_value(value: str) -> str: if re.findall("\\s+", value): diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index cefcd3188f9..9bc423ea4ad 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -27,14 +27,17 @@ from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.environment import Environment, EnvironmentRequest from pants.engine.fs import ( + AddPrefix, CreateDigest, Digest, DigestContents, DigestSubset, Directory, FileDigest, + MergeDigests, PathGlobs, RemovePrefix, + Snapshot, ) from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, MergeDigests, Snapshot from pants.engine.platform import Platform @@ -433,8 +436,11 @@ async def setup_helm(helm_subsytem: HelmSubsystem, global_plugins: HelmPlugins) @rule -def helm_process(request: HelmProcess, helm_binary: HelmBinary) -> Process: - env = {**helm_binary.env, **request.extra_env} +async def helm_process( + request: HelmProcess, helm_binary: HelmBinary, helm_subsytem: HelmSubsystem +) -> Process: + global_extra_env = await Get(Environment, EnvironmentRequest(helm_subsytem.extra_env_vars)) + env = {**global_extra_env, **helm_binary.env, **request.extra_env} immutable_input_digests = { **helm_binary.immutable_input_digests, diff --git a/src/python/pants/backend/helm/value_interpolation.py b/src/python/pants/backend/helm/value_interpolation.py new file mode 100644 index 00000000000..b68265f0feb --- /dev/null +++ b/src/python/pants/backend/helm/value_interpolation.py @@ -0,0 +1,24 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from pants.util.value_interpolation import InterpolationError, InterpolationValue + + +class HelmEnvironmentInterpolationError(InterpolationError): + @classmethod + def attribute_error( + cls, value: str | HelmEnvironmentInterpolationValue, attribute: str + ) -> HelmEnvironmentInterpolationError: + msg = f"The environment variable {attribute!r} is undefined." + if value and isinstance(value, HelmEnvironmentInterpolationValue): + msg += f' Available environment variables are: {", ".join(value.keys())}.' + msg += "\n\nAvailable environment variables are defined using the `[helm].extra_env_vars` option." + return cls(msg) + + +class HelmEnvironmentInterpolationValue(InterpolationValue): + """Interpolation context value with specific error handling for environment variables.""" + + _attribute_error_type = HelmEnvironmentInterpolationError diff --git a/src/python/pants/util/value_interpolation.py b/src/python/pants/util/value_interpolation.py new file mode 100644 index 00000000000..9168fd17cd3 --- /dev/null +++ b/src/python/pants/util/value_interpolation.py @@ -0,0 +1,109 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar, Mapping, TypeVar, Union + +from pants.engine.addresses import Address +from pants.util.frozendict import FrozenDict + + +class InterpolationError(ValueError): + @classmethod + def attribute_error(cls, value: str | InterpolationValue, attribute: str) -> InterpolationError: + msg = f"The placeholder {attribute!r} is unknown." + if value and isinstance(value, InterpolationValue): + msg += f' Try with one of: {", ".join(value.keys())}.' + return cls(msg) + + +ErrorT = TypeVar("ErrorT", bound=InterpolationError) + + +class InterpolationValue(FrozenDict[str, str]): + """Dict class suitable for use as a format string context object, as it allows to use attribute + access rather than item access.""" + + _attribute_error_type: ClassVar[type[InterpolationError]] = InterpolationError + + def __getattr__(self, attribute: str) -> str: + if attribute not in self: + raise self._attribute_error_type.attribute_error(self, attribute) + return self[attribute] + + +class InterpolationContext(FrozenDict[str, Union[str, InterpolationValue]]): + @classmethod + def from_dict(cls, data: Mapping[str, str | Mapping[str, str]]) -> InterpolationContext: + return InterpolationContext({key: cls.create_value(value) for key, value in data.items()}) + + @staticmethod + def create_value(value: str | Mapping[str, str]) -> str | InterpolationValue: + """Ensure that `value` satisfies the type `InterpolationValue`.""" + if isinstance(value, (str, InterpolationValue)): + return value + return InterpolationValue(value) + + def merge(self, other: Mapping[str, str | Mapping[str, str]]) -> InterpolationContext: + return InterpolationContext.from_dict({**self, **other}) + + def format( + self, text: str, *, source: TextSource, error_cls: type[ErrorT] | None = None + ) -> str: + stack = [text] + try: + while "{" in stack[-1] and "}" in stack[-1]: + if len(stack) >= 5: + raise InterpolationError( + "The formatted placeholders recurse too deep.\n" + + " => ".join(map(repr, stack)) + ) + stack.append(stack[-1].format(**self)) + if stack[-1] == stack[-2]: + break + return stack[-1] + except (KeyError, InterpolationError) as e: + default_error_cls = InterpolationError + msg = f"Invalid value for the {source}: {text!r}.\n\n" + if isinstance(e, InterpolationError): + default_error_cls = type(e) + msg += str(e) + else: + # KeyError + msg += f"The placeholder {e} is unknown." + if self: + msg += f" Try with one of: {', '.join(sorted(self.keys()))}." + else: + msg += ( + " There are currently no known placeholders to use. These placeholders " + "can come from `[docker].build_args` or parsed from instructions in your " + "`Dockerfile`." + ) + raise (error_cls or default_error_cls)(msg) from e + + @dataclass(frozen=True) + class TextSource: + address: Address | None = None + target_alias: str | None = None + field_alias: str | None = None + options_scope: str | None = None + + def __post_init__(self): + field_infos_is_none = ( + x is None for x in [self.address, self.target_alias, self.field_alias] + ) + if self.options_scope is None: + assert not any(field_infos_is_none), f"Missing target field details in {self!r}." + else: + assert all( + field_infos_is_none + ), f"Must not refer to both configuration option and target field in {self!r}." + + def __str__(self) -> str: + if self.options_scope: + return f"`{self.options_scope}` configuration option" + return ( + f"`{self.field_alias}` field of the `{self.target_alias}` target at {self.address}" + ) From bf7ab67d8049128c9c79bd65344de893a2b60ec4 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Mon, 22 Aug 2022 16:06:18 +0200 Subject: [PATCH 2/9] Test string interpolation during deployment --- .../backend/docker/value_interpolation.py | 6 +-- .../pants/backend/helm/goals/deploy_test.py | 39 ++++++++++++------- src/python/pants/backend/helm/target_types.py | 23 ++++------- .../pants/backend/helm/util_rules/tool.py | 2 +- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/python/pants/backend/docker/value_interpolation.py b/src/python/pants/backend/docker/value_interpolation.py index 9966f1aee3d..47f1ad83192 100644 --- a/src/python/pants/backend/docker/value_interpolation.py +++ b/src/python/pants/backend/docker/value_interpolation.py @@ -6,11 +6,11 @@ from pants.util.value_interpolation import InterpolationError, InterpolationValue -class DockerBuildArgInterpolationError(InterpolationError): +class DockerBuildArgsInterpolationError(InterpolationError): @classmethod def attribute_error( cls, value: str | DockerBuildArgsInterpolationValue, attribute: str - ) -> DockerBuildArgInterpolationError: + ) -> DockerBuildArgsInterpolationError: msg = f"The build arg {attribute!r} is undefined." if value and isinstance(value, DockerBuildArgsInterpolationValue): msg += f' Defined build args are: {", ".join(value.keys())}.' @@ -24,4 +24,4 @@ def attribute_error( class DockerBuildArgsInterpolationValue(InterpolationValue): """Interpolation context value with specific error handling for build args.""" - _attribute_error_type = DockerBuildArgInterpolationError + _attribute_error_type = DockerBuildArgsInterpolationError diff --git a/src/python/pants/backend/helm/goals/deploy_test.py b/src/python/pants/backend/helm/goals/deploy_test.py index b58912c7e5b..e51c4b08ace 100644 --- a/src/python/pants/backend/helm/goals/deploy_test.py +++ b/src/python/pants/backend/helm/goals/deploy_test.py @@ -4,7 +4,7 @@ from __future__ import annotations from textwrap import dedent -from typing import Iterable +from typing import Iterable, Mapping import pytest @@ -40,9 +40,14 @@ def rule_runner() -> RuleRunner: def _run_deployment( - rule_runner: RuleRunner, spec_path: str, target_name: str, *, args: Iterable[str] | None = None + rule_runner: RuleRunner, + spec_path: str, + target_name: str, + *, + args: Iterable[str] | None = None, + env: Mapping[str, str] | None = None, ) -> DeployProcess: - rule_runner.set_options(args or (), env_inherit=PYTHON_BOOTSTRAP_ENV) + rule_runner.set_options(args or (), env=env, env_inherit=PYTHON_BOOTSTRAP_ENV) target = rule_runner.get_target(Address(spec_path, target_name=target_name)) field_set = DeployHelmDeploymentFieldSet.create(target) @@ -70,6 +75,7 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None: "key": "foo", "amount": "300", "long_string": "This is a long string", + "build_number": "{env.BUILD_NUMBER}", }, timeout=150, ) @@ -96,18 +102,7 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None: } ) - deploy_args = ["--kubeconfig", "./kubeconfig"] - deploy_process = _run_deployment( - rule_runner, - "src/deployment", - "foo", - args=[ - f"--helm-args={repr(deploy_args)}", - ], - ) - - helm = rule_runner.request(HelmBinary, []) - + expected_build_number = "34" expected_value_files_order = [ "common.yaml", "bar.yaml", @@ -119,6 +114,18 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None: "subdir/last.yaml", ] + extra_env_vars = ["BUILD_NUMBER"] + deploy_args = ["--kubeconfig", "./kubeconfig"] + deploy_process = _run_deployment( + rule_runner, + "src/deployment", + "foo", + args=[f"--helm-args={repr(deploy_args)}", f"--helm-extra-env-vars={repr(extra_env_vars)}"], + env={"BUILD_NUMBER": expected_build_number}, + ) + + helm = rule_runner.request(HelmBinary, []) + assert deploy_process.process assert deploy_process.process.process.argv == ( helm.path, @@ -144,6 +151,8 @@ def test_run_helm_deploy(rule_runner: RuleRunner) -> None: "amount=300", "--set", 'long_string="This is a long string"', + "--set", + f"build_number={expected_build_number}", "--install", "--timeout", "150s", diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index d20fcb08896..ede06c90f93 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -7,7 +7,6 @@ from dataclasses import dataclass from pants.backend.helm.resolve.remotes import ALL_DEFAULT_HELM_REGISTRIES -from pants.backend.helm.value_interpolation import HelmEnvironmentInterpolationError from pants.core.goals.package import OutputPathField from pants.core.goals.test import TestTimeoutField from pants.engine.rules import collect_rules, rule @@ -471,27 +470,19 @@ class HelmDeploymentFieldSet(FieldSet): values: HelmDeploymentValuesField def format_values(self, interpolation_context: InterpolationContext) -> dict[str, str]: - def format_value(text: str) -> str | None: + def format_value(text: str) -> str: source = InterpolationContext.TextSource( self.address, target_alias=HelmDeploymentTarget.alias, field_alias=HelmDeploymentValuesField.alias, ) - try: - return interpolation_context.format( - text, source=source, error_cls=HelmEnvironmentInterpolationError - ) - except HelmEnvironmentInterpolationError as err: - logger.warning(err) - return None - - result = {} - for key, value in (self.values.value or {}).items(): - interpolated_value = format_value(value) - if interpolated_value is not None: - result[key] = interpolated_value - return result + return interpolation_context.format( + text, + source=source, + ) + + return {key: format_value(value) for key, value in (self.values.value or {}).items()} class AllHelmDeploymentTargets(Targets): diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index 9bc423ea4ad..8cdfd3235ee 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -27,6 +27,7 @@ from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.environment import Environment, EnvironmentRequest from pants.engine.fs import ( + EMPTY_DIGEST, AddPrefix, CreateDigest, Digest, @@ -39,7 +40,6 @@ RemovePrefix, Snapshot, ) -from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, MergeDigests, Snapshot from pants.engine.platform import Platform from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import Get, MultiGet, collect_rules, rule From 26d45632a654f659e4136d79aa0657aebc4dc238 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Mon, 22 Aug 2022 16:10:05 +0200 Subject: [PATCH 3/9] Fix typing errors --- src/python/pants/backend/docker/value_interpolation.py | 2 +- src/python/pants/backend/helm/value_interpolation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/docker/value_interpolation.py b/src/python/pants/backend/docker/value_interpolation.py index 47f1ad83192..981a9b97e73 100644 --- a/src/python/pants/backend/docker/value_interpolation.py +++ b/src/python/pants/backend/docker/value_interpolation.py @@ -9,7 +9,7 @@ class DockerBuildArgsInterpolationError(InterpolationError): @classmethod def attribute_error( - cls, value: str | DockerBuildArgsInterpolationValue, attribute: str + cls, value: str | InterpolationValue, attribute: str ) -> DockerBuildArgsInterpolationError: msg = f"The build arg {attribute!r} is undefined." if value and isinstance(value, DockerBuildArgsInterpolationValue): diff --git a/src/python/pants/backend/helm/value_interpolation.py b/src/python/pants/backend/helm/value_interpolation.py index b68265f0feb..f6e6dfd0b0b 100644 --- a/src/python/pants/backend/helm/value_interpolation.py +++ b/src/python/pants/backend/helm/value_interpolation.py @@ -9,7 +9,7 @@ class HelmEnvironmentInterpolationError(InterpolationError): @classmethod def attribute_error( - cls, value: str | HelmEnvironmentInterpolationValue, attribute: str + cls, value: str | InterpolationValue, attribute: str ) -> HelmEnvironmentInterpolationError: msg = f"The environment variable {attribute!r} is undefined." if value and isinstance(value, HelmEnvironmentInterpolationValue): From 695530fa75cc70b0a4fe20a27f3c0240eab1303e Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Mon, 22 Aug 2022 16:11:25 +0200 Subject: [PATCH 4/9] Do not allow to override the default Helm binary env vars --- src/python/pants/backend/helm/util_rules/tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index 8cdfd3235ee..a826ab542ab 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -440,7 +440,7 @@ async def helm_process( request: HelmProcess, helm_binary: HelmBinary, helm_subsytem: HelmSubsystem ) -> Process: global_extra_env = await Get(Environment, EnvironmentRequest(helm_subsytem.extra_env_vars)) - env = {**global_extra_env, **helm_binary.env, **request.extra_env} + env = {**global_extra_env, **request.extra_env, **helm_binary.env} immutable_input_digests = { **helm_binary.immutable_input_digests, From 1a44305d1f43ace271425cbeedd950b9c3551677 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Tue, 23 Aug 2022 09:18:07 +0200 Subject: [PATCH 5/9] Fix order of keys in immutable digest maps --- src/python/pants/backend/helm/util_rules/tool.py | 9 +++++---- src/python/pants/backend/helm/util_rules/tool_test.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index a826ab542ab..70f5542228a 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -440,14 +440,15 @@ async def helm_process( request: HelmProcess, helm_binary: HelmBinary, helm_subsytem: HelmSubsystem ) -> Process: global_extra_env = await Get(Environment, EnvironmentRequest(helm_subsytem.extra_env_vars)) - env = {**global_extra_env, **request.extra_env, **helm_binary.env} + # Helm binary's setup parameters go last to prevent end users overriding any of its values. + + env = {**global_extra_env, **request.extra_env, **helm_binary.env} immutable_input_digests = { - **helm_binary.immutable_input_digests, **request.extra_immutable_input_digests, + **helm_binary.immutable_input_digests, } - - append_only_caches = {**helm_binary.append_only_caches, **request.extra_append_only_caches} + append_only_caches = {**request.extra_append_only_caches, **helm_binary.append_only_caches} return Process( [helm_binary.path, *request.argv], diff --git a/src/python/pants/backend/helm/util_rules/tool_test.py b/src/python/pants/backend/helm/util_rules/tool_test.py index 422a71be5fa..a1e153617f1 100644 --- a/src/python/pants/backend/helm/util_rules/tool_test.py +++ b/src/python/pants/backend/helm/util_rules/tool_test.py @@ -58,9 +58,9 @@ def test_create_helm_process(rule_runner: RuleRunner) -> None: assert process.level == helm_process.level assert process.input_digest == helm_process.input_digest assert process.immutable_input_digests == FrozenDict( - {**helm_binary.immutable_input_digests, **helm_process.extra_immutable_input_digests} + {**helm_process.extra_immutable_input_digests, **helm_binary.immutable_input_digests} ) - assert process.env == FrozenDict({**helm_binary.env, **helm_process.extra_env}) + assert process.env == FrozenDict({**helm_process.extra_env, **helm_binary.env}) assert process.output_directories == helm_process.output_directories assert process.cache_scope == helm_process.cache_scope assert process.timeout_seconds == helm_process.timeout_seconds From a005a5ba52409177da51bacf9e5e6f9eb4722535 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Tue, 23 Aug 2022 11:31:30 +0200 Subject: [PATCH 6/9] Add documentation about interpolated values --- docs/markdown/Helm/helm-deployments.md | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/markdown/Helm/helm-deployments.md b/docs/markdown/Helm/helm-deployments.md index 637fc1e6392..b6752727fa7 100644 --- a/docs/markdown/Helm/helm-deployments.md +++ b/docs/markdown/Helm/helm-deployments.md @@ -176,8 +176,48 @@ src/deployment/last.yaml We believe that this approach gives a very consistent and predictable ordering while at the same time total flexibility to the end user to organise their files as they best fit each particular case of a deployment. +Inline values +------------- + In addition to value files, you can also use inline values in your `helm_deployment` targets by means of the `values` field. All inlines values that are set this way will override any entry that may come from value files. +Inline values are defined as a key-value dictionary, like in the following example: + +```python src/deployment/BUILD +helm_deployment( + name="dev", + dependencies=["//src/chart"], + values={ + "nameOverride": "my_custom_name", + "image.pullPolicy": "Always", + }, +) +``` + +### Using interpolated values + +Inline values also support interpolation of environment variables. Since Pants runs all processes in a hermetic sandbox, to be able to use environment variables you must first tell Pants what variables to make available to the Helm process using the `[helm].extra_env_vars` option. Consider the following example: + +```python src/deployment/BUILD +helm_deployment( + name="dev", + dependencies=["//src/chart"], + values={ + "configmap.deployedAt": "{env.DEPLOY_TIME}", + }, +) +``` +```toml pants.toml +[helm] +extra_env_vars = ["DEPLOY_TIME"] +``` + +Now you can launch a deployment using the following command: + +``` +DEPLOY_TIME=$(date) ./pants experimental-deploy src/deployment:dev +``` + Third party chart artifacts --------------------------- From e46bf835bca379d6e12232ffc7fbf9f1c6e9e704 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Tue, 23 Aug 2022 12:01:36 +0200 Subject: [PATCH 7/9] Ignore missing keys in interpolated context when only rendering the deployment --- docs/markdown/Helm/helm-deployments.md | 7 +++- src/python/pants/backend/helm/BUILD | 2 +- .../backend/helm/dependency_inference/BUILD | 2 +- src/python/pants/backend/helm/resolve/BUILD | 2 +- src/python/pants/backend/helm/target_types.py | 31 ++++++++++++------ .../pants/backend/helm/util_rules/renderer.py | 4 ++- .../backend/helm/util_rules/renderer_test.py | 32 +++++++++++++++++++ src/python/pants/backend/helm/utils/BUILD | 2 +- 8 files changed, 67 insertions(+), 15 deletions(-) diff --git a/docs/markdown/Helm/helm-deployments.md b/docs/markdown/Helm/helm-deployments.md index b6752727fa7..1f160ec7582 100644 --- a/docs/markdown/Helm/helm-deployments.md +++ b/docs/markdown/Helm/helm-deployments.md @@ -194,7 +194,7 @@ helm_deployment( ) ``` -### Using interpolated values +### Using dynamic values Inline values also support interpolation of environment variables. Since Pants runs all processes in a hermetic sandbox, to be able to use environment variables you must first tell Pants what variables to make available to the Helm process using the `[helm].extra_env_vars` option. Consider the following example: @@ -218,6 +218,11 @@ Now you can launch a deployment using the following command: DEPLOY_TIME=$(date) ./pants experimental-deploy src/deployment:dev ``` +> 🚧 Ensuring repeatable deployments +> +> You should always favor using static values (or value files) VS dynamic values in your deployments. Using interpolated environment variables in your deployments can render your deployments non-repetable anymore if those values can affect the behaviour of the system deployed, or what gets deployed (i.e. Docker image addresses). +> Dynamic values are supported to give the option of passing some info or metadata to the software being deployed (i.e. deploy time, commit hash, etc) or some less harmful settings of a deployment (i.e. replica count. etc). Be careful when chossing the values that are going to be calculated dynamically. + Third party chart artifacts --------------------------- diff --git a/src/python/pants/backend/helm/BUILD b/src/python/pants/backend/helm/BUILD index f9b4a7be9f0..27e5628650a 100644 --- a/src/python/pants/backend/helm/BUILD +++ b/src/python/pants/backend/helm/BUILD @@ -3,4 +3,4 @@ python_sources() -python_tests(name="tests") \ No newline at end of file +python_tests(name="tests") diff --git a/src/python/pants/backend/helm/dependency_inference/BUILD b/src/python/pants/backend/helm/dependency_inference/BUILD index a526ceeec22..8bd8c247152 100644 --- a/src/python/pants/backend/helm/dependency_inference/BUILD +++ b/src/python/pants/backend/helm/dependency_inference/BUILD @@ -3,4 +3,4 @@ python_sources() -python_tests(name="tests", overrides={"deployment_test.py": {"timeout": 120}}) \ No newline at end of file +python_tests(name="tests", overrides={"deployment_test.py": {"timeout": 120}}) diff --git a/src/python/pants/backend/helm/resolve/BUILD b/src/python/pants/backend/helm/resolve/BUILD index f9b4a7be9f0..27e5628650a 100644 --- a/src/python/pants/backend/helm/resolve/BUILD +++ b/src/python/pants/backend/helm/resolve/BUILD @@ -3,4 +3,4 @@ python_sources() -python_tests(name="tests") \ No newline at end of file +python_tests(name="tests") diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index ede06c90f93..ab778178764 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -34,7 +34,7 @@ ) from pants.util.docutil import bin_name from pants.util.strutil import softwrap -from pants.util.value_interpolation import InterpolationContext +from pants.util.value_interpolation import InterpolationContext, InterpolationError logger = logging.getLogger(__name__) @@ -469,20 +469,33 @@ class HelmDeploymentFieldSet(FieldSet): dependencies: HelmDeploymentDependenciesField values: HelmDeploymentValuesField - def format_values(self, interpolation_context: InterpolationContext) -> dict[str, str]: - def format_value(text: str) -> str: + def format_values( + self, interpolation_context: InterpolationContext, *, ignore_missing: bool = False + ) -> dict[str, str]: + def format_value(text: str) -> str | None: source = InterpolationContext.TextSource( self.address, target_alias=HelmDeploymentTarget.alias, field_alias=HelmDeploymentValuesField.alias, ) - return interpolation_context.format( - text, - source=source, - ) - - return {key: format_value(value) for key, value in (self.values.value or {}).items()} + try: + return interpolation_context.format( + text, + source=source, + ) + except InterpolationError as err: + if ignore_missing: + return None + raise err + + result = {} + for key, value in (self.values.value or {}).items(): + formatted_value = format_value(value) + if formatted_value is not None: + result[key] = formatted_value + + return result class AllHelmDeploymentTargets(Targets): diff --git a/src/python/pants/backend/helm/util_rules/renderer.py b/src/python/pants/backend/helm/util_rules/renderer.py index 77aeae5fd82..1b53b4fbb85 100644 --- a/src/python/pants/backend/helm/util_rules/renderer.py +++ b/src/python/pants/backend/helm/util_rules/renderer.py @@ -302,7 +302,9 @@ async def setup_render_helm_deployment_process( request.field_set.release_name.value or request.field_set.address.target_name.replace("_", "-") ) - inline_values = request.field_set.format_values(interpolation_context) + inline_values = request.field_set.format_values( + interpolation_context, ignore_missing=request.cmd == HelmDeploymentCmd.RENDER + ) def maybe_escape_string_value(value: str) -> str: if re.findall("\\s+", value): diff --git a/src/python/pants/backend/helm/util_rules/renderer_test.py b/src/python/pants/backend/helm/util_rules/renderer_test.py index e848e497993..c44bee19b92 100644 --- a/src/python/pants/backend/helm/util_rules/renderer_test.py +++ b/src/python/pants/backend/helm/util_rules/renderer_test.py @@ -198,3 +198,35 @@ def test_renders_files(rule_runner: RuleRunner) -> None: filename="mychart/templates/configmap.yaml", ) assert template_output == expected_config_map_file + + +def test_ignore_missing_interpolated_keys_during_render(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + **_common_workspace_files(), + "src/deployment/BUILD": dedent( + """\ + helm_deployment( + name='foo', + dependencies=['//src/mychart'], + values={"foo": "{env.foo}"}, + ) + """ + ), + } + ) + + tgt = rule_runner.get_target(Address("src/deployment", target_name="foo")) + field_set = HelmDeploymentFieldSet.create(tgt) + + rendered = rule_runner.request( + RenderedHelmFiles, + [ + HelmDeploymentRequest( + field_set, + cmd=HelmDeploymentCmd.RENDER, + description="Test ignore missing interpolated values", + ) + ], + ) + assert rendered.snapshot.files diff --git a/src/python/pants/backend/helm/utils/BUILD b/src/python/pants/backend/helm/utils/BUILD index f9b4a7be9f0..27e5628650a 100644 --- a/src/python/pants/backend/helm/utils/BUILD +++ b/src/python/pants/backend/helm/utils/BUILD @@ -3,4 +3,4 @@ python_sources() -python_tests(name="tests") \ No newline at end of file +python_tests(name="tests") From fafc4c2fa518e707efaf3f22f2bf488246c30f10 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Wed, 24 Aug 2022 10:52:56 +0200 Subject: [PATCH 8/9] Reuse the `source` instance for all formatted values --- src/python/pants/backend/helm/target_types.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index ab778178764..accd9b6ef91 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -472,13 +472,13 @@ class HelmDeploymentFieldSet(FieldSet): def format_values( self, interpolation_context: InterpolationContext, *, ignore_missing: bool = False ) -> dict[str, str]: - def format_value(text: str) -> str | None: - source = InterpolationContext.TextSource( - self.address, - target_alias=HelmDeploymentTarget.alias, - field_alias=HelmDeploymentValuesField.alias, - ) + source = InterpolationContext.TextSource( + self.address, + target_alias=HelmDeploymentTarget.alias, + field_alias=HelmDeploymentValuesField.alias, + ) + def format_value(text: str) -> str | None: try: return interpolation_context.format( text, From f0f76972387d148d062a2717d131a60de73abe35 Mon Sep 17 00:00:00 2001 From: Antonio Alonso Dominguez Date: Wed, 24 Aug 2022 11:13:31 +0200 Subject: [PATCH 9/9] Amend error message on InterpolationContext --- src/python/pants/backend/helm/target_types.py | 30 ++++++++++++++++++- src/python/pants/util/value_interpolation.py | 13 +++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/helm/target_types.py b/src/python/pants/backend/helm/target_types.py index accd9b6ef91..bbc5b39c28c 100644 --- a/src/python/pants/backend/helm/target_types.py +++ b/src/python/pants/backend/helm/target_types.py @@ -413,7 +413,35 @@ class HelmDeploymentSourcesField(MultipleSourcesField): class HelmDeploymentValuesField(DictStringToStringField): alias = "values" required = False - help = "Individual values to use when rendering a given deployment." + help = softwrap( + """ + Individual values to use when rendering a given deployment. + + Value names should be defined using dot-syntax as in the following example: + + ``` + helm_deployment( + values={ + "nameOverride": "my_custom_name", + "image.pullPolicy": "Always", + }, + ) + ``` + + Values can be dynamically calculated using interpolation as shown in the following example: + + ``` + helm_deployment( + values={ + "configmap.deployedAt": "{env.DEPLOY_TIME}", + }, + ) + ``` + + Check the Helm backend documentation on what are the options available and its caveats when making + usage of dynamic values in your deployments. + """ + ) class HelmDeploymentCreateNamespaceField(BoolField): diff --git a/src/python/pants/util/value_interpolation.py b/src/python/pants/util/value_interpolation.py index 9168fd17cd3..ed1dd074fab 100644 --- a/src/python/pants/util/value_interpolation.py +++ b/src/python/pants/util/value_interpolation.py @@ -8,6 +8,7 @@ from pants.engine.addresses import Address from pants.util.frozendict import FrozenDict +from pants.util.strutil import softwrap class InterpolationError(ValueError): @@ -76,10 +77,14 @@ def format( if self: msg += f" Try with one of: {', '.join(sorted(self.keys()))}." else: - msg += ( - " There are currently no known placeholders to use. These placeholders " - "can come from `[docker].build_args` or parsed from instructions in your " - "`Dockerfile`." + msg += " " + msg += softwrap( + f""" + There are currently no known placeholders to use. + + Check the documentation of the {source} to understand where you may need + to configure your placeholders. + """ ) raise (error_cls or default_error_cls)(msg) from e