diff --git a/src/python/pants/backend/helm/goals/lint.py b/src/python/pants/backend/helm/goals/lint.py index a091f1603bbd..15bd9a92608d 100644 --- a/src/python/pants/backend/helm/goals/lint.py +++ b/src/python/pants/backend/helm/goals/lint.py @@ -55,7 +55,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess return HelmProcess( argv, input_digest=chart.snapshot.digest, - description=f"Linting chart: {chart.metadata.name}", + description=f"Linting chart: {chart.info.name}", ) process_results = await MultiGet( @@ -68,7 +68,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess ) results = [ LintResult.from_fallible_process_result( - process_result, partition_description=chart.metadata.name + process_result, partition_description=chart.info.name ) for chart, process_result in zip(charts, process_results) ] diff --git a/src/python/pants/backend/helm/goals/package.py b/src/python/pants/backend/helm/goals/package.py index af534a0a8aaf..85faf708556b 100644 --- a/src/python/pants/backend/helm/goals/package.py +++ b/src/python/pants/backend/helm/goals/package.py @@ -31,13 +31,13 @@ @dataclass(frozen=True) class BuiltHelmArtifact(BuiltPackageArtifact): - metadata: HelmChartMetadata | None = None + info: HelmChartMetadata | None = None @classmethod - def create(cls, relpath: str, metadata: HelmChartMetadata) -> BuiltHelmArtifact: + def create(cls, relpath: str, info: HelmChartMetadata) -> BuiltHelmArtifact: return cls( relpath=relpath, - metadata=metadata, + info=info, extra_log_lines=(f"Built Helm chart artifact: {relpath}",), ) @@ -57,7 +57,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage: ) input_digest = await Get(Digest, MergeDigests([chart.snapshot.digest, result_digest])) - process_output_file = os.path.join(result_dir, f"{chart.metadata.artifact_name}.tgz") + process_output_file = os.path.join(result_dir, f"{chart.info.artifact_name}.tgz") process_result = await Get( ProcessResult, @@ -80,7 +80,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage: return BuiltPackage( final_snapshot.digest, artifacts=tuple( - BuiltHelmArtifact.create(file, chart.metadata) for file in final_snapshot.files + BuiltHelmArtifact.create(file, chart.info) for file in final_snapshot.files ), ) diff --git a/src/python/pants/backend/helm/goals/package_test.py b/src/python/pants/backend/helm/goals/package_test.py index c1a99fa69809..87725008c4d1 100644 --- a/src/python/pants/backend/helm/goals/package_test.py +++ b/src/python/pants/backend/helm/goals/package_test.py @@ -59,7 +59,7 @@ def _assert_build_package(rule_runner: RuleRunner, *, chart_name: str, chart_ver assert result.artifacts[0].relpath == os.path.join( dest_dir, f"{chart_name}-{chart_version}.tgz" ) - assert result.artifacts[0].metadata + assert result.artifacts[0].info def test_helm_package(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/helm/goals/publish.py b/src/python/pants/backend/helm/goals/publish.py index b4307ebbda2e..0bbe486da11c 100644 --- a/src/python/pants/backend/helm/goals/publish.py +++ b/src/python/pants/backend/helm/goals/publish.py @@ -57,10 +57,10 @@ async def publish_helm_chart( ) -> PublishProcesses: remotes = helm_subsystem.remotes() built_artifacts = [ - (pkg, artifact, artifact.metadata) + (pkg, artifact, artifact.info) for pkg in request.packages for artifact in pkg.artifacts - if isinstance(artifact, BuiltHelmArtifact) and artifact.metadata + if isinstance(artifact, BuiltHelmArtifact) and artifact.info ] registries_to_push = list(remotes.get(*(request.field_set.registries.value or []))) diff --git a/src/python/pants/backend/helm/subsystems/BUILD b/src/python/pants/backend/helm/subsystems/BUILD index 0430fbb073df..5a6509dd50b9 100644 --- a/src/python/pants/backend/helm/subsystems/BUILD +++ b/src/python/pants/backend/helm/subsystems/BUILD @@ -5,7 +5,7 @@ python_requirement( name="yamlpath", requirements=[ "yamlpath>=3.6,<3.7", - "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17", + "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21", ], resolve="helm-post-renderer", ) diff --git a/src/python/pants/backend/helm/subsystems/post_renderer.lock b/src/python/pants/backend/helm/subsystems/post_renderer.lock index ea317f99358a..b5f1e4dc8c3c 100644 --- a/src/python/pants/backend/helm/subsystems/post_renderer.lock +++ b/src/python/pants/backend/helm/subsystems/post_renderer.lock @@ -9,7 +9,7 @@ // "CPython<3.10,>=3.7" // ], // "generated_with_requirements": [ -// "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17,>=0.15.96", +// "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96", // "yamlpath<3.7,>=3.6" // ] // } @@ -143,7 +143,7 @@ "platform_tag": [ "cp39", "cp39", - "macosx_12_0_x86_64" + "macosx_12_0_arm64" ] } ], @@ -151,7 +151,7 @@ "pex_version": "2.1.90", "prefer_older_binary": false, "requirements": [ - "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17,>=0.15.96", + "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96", "yamlpath<3.7,>=3.6" ], "requires_python": [ diff --git a/src/python/pants/backend/helm/subsystems/post_renderer.py b/src/python/pants/backend/helm/subsystems/post_renderer.py index aac1f7a88879..e0f1f5279650 100644 --- a/src/python/pants/backend/helm/subsystems/post_renderer.py +++ b/src/python/pants/backend/helm/subsystems/post_renderer.py @@ -3,6 +3,7 @@ from __future__ import annotations +import logging import os import pkgutil from dataclasses import dataclass @@ -19,6 +20,7 @@ from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.util_rules.system_binaries import CatBinary +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.internals.native_engine import MergeDigests from pants.engine.process import Process @@ -28,6 +30,8 @@ from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel +logger = logging.getLogger(__name__) + _HELM_POSTRENDERER_SOURCE = "post_renderer_launcher.py" _HELM_POSTRENDERER_PACKAGE = "pants.backend.helm.subsystems" @@ -38,7 +42,7 @@ class HelmPostRenderer(PythonToolRequirementsBase): default_version = "yamlpath>=3.6,<3.7" default_extra_requirements = [ - "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17" + "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21" ] register_interpreter_constraints = True @@ -73,7 +77,7 @@ class HelmPostRendererTool: pex: VenvPex -@rule(desc="Prepare Helm post renderer", level=LogLevel.DEBUG) +@rule(desc="Setup Helm post renderer binaries", level=LogLevel.DEBUG) async def setup_internal_post_renderer( post_renderer: HelmPostRenderer, ) -> HelmPostRendererTool: @@ -103,20 +107,31 @@ async def setup_internal_post_renderer( @dataclass(frozen=True) -class SetupHelmPostRenderer: +class SetupHelmPostRenderer(EngineAwareParameter): replacements: YamlElements[str] + description_of_origin: str + + def debug_hint(self) -> str | None: + return self.description_of_origin @dataclass(frozen=True) -class HelmPostRendererRunnable: +class HelmPostRendererRunnable(EngineAwareReturnType): exe: str digest: Digest immutable_input_digests: FrozenDict[str, Digest] env: FrozenDict[str, str] append_only_caches: FrozenDict[str, str] + description_of_origin: str + + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + def message(self) -> str | None: + return f"runnable {self.exe} for {self.description_of_origin} is ready." -@rule(desc="Configure Helm Post Renderer", level=LogLevel.DEBUG) + +@rule(desc="Configure Helm post-renderer", level=LogLevel.DEBUG) async def setup_post_renderer_launcher( request: SetupHelmPostRenderer, post_renderer_tool: HelmPostRendererTool, @@ -137,17 +152,21 @@ async def setup_post_renderer_launcher( ), ) + post_renderer_cfg_file = os.path.join(".", HELM_POST_RENDERER_CFG_FILENAME) post_renderer_stdin_file = os.path.join(".", "__stdin.yaml") post_renderer_process = await Get( Process, VenvPexProcess( post_renderer_tool.pex, - argv=[os.path.join(".", HELM_POST_RENDERER_CFG_FILENAME), post_renderer_stdin_file], + argv=[post_renderer_cfg_file, post_renderer_stdin_file], input_digest=post_renderer_cfg_digest, description="", ), ) + post_renderer_process_cli = " ".join(post_renderer_process.argv) + logger.debug(f"Built post-renderer process CLI: {post_renderer_process_cli}") + postrenderer_wrapper_script = dedent( f"""\ #!/bin/bash @@ -155,7 +174,7 @@ async def setup_post_renderer_launcher( # Output stdin into a file in disk {cat_binary.path} <&0 > {post_renderer_stdin_file} - {' '.join(post_renderer_process.argv)} + {post_renderer_process_cli} """ ) wrapper_digest = await Get( @@ -180,6 +199,7 @@ async def setup_post_renderer_launcher( env=post_renderer_process.env, append_only_caches=post_renderer_process.append_only_caches, immutable_input_digests=post_renderer_process.immutable_input_digests, + description_of_origin=request.description_of_origin, ) diff --git a/src/python/pants/backend/helm/subsystems/post_renderer_test.py b/src/python/pants/backend/helm/subsystems/post_renderer_test.py index cb079416e1d1..c4f6e282b007 100644 --- a/src/python/pants/backend/helm/subsystems/post_renderer_test.py +++ b/src/python/pants/backend/helm/subsystems/post_renderer_test.py @@ -49,7 +49,12 @@ def test_post_renderer_is_runnable(rule_runner: RuleRunner) -> None: ) post_renderer_setup = rule_runner.request( - HelmPostRendererRunnable, [SetupHelmPostRenderer(replacements)] + HelmPostRendererRunnable, + [ + SetupHelmPostRenderer( + replacements, description_of_origin="test_post_renderer_is_runnable" + ) + ], ) assert post_renderer_setup.exe == "post_renderer_wrapper.sh" diff --git a/src/python/pants/backend/helm/util_rules/chart.py b/src/python/pants/backend/helm/util_rules/chart.py index 701ac68382c9..62ed90bafb04 100644 --- a/src/python/pants/backend/helm/util_rules/chart.py +++ b/src/python/pants/backend/helm/util_rules/chart.py @@ -30,6 +30,7 @@ ) from pants.backend.helm.util_rules.sources import HelmChartSourceFiles, HelmChartSourceFilesRequest from pants.engine.addresses import Address, Addresses +from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import ( EMPTY_DIGEST, AddPrefix, @@ -43,7 +44,7 @@ from pants.engine.target import DependenciesRequest, ExplicitlyProvidedDependencies, Target, Targets from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet -from pants.util.strutil import pluralize +from pants.util.strutil import pluralize, softwrap logger = logging.getLogger(__name__) @@ -56,17 +57,17 @@ def __init__(self, target: Target) -> None: @dataclass(frozen=True) class HelmChart: address: Address - metadata: HelmChartMetadata + info: HelmChartMetadata snapshot: Snapshot artifact: ResolvedHelmArtifact | None = None @property def path(self) -> str: - return self.metadata.name + return self.info.name @dataclass(frozen=True) -class HelmChartRequest: +class HelmChartRequest(EngineAwareParameter): field_set: HelmChartFieldSet @classmethod @@ -75,6 +76,9 @@ def from_target(cls, target: Target) -> HelmChartRequest: raise InvalidHelmChartTarget(target) return cls(HelmChartFieldSet.create(target)) + def debug_hint(self) -> str | None: + return self.field_set.address.spec + @rule async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> HelmChart: @@ -96,7 +100,7 @@ async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> H @rule(desc="Collect all source code and subcharts of a Helm Chart", level=LogLevel.DEBUG) async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> HelmChart: - dependencies, source_files, metadata = await MultiGet( + dependencies, source_files, chart_info = await MultiGet( Get(Targets, DependenciesRequest(request.field_set.dependencies)), Get( HelmChartSourceFiles, @@ -132,7 +136,12 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> subcharts_digest = EMPTY_DIGEST if subcharts: logger.debug( - f"Found {pluralize(len(subcharts), 'subchart')} as direct dependencies on Helm chart at: {request.field_set.address}" + softwrap( + f""" + Found {pluralize(len(subcharts), 'subchart')} as direct dependencies + on Helm chart at: {request.field_set.address}. + """ + ) ) merged_subcharts = await Get( @@ -142,9 +151,9 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> # Update subchart dependencies in the metadata and re-render it. remotes = subsystem.remotes() - subchart_map: dict[str, HelmChart] = {chart.metadata.name: chart for chart in subcharts} + subchart_map: dict[str, HelmChart] = {chart.info.name: chart for chart in subcharts} updated_dependencies: OrderedSet[HelmChartDependency] = OrderedSet() - for dep in metadata.dependencies: + for dep in chart_info.dependencies: updated_dep = dep if not dep.repository and remotes.default_registry: @@ -157,7 +166,7 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> if dep.name in subchart_map: updated_dep = dataclasses.replace( - updated_dep, version=subchart_map[dep.name].metadata.version + updated_dep, version=subchart_map[dep.name].info.version ) updated_dependencies.add(updated_dep) @@ -165,7 +174,7 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> # Include the explicitly provided subchats in the set of dependencies if not already present. updated_dependencies_names = {dep.name for dep in updated_dependencies} remaining_subcharts = [ - chart for chart in subcharts if chart.metadata.name not in updated_dependencies_names + chart for chart in subcharts if chart.info.name not in updated_dependencies_names ] for chart in remaining_subcharts: if chart.artifact: @@ -176,16 +185,16 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> ) else: dependency = HelmChartDependency( - name=chart.metadata.name, version=chart.metadata.version + name=chart.info.name, version=chart.info.version ) updated_dependencies.add(dependency) # Update metadata with the information about charts' dependencies. - metadata = dataclasses.replace(metadata, dependencies=tuple(updated_dependencies)) + chart_info = dataclasses.replace(chart_info, dependencies=tuple(updated_dependencies)) # Re-render the Chart.yaml file with the updated dependencies. metadata_digest, sources_without_metadata = await MultiGet( - Get(Digest, HelmChartMetadata, metadata), + Get(Digest, HelmChartMetadata, chart_info), Get( Digest, DigestSubset( @@ -202,8 +211,8 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> Digest, MergeDigests([metadata_digest, sources_without_metadata, subcharts_digest]) ) - chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, metadata.name)) - return HelmChart(address=request.field_set.address, metadata=metadata, snapshot=chart_snapshot) + chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, chart_info.name)) + return HelmChart(address=request.field_set.address, info=chart_info, snapshot=chart_snapshot) class MissingHelmDeploymentChartError(ValueError): @@ -222,11 +231,14 @@ def __init__(self, address: Address) -> None: @dataclass(frozen=True) -class FindHelmDeploymentChart: +class FindHelmDeploymentChart(EngineAwareParameter): field_set: HelmDeploymentFieldSet + def debug_hint(self) -> str | None: + return self.field_set.address.spec -@rule + +@rule(desc="Find Helm deployment's chart", level=LogLevel.DEBUG) async def find_chart_for_deployment(request: FindHelmDeploymentChart) -> HelmChartRequest: explicit_dependencies = await Get( ExplicitlyProvidedDependencies, DependenciesRequest(request.field_set.dependencies) diff --git a/src/python/pants/backend/helm/util_rules/chart_test.py b/src/python/pants/backend/helm/util_rules/chart_test.py index 836f9fdb83c5..fbe24c706c6e 100644 --- a/src/python/pants/backend/helm/util_rules/chart_test.py +++ b/src/python/pants/backend/helm/util_rules/chart_test.py @@ -86,7 +86,7 @@ def test_collects_single_chart_sources( helm_chart = rule_runner.request(HelmChart, [HelmChartRequest.from_target(tgt)]) assert not helm_chart.artifact - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert len(helm_chart.snapshot.files) == 4 assert helm_chart.address == address @@ -127,9 +127,9 @@ def test_gathers_local_subchart_sources_using_explicit_dependency(rule_runner: R assert "chart2/charts/chart1" in helm_chart.snapshot.dirs assert "chart2/charts/chart1/templates/service.yaml" in helm_chart.snapshot.files - assert len(helm_chart.metadata.dependencies) == 1 - assert helm_chart.metadata.dependencies[0].name == "chart1" - assert helm_chart.metadata.dependencies[0].alias == "foo" + assert len(helm_chart.info.dependencies) == 1 + assert helm_chart.info.dependencies[0].name == "chart1" + assert helm_chart.info.dependencies[0].alias == "foo" def test_gathers_all_subchart_sources_inferring_dependencies(rule_runner: RuleRunner) -> None: @@ -198,7 +198,7 @@ def test_gathers_all_subchart_sources_inferring_dependencies(rule_runner: RuleRu target = rule_runner.get_target(Address("src/chart2", target_name="chart2")) helm_chart = rule_runner.request(HelmChart, [HelmChartRequest.from_target(target)]) - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert "chart2/charts/chart1" in helm_chart.snapshot.dirs assert "chart2/charts/chart1/templates/service.yaml" in helm_chart.snapshot.files assert "chart2/charts/cert-manager" in helm_chart.snapshot.dirs @@ -276,7 +276,7 @@ def test_chart_metadata_is_updated_with_explicit_dependencies(rule_runner: RuleR ], ) - assert helm_chart.metadata == expected_metadata + assert helm_chart.info == expected_metadata assert new_metadata == expected_metadata @@ -301,8 +301,8 @@ def test_obtain_chart_from_deployment(rule_runner: RuleRunner) -> None: chart = rule_runner.request(HelmChart, [FindHelmDeploymentChart(field_set)]) - assert chart.metadata.name == "foo" - assert chart.metadata.version == "1.0.0" + assert chart.info.name == "foo" + assert chart.info.version == "1.0.0" def test_fail_when_no_chart_dependency_is_found(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/helm/util_rules/manifest.py b/src/python/pants/backend/helm/util_rules/manifest.py index ae3193ee7aeb..a132fa7a2412 100644 --- a/src/python/pants/backend/helm/util_rules/manifest.py +++ b/src/python/pants/backend/helm/util_rules/manifest.py @@ -13,9 +13,11 @@ from pants.backend.helm.util_rules.yaml_utils import YamlElement, YamlPath from pants.engine.collection import Collection +from pants.engine.engine_aware import EngineAwareParameter from pants.engine.fs import Digest, DigestContents, DigestSubset, PathGlobs from pants.engine.rules import Get, collect_rules, rule -from pants.util.strutil import pluralize +from pants.util.logging import LogLevel +from pants.util.strutil import pluralize, softwrap logger = logging.getLogger(__name__) @@ -162,13 +164,16 @@ def all_containers(self) -> tuple[KubeContainer, ...]: @dataclass(frozen=True) class KubeManifest: filename: PurePath + document_index: int api_version: str kind: StandardKind | CustomResourceKind pod_spec: KubePodSpec | None @classmethod - def from_dict(cls, filename: PurePath, d: dict[str, Any]) -> KubeManifest: + def from_dict( + cls, filename: PurePath, d: dict[str, Any], *, document_index: int = 0 + ) -> KubeManifest: std_kind: StandardKind | None = None try: std_kind = StandardKind(d["kind"]) @@ -185,6 +190,7 @@ def from_dict(cls, filename: PurePath, d: dict[str, Any]) -> KubeManifest: api_version=d["apiVersion"], kind=std_kind or custom_kind, pod_spec=spec, + document_index=document_index, ) @property @@ -199,12 +205,15 @@ class KubeManifests(Collection[KubeManifest]): @dataclass(frozen=True) -class ParseKubeManifests: +class ParseKubeManifests(EngineAwareParameter): digest: Digest description_of_origin: str + def debug_hint(self) -> str | None: + return self.description_of_origin -@rule + +@rule(desc="Parsing Kubernetes manifests", level=LogLevel.DEBUG) async def parse_kubernetes_manifests(request: ParseKubeManifests) -> KubeManifests: yaml_subset = await Get( Digest, DigestSubset(request.digest, PathGlobs(["**/*.yaml", "**/*.yml"])) @@ -212,13 +221,18 @@ async def parse_kubernetes_manifests(request: ParseKubeManifests) -> KubeManifes digest_contents = await Get(DigestContents, Digest, yaml_subset) manifests = [ - KubeManifest.from_dict(PurePath(file.path), parsed_yaml) + KubeManifest.from_dict(PurePath(file.path), parsed_yaml, document_index=idx) for file in digest_contents - for parsed_yaml in yaml.safe_load_all(file.content) + for idx, parsed_yaml in enumerate(yaml.safe_load_all(file.content)) ] logger.debug( - f"Found {pluralize(len(manifests), 'manifest')} in {request.description_of_origin}" + softwrap( + f""" + Found {pluralize(len(manifests), 'manifest')} in + {pluralize(len(digest_contents), 'file')} at {request.description_of_origin}. + """ + ) ) return KubeManifests(manifests) diff --git a/src/python/pants/backend/helm/util_rules/post_renderer.py b/src/python/pants/backend/helm/util_rules/post_renderer.py index 04860d2de54c..707754d69c04 100644 --- a/src/python/pants/backend/helm/util_rules/post_renderer.py +++ b/src/python/pants/backend/helm/util_rules/post_renderer.py @@ -3,6 +3,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass from pants.backend.docker.goals.package_image import DockerFieldSet @@ -28,22 +29,39 @@ from pants.backend.helm.target_types import HelmDeploymentFieldSet from pants.backend.helm.util_rules.yaml_utils import YamlElements from pants.engine.addresses import Address, Addresses +from pants.engine.engine_aware import EngineAwareParameter from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import Targets +from pants.util.logging import LogLevel +from pants.util.strutil import bullet_list, softwrap + +logger = logging.getLogger(__name__) @dataclass(frozen=True) -class HelmDeploymentPostRendererRequest: +class HelmDeploymentPostRendererRequest(EngineAwareParameter): field_set: HelmDeploymentFieldSet + def debug_hint(self) -> str | None: + return self.field_set.address.spec + -@rule +@rule(desc="Prepare Helm deployment post-renderer", level=LogLevel.DEBUG) async def prepare_post_renderer_for_helm_deployment( request: HelmDeploymentPostRendererRequest, mappings: FirstPartyHelmDeploymentMappings, docker_options: DockerOptions, ) -> HelmPostRendererRunnable: docker_addresses = mappings.docker_images[request.field_set.address] + logger.debug( + softwrap( + f""" + Resolving Docker image references for targets: + + {bullet_list([addr.spec for addr in docker_addresses.values()])} + """ + ) + ) docker_contexts = await MultiGet( Get( DockerBuildContext, @@ -63,21 +81,36 @@ def resolve_docker_image_ref(address: Address, context: DockerBuildContext) -> s if not docker_field_sets: return None - result = None docker_field_set = docker_field_sets[0] image_refs = docker_field_set.image_refs( default_repository=docker_options.default_repository, registries=docker_options.registries(), interpolation_context=context.interpolation_context, ) - if image_refs: - result = image_refs[0] - return result + + # Choose first non-latest image reference found, or fallback to 'latest'. + found_ref: str | None = None + fallback_ref: str | None = None + for ref in image_refs: + if ref.endswith(":latest"): + fallback_ref = ref + else: + found_ref = ref + break + + resolved_ref = found_ref or fallback_ref + if resolved_ref: + logger.debug(f"Resolved Docker image ref '{resolved_ref}' for address {address}.") + else: + logger.warning(f"Could not resolve a valid image ref for Docker target {address}.") + + return resolved_ref docker_addr_ref_mapping = { addr: resolve_docker_image_ref(addr, ctx) for addr, ctx in zip(docker_addresses.values(), docker_contexts) } + replacements = YamlElements( { manifest: { @@ -89,7 +122,10 @@ def resolve_docker_image_ref(address: Address, context: DockerBuildContext) -> s } ) - return await Get(HelmPostRendererRunnable, SetupHelmPostRenderer(replacements)) + return await Get( + HelmPostRendererRunnable, + SetupHelmPostRenderer(replacements, description_of_origin=request.field_set.address.spec), + ) def rules(): diff --git a/src/python/pants/backend/helm/util_rules/renderer.py b/src/python/pants/backend/helm/util_rules/renderer.py index 00868039c98e..4948e58cba37 100644 --- a/src/python/pants/backend/helm/util_rules/renderer.py +++ b/src/python/pants/backend/helm/util_rules/renderer.py @@ -4,12 +4,14 @@ from __future__ import annotations import dataclasses +import logging import os +from collections import defaultdict from dataclasses import dataclass from enum import Enum from itertools import chain from pathlib import PurePath -from typing import Iterable, Mapping +from typing import Any, Iterable, Mapping from pants.backend.helm.subsystems import post_renderer from pants.backend.helm.subsystems.post_renderer import HelmPostRendererRunnable @@ -19,6 +21,8 @@ from pants.backend.helm.util_rules.tool import HelmProcess from pants.core.util_rules.source_files import SourceFilesRequest from pants.core.util_rules.stripped_source_files import StrippedSourceFiles +from pants.engine.addresses import Address +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.fs import ( EMPTY_DIGEST, EMPTY_SNAPSHOT, @@ -30,10 +34,14 @@ RemovePrefix, Snapshot, ) +from pants.engine.internals.native_engine import FileDigest from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.util.logging import LogLevel from pants.util.meta import frozen_after_init -from pants.util.strutil import softwrap +from pants.util.strutil import pluralize, softwrap + +logger = logging.getLogger(__name__) class HelmDeploymentRendererCmd(Enum): @@ -45,7 +53,7 @@ class HelmDeploymentRendererCmd(Enum): @dataclass(unsafe_hash=True) @frozen_after_init -class HelmDeploymentRendererRequest: +class HelmDeploymentRendererRequest(EngineAwareParameter): field_set: HelmDeploymentFieldSet cmd: HelmDeploymentRendererCmd @@ -83,18 +91,78 @@ def __init__( ) ) + def debug_hint(self) -> str | None: + return self.field_set.address.spec + + def metadata(self) -> dict[str, Any] | None: + return { + "cmd": self.cmd.value, + "description": self.description, + "extra_argv": self.extra_argv, + "output_directory": self.output_directory, + "post_renderer": True if self.post_renderer else False, + } + @dataclass(frozen=True) -class HelmDeploymentRenderer: +class HelmDeploymentRenderer(EngineAwareParameter, EngineAwareReturnType): + address: Address chart: HelmChart process: HelmProcess + post_renderer: bool output_directory: str | None + def debug_hint(self) -> str | None: + return self.address.spec + + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + + def message(self) -> str | None: + msg = softwrap( + f""" + Built renderer for {self.address} using chart {self.chart.address} + with{'out' if not self.post_renderer else ''} a post-renderer stage + """ + ) + if self.output_directory: + msg += f" and output directory: {self.output_directory}." + else: + msg += " and output to stdout." + return msg + + def metadata(self) -> dict[str, Any] | None: + return { + "chart": self.chart.address, + "helm_argv": self.process.argv, + "post_renderer": self.post_renderer, + "output_directory": self.output_directory, + } + @dataclass(frozen=True) -class RenderedFiles: +class RenderedFiles(EngineAwareReturnType): + address: Address + chart: HelmChart snapshot: Snapshot + def level(self) -> LogLevel | None: + return LogLevel.DEBUG + + def message(self) -> str | None: + return softwrap( + f""" + Generated {pluralize(len(self.snapshot.files), 'file')} from deployment {self.address} + using chart {self.chart}. + """ + ) + + def artifacts(self) -> dict[str, FileDigest | Snapshot] | None: + return {"content": self.snapshot} + + def metadata(self) -> dict[str, Any] | None: + return {"deployment": self.address, "chart": self.chart.address} + def _sort_value_file_names_for_evaluation(filenames: Iterable[str]) -> list[str]: """Breaks the list of files into two main buckets: overrides and non-overrides, and then sorts @@ -122,7 +190,7 @@ def by_path_length(p: PurePath) -> int: return [str(path) for path in [*non_overrides, *overrides]] -@rule +@rule(desc="Prepare Helm deployment renderer", level=LogLevel.DEBUG) async def setup_render_helm_deployment_process( request: HelmDeploymentRendererRequest, ) -> HelmDeploymentRenderer: @@ -131,24 +199,28 @@ async def setup_render_helm_deployment_process( Get(StrippedSourceFiles, SourceFilesRequest([request.field_set.sources])), ) + logger.debug(f"Using Helm chart {chart.address} in deployment {request.field_set.address}.") + output_digest = EMPTY_DIGEST if request.output_directory: output_digest = await Get(Digest, CreateDigest([Directory(request.output_directory)])) - # Ordering the value file names needs to be consistent so overrides are respected + # Ordering the value file names needs to be consistent so overrides are respected. sorted_value_files = _sort_value_file_names_for_evaluation(value_files.snapshot.files) - # Digests to be used as an input into the renderer process + # Digests to be used as an input into the renderer process. input_digests = [ chart.snapshot.digest, value_files.snapshot.digest, output_digest, ] + # Additional process values in case a post_renderer has been requested. env: Mapping[str, str] = {} immutable_input_digests: Mapping[str, Digest] = {} append_only_caches: Mapping[str, str] = {} if request.post_renderer: + logger.debug(f"Using post-renderer stage in deployment {request.field_set.address}") input_digests.append(request.post_renderer.digest) env = request.post_renderer.env immutable_input_digests = request.post_renderer.immutable_input_digests @@ -200,14 +272,18 @@ async def setup_render_helm_deployment_process( ) return HelmDeploymentRenderer( - chart=chart, process=process, output_directory=request.output_directory + address=request.field_set.address, + chart=chart, + process=process, + output_directory=request.output_directory, + post_renderer=True if request.post_renderer else False, ) _HELM_OUTPUT_FILE_MARKER = "# Source: " -@rule +@rule(desc="Run Helm deployment renderer", level=LogLevel.DEBUG) async def run_renderer(renderer: HelmDeploymentRenderer) -> RenderedFiles: def file_content(file_name: str, lines: Iterable[str]) -> FileContent: content = "\n".join(lines) + "\n" @@ -217,10 +293,9 @@ def file_content(file_name: str, lines: Iterable[str]) -> FileContent: def parse_renderer_output(result: ProcessResult) -> list[FileContent]: rendered_files_contents = result.stdout.decode("utf-8") - rendered_files: dict[str, list[str]] = {} + rendered_files: dict[str, list[str]] = defaultdict(list) curr_file_name = None - curr_file_lines: list[str] = [] for line in rendered_files_contents.splitlines(): if not line: continue @@ -231,25 +306,28 @@ def parse_renderer_output(result: ProcessResult) -> list[FileContent]: if not curr_file_name: continue - curr_file_lines = rendered_files.get(curr_file_name, []) - if not curr_file_lines: - curr_file_lines = [] - rendered_files[curr_file_name] = curr_file_lines - curr_file_lines.append(line) + rendered_files[curr_file_name].append(line) return [file_content(file_name, lines) for file_name, lines in rendered_files.items()] + logger.debug(f"Running Helm renderer process for deployment {renderer.address}") result = await Get(ProcessResult, HelmProcess, renderer.process) output_snapshot = EMPTY_SNAPSHOT if not renderer.output_directory: + logger.debug( + f"Parsing Helm renderer files from the process' output of deployment {renderer.address}." + ) output_snapshot = await Get(Snapshot, CreateDigest(parse_renderer_output(result))) else: + logger.debug( + f"Obtaining Helm renderer files from the process' output directory of deployment {renderer.address}." + ) output_snapshot = await Get( Snapshot, RemovePrefix(result.output_digest, renderer.output_directory) ) - return RenderedFiles(output_snapshot) + return RenderedFiles(address=renderer.address, chart=renderer.chart, snapshot=output_snapshot) def rules(): diff --git a/src/python/pants/backend/helm/util_rules/tool.py b/src/python/pants/backend/helm/util_rules/tool.py index 2301fce2817e..c61022f9653a 100644 --- a/src/python/pants/backend/helm/util_rules/tool.py +++ b/src/python/pants/backend/helm/util_rules/tool.py @@ -24,6 +24,7 @@ ) from pants.engine import process from pants.engine.collection import Collection +from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType from pants.engine.environment import Environment, EnvironmentRequest from pants.engine.fs import ( CreateDigest, @@ -31,10 +32,11 @@ DigestContents, DigestSubset, Directory, + FileDigest, PathGlobs, RemovePrefix, ) -from pants.engine.internals.native_engine import AddPrefix, MergeDigests +from pants.engine.internals.native_engine import AddPrefix, MergeDigests, Snapshot from pants.engine.platform import Platform from pants.engine.process import Process, ProcessCacheScope from pants.engine.rules import Get, MultiGet, collect_rules, rule @@ -127,7 +129,7 @@ def from_dict(cls, d: dict[str, Any]) -> HelmPluginPlatformCommand: @dataclass(frozen=True) -class HelmPluginMetadata: +class HelmPluginInfo: name: str version: str usage: str | None = None @@ -140,7 +142,7 @@ class HelmPluginMetadata: hooks: FrozenDict[str, str] = dataclasses.field(default_factory=FrozenDict) @classmethod - def from_dict(cls, d: dict[str, Any]) -> HelmPluginMetadata: + def from_dict(cls, d: dict[str, Any]) -> HelmPluginInfo: platform_command = [ HelmPluginPlatformCommand.from_dict(d) for d in d.pop("platformCommand", []) ] @@ -150,8 +152,8 @@ def from_dict(cls, d: dict[str, Any]) -> HelmPluginMetadata: return cls(platform_command=tuple(platform_command), hooks=FrozenDict(hooks), **attrs) @classmethod - def from_bytes(cls, content: bytes) -> HelmPluginMetadata: - return HelmPluginMetadata.from_dict(yaml.safe_load(content)) + def from_bytes(cls, content: bytes) -> HelmPluginInfo: + return HelmPluginInfo.from_dict(yaml.safe_load(content)) _ExternalHelmPlugin = TypeVar("_ExternalHelmPlugin", bound=ExternalHelmPlugin) @@ -174,31 +176,58 @@ def create(cls: Type[_EHPB]) -> _EHPB: @dataclass(frozen=True) -class ExternalHelmPluginRequest: +class ExternalHelmPluginRequest(EngineAwareParameter): """Helper class to create a download request for an external Helm plugin.""" plugin_name: str - tool_request: ExternalToolRequest + platform: Platform + + _tool_request: ExternalToolRequest @classmethod def from_subsystem(cls, subsystem: ExternalHelmPlugin) -> ExternalHelmPluginRequest: + platform = Platform.current return cls( - plugin_name=subsystem.plugin_name, tool_request=subsystem.get_request(Platform.current) + plugin_name=subsystem.plugin_name, + platform=platform, + _tool_request=subsystem.get_request(platform), ) + def debug_hint(self) -> str | None: + return self.plugin_name + + def metadata(self) -> dict[str, Any] | None: + return {"platform": self.platform, "url": self._tool_request.download_file_request.url} + @dataclass(frozen=True) -class HelmPlugin: - metadata: HelmPluginMetadata - digest: Digest +class HelmPlugin(EngineAwareReturnType): + info: HelmPluginInfo + platform: Platform + snapshot: Snapshot @property def name(self) -> str: - return self.metadata.name + return self.info.name @property def version(self) -> str: - return self.metadata.version + return self.info.version + + def level(self) -> LogLevel | None: + return LogLevel.INFO + + def message(self) -> str | None: + return f"Materialized Helm plugin {self.name} with version {self.version} for {self.platform} platform." + + def metadata(self) -> dict[str, Any] | None: + return {"name": self.name, "version": self.version, "platform": self.platform} + + def artifacts(self) -> dict[str, FileDigest | Snapshot] | None: + return {"content": self.snapshot} + + def cacheable(self) -> bool: + return True class HelmPlugins(Collection[HelmPlugin]): @@ -221,28 +250,29 @@ async def all_helm_plugins(union_membership: UnionMembership) -> HelmPlugins: @rule(desc="Download external Helm plugin", level=LogLevel.DEBUG) async def download_external_helm_plugin(request: ExternalHelmPluginRequest) -> HelmPlugin: - downloaded_tool = await Get(DownloadedExternalTool, ExternalToolRequest, request.tool_request) + downloaded_tool = await Get(DownloadedExternalTool, ExternalToolRequest, request._tool_request) - metadata_file = await Get( + plugin_info_file = await Get( Digest, DigestSubset( downloaded_tool.digest, PathGlobs( - ["plugin.yaml"], + ["plugin.yaml", "plugin.yml"], glob_match_error_behavior=GlobMatchErrorBehavior.error, - description_of_origin=f"The Helm plugin `{request.plugin_name}`", + description_of_origin=request.plugin_name, ), ), ) - metadata_content = await Get(DigestContents, Digest, metadata_file) - if len(metadata_content) == 0: + plugin_info_contents = await Get(DigestContents, Digest, plugin_info_file) + if len(plugin_info_contents) == 0: raise HelmPluginMetadataFileNotFound(request.plugin_name) - metadata = HelmPluginMetadata.from_bytes(metadata_content[0].content) - if not metadata.command and not metadata.platform_command: + plugin_info = HelmPluginInfo.from_bytes(plugin_info_contents[0].content) + if not plugin_info.command and not plugin_info.platform_command: raise HelmPluginMissingCommand(request.plugin_name) - return HelmPlugin(metadata=metadata, digest=downloaded_tool.digest) + plugin_snapshot = await Get(Snapshot, Digest, downloaded_tool.digest) + return HelmPlugin(info=plugin_info, platform=request.platform, snapshot=plugin_snapshot) # --------------------------------------------- @@ -315,10 +345,13 @@ async def setup_helm(helm_subsytem: HelmSubsystem, global_plugins: HelmPlugins) # Install all global Helm plugins if global_plugins: + logger.debug(f"Installing {pluralize(len(global_plugins), 'global Helm plugin')}.") prefixed_plugins_digests = await MultiGet( Get( Digest, - AddPrefix(plugin.digest, os.path.join(_HELM_DATA_DIR, "plugins", plugin.name)), + AddPrefix( + plugin.snapshot.digest, os.path.join(_HELM_DATA_DIR, "plugins", plugin.name) + ), ) for plugin in global_plugins )