Improve logging in the Helm backend

src/python/pants/backend/helm/goals/lint.py

@@ -55,7 +55,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess
         return HelmProcess(
             argv,
             input_digest=chart.snapshot.digest,
-            description=f"Linting chart: {chart.metadata.name}",
+            description=f"Linting chart: {chart.info.name}",
         )
 
     process_results = await MultiGet(
@@ -68,7 +68,7 @@ def create_process(chart: HelmChart, field_set: HelmLintFieldSet) -> HelmProcess
     )
     results = [
         LintResult.from_fallible_process_result(
-            process_result, partition_description=chart.metadata.name
+            process_result, partition_description=chart.info.name
         )
         for chart, process_result in zip(charts, process_results)
     ]

src/python/pants/backend/helm/goals/package.py

@@ -31,13 +31,13 @@
 
 @dataclass(frozen=True)
 class BuiltHelmArtifact(BuiltPackageArtifact):
-    metadata: HelmChartMetadata | None = None
+    info: HelmChartMetadata | None = None
 
     @classmethod
-    def create(cls, relpath: str, metadata: HelmChartMetadata) -> BuiltHelmArtifact:
+    def create(cls, relpath: str, info: HelmChartMetadata) -> BuiltHelmArtifact:
         return cls(
             relpath=relpath,
-            metadata=metadata,
+            info=info,
             extra_log_lines=(f"Built Helm chart artifact: {relpath}",),
         )
 
@@ -57,7 +57,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage:
     )
 
     input_digest = await Get(Digest, MergeDigests([chart.snapshot.digest, result_digest]))
-    process_output_file = os.path.join(result_dir, f"{chart.metadata.artifact_name}.tgz")
+    process_output_file = os.path.join(result_dir, f"{chart.info.artifact_name}.tgz")
 
     process_result = await Get(
         ProcessResult,
@@ -80,7 +80,7 @@ async def run_helm_package(field_set: HelmPackageFieldSet) -> BuiltPackage:
     return BuiltPackage(
         final_snapshot.digest,
         artifacts=tuple(
-            BuiltHelmArtifact.create(file, chart.metadata) for file in final_snapshot.files
+            BuiltHelmArtifact.create(file, chart.info) for file in final_snapshot.files
         ),
     )
 

src/python/pants/backend/helm/goals/package_test.py

@@ -59,7 +59,7 @@ def _assert_build_package(rule_runner: RuleRunner, *, chart_name: str, chart_ver
     assert result.artifacts[0].relpath == os.path.join(
         dest_dir, f"{chart_name}-{chart_version}.tgz"
     )
-    assert result.artifacts[0].metadata
+    assert result.artifacts[0].info
 
 
 def test_helm_package(rule_runner: RuleRunner) -> None:

src/python/pants/backend/helm/goals/publish.py

@@ -57,10 +57,10 @@ async def publish_helm_chart(
 ) -> PublishProcesses:
     remotes = helm_subsystem.remotes()
     built_artifacts = [
-        (pkg, artifact, artifact.metadata)
+        (pkg, artifact, artifact.info)
         for pkg in request.packages
         for artifact in pkg.artifacts
-        if isinstance(artifact, BuiltHelmArtifact) and artifact.metadata
+        if isinstance(artifact, BuiltHelmArtifact) and artifact.info
     ]
 
     registries_to_push = list(remotes.get(*(request.field_set.registries.value or [])))

src/python/pants/backend/helm/subsystems/BUILD

@@ -5,7 +5,7 @@ python_requirement(
     name="yamlpath",
     requirements=[
         "yamlpath>=3.6,<3.7",
-        "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17",
+        "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21",
     ],
     resolve="helm-post-renderer",
 )

src/python/pants/backend/helm/subsystems/post_renderer.lock

@@ -9,7 +9,7 @@
 //     "CPython<3.10,>=3.7"
 //   ],
 //   "generated_with_requirements": [
-//     "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17,>=0.15.96",
+//     "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96",
 //     "yamlpath<3.7,>=3.6"
 //   ]
 // }
@@ -143,15 +143,15 @@
       "platform_tag": [
         "cp39",
         "cp39",
-        "macosx_12_0_x86_64"
+        "macosx_12_0_arm64"
       ]
     }
   ],
   "path_mappings": {},
   "pex_version": "2.1.90",
   "prefer_older_binary": false,
   "requirements": [
-    "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17,>=0.15.96",
+    "ruamel.yaml!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21,>=0.15.96",
     "yamlpath<3.7,>=3.6"
   ],
   "requires_python": [

src/python/pants/backend/helm/subsystems/post_renderer.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import logging
 import os
 import pkgutil
 from dataclasses import dataclass
@@ -19,6 +20,7 @@
 from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
 from pants.core.util_rules.system_binaries import CatBinary
+from pants.engine.engine_aware import EngineAwareParameter, EngineAwareReturnType
 from pants.engine.fs import CreateDigest, Digest, FileContent
 from pants.engine.internals.native_engine import MergeDigests
 from pants.engine.process import Process
@@ -28,6 +30,8 @@
 from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 
+logger = logging.getLogger(__name__)
+
 _HELM_POSTRENDERER_SOURCE = "post_renderer_launcher.py"
 _HELM_POSTRENDERER_PACKAGE = "pants.backend.helm.subsystems"
 
@@ -38,7 +42,7 @@ class HelmPostRenderer(PythonToolRequirementsBase):
 
     default_version = "yamlpath>=3.6,<3.7"
     default_extra_requirements = [
-        "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.17"
+        "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,!=0.17.5,<=0.17.21"
     ]
 
     register_interpreter_constraints = True
@@ -73,7 +77,7 @@ class HelmPostRendererTool:
     pex: VenvPex
 
 
-@rule(desc="Prepare Helm post renderer", level=LogLevel.DEBUG)
+@rule(desc="Setup Helm post renderer binaries", level=LogLevel.DEBUG)
 async def setup_internal_post_renderer(
     post_renderer: HelmPostRenderer,
 ) -> HelmPostRendererTool:
@@ -103,20 +107,31 @@ async def setup_internal_post_renderer(
 
 
 @dataclass(frozen=True)
-class SetupHelmPostRenderer:
+class SetupHelmPostRenderer(EngineAwareParameter):
     replacements: YamlElements[str]
+    description_of_origin: str
+
+    def debug_hint(self) -> str | None:
+        return self.description_of_origin
 
 
 @dataclass(frozen=True)
-class HelmPostRendererRunnable:
+class HelmPostRendererRunnable(EngineAwareReturnType):
     exe: str
     digest: Digest
     immutable_input_digests: FrozenDict[str, Digest]
     env: FrozenDict[str, str]
     append_only_caches: FrozenDict[str, str]
+    description_of_origin: str
+
+    def level(self) -> LogLevel | None:
+        return LogLevel.DEBUG
 
+    def message(self) -> str | None:
+        return f"runnable {self.exe} for {self.description_of_origin} is ready."
 
-@rule(desc="Configure Helm Post Renderer", level=LogLevel.DEBUG)
+
+@rule(desc="Configure Helm post-renderer", level=LogLevel.DEBUG)
 async def setup_post_renderer_launcher(
     request: SetupHelmPostRenderer,
     post_renderer_tool: HelmPostRendererTool,
@@ -137,25 +152,29 @@ async def setup_post_renderer_launcher(
         ),
     )
 
+    post_renderer_cfg_file = os.path.join(".", HELM_POST_RENDERER_CFG_FILENAME)
     post_renderer_stdin_file = os.path.join(".", "__stdin.yaml")
     post_renderer_process = await Get(
         Process,
         VenvPexProcess(
             post_renderer_tool.pex,
-            argv=[os.path.join(".", HELM_POST_RENDERER_CFG_FILENAME), post_renderer_stdin_file],
+            argv=[post_renderer_cfg_file, post_renderer_stdin_file],
             input_digest=post_renderer_cfg_digest,
             description="",
         ),
     )
 
+    post_renderer_process_cli = " ".join(post_renderer_process.argv)
+    logger.debug(f"Built post-renderer process CLI: {post_renderer_process_cli}")
+
     postrenderer_wrapper_script = dedent(
         f"""\
         #!/bin/bash
 
         # Output stdin into a file in disk
         {cat_binary.path} <&0 > {post_renderer_stdin_file}
 
-        {' '.join(post_renderer_process.argv)}
+        {post_renderer_process_cli}
         """
     )
     wrapper_digest = await Get(
@@ -180,6 +199,7 @@ async def setup_post_renderer_launcher(
         env=post_renderer_process.env,
         append_only_caches=post_renderer_process.append_only_caches,
         immutable_input_digests=post_renderer_process.immutable_input_digests,
+        description_of_origin=request.description_of_origin,
     )
 
 

src/python/pants/backend/helm/subsystems/post_renderer_test.py

@@ -49,7 +49,12 @@ def test_post_renderer_is_runnable(rule_runner: RuleRunner) -> None:
     )
 
     post_renderer_setup = rule_runner.request(
-        HelmPostRendererRunnable, [SetupHelmPostRenderer(replacements)]
+        HelmPostRendererRunnable,
+        [
+            SetupHelmPostRenderer(
+                replacements, description_of_origin="test_post_renderer_is_runnable"
+            )
+        ],
     )
     assert post_renderer_setup.exe == "post_renderer_wrapper.sh"
 

src/python/pants/backend/helm/util_rules/chart.py

@@ -30,6 +30,7 @@
 )
 from pants.backend.helm.util_rules.sources import HelmChartSourceFiles, HelmChartSourceFilesRequest
 from pants.engine.addresses import Address, Addresses
+from pants.engine.engine_aware import EngineAwareParameter
 from pants.engine.fs import (
     EMPTY_DIGEST,
     AddPrefix,
@@ -43,7 +44,7 @@
 from pants.engine.target import DependenciesRequest, ExplicitlyProvidedDependencies, Target, Targets
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import OrderedSet
-from pants.util.strutil import pluralize
+from pants.util.strutil import pluralize, softwrap
 
 logger = logging.getLogger(__name__)
 
@@ -56,17 +57,17 @@ def __init__(self, target: Target) -> None:
 @dataclass(frozen=True)
 class HelmChart:
     address: Address
-    metadata: HelmChartMetadata
+    info: HelmChartMetadata
     snapshot: Snapshot
     artifact: ResolvedHelmArtifact | None = None
 
     @property
     def path(self) -> str:
-        return self.metadata.name
+        return self.info.name
 
 
 @dataclass(frozen=True)
-class HelmChartRequest:
+class HelmChartRequest(EngineAwareParameter):
     field_set: HelmChartFieldSet
 
     @classmethod
@@ -75,6 +76,9 @@ def from_target(cls, target: Target) -> HelmChartRequest:
             raise InvalidHelmChartTarget(target)
         return cls(HelmChartFieldSet.create(target))
 
+    def debug_hint(self) -> str | None:
+        return self.field_set.address.spec
+
 
 @rule
 async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> HelmChart:
@@ -96,7 +100,7 @@ async def create_chart_from_artifact(fetched_artifact: FetchedHelmArtifact) -> H
 
 @rule(desc="Collect all source code and subcharts of a Helm Chart", level=LogLevel.DEBUG)
 async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) -> HelmChart:
-    dependencies, source_files, metadata = await MultiGet(
+    dependencies, source_files, chart_info = await MultiGet(
         Get(Targets, DependenciesRequest(request.field_set.dependencies)),
         Get(
             HelmChartSourceFiles,
@@ -132,7 +136,12 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) ->
     subcharts_digest = EMPTY_DIGEST
     if subcharts:
         logger.debug(
-            f"Found {pluralize(len(subcharts), 'subchart')} as direct dependencies on Helm chart at: {request.field_set.address}"
+            softwrap(
+                f"""
+                Found {pluralize(len(subcharts), 'subchart')} as direct dependencies
+                on Helm chart at: {request.field_set.address}.
+                """
+            )
         )
 
         merged_subcharts = await Get(
@@ -142,9 +151,9 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) ->
 
         # Update subchart dependencies in the metadata and re-render it.
         remotes = subsystem.remotes()
-        subchart_map: dict[str, HelmChart] = {chart.metadata.name: chart for chart in subcharts}
+        subchart_map: dict[str, HelmChart] = {chart.info.name: chart for chart in subcharts}
         updated_dependencies: OrderedSet[HelmChartDependency] = OrderedSet()
-        for dep in metadata.dependencies:
+        for dep in chart_info.dependencies:
             updated_dep = dep
 
             if not dep.repository and remotes.default_registry:
@@ -157,15 +166,15 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) ->
 
             if dep.name in subchart_map:
                 updated_dep = dataclasses.replace(
-                    updated_dep, version=subchart_map[dep.name].metadata.version
+                    updated_dep, version=subchart_map[dep.name].info.version
                 )
 
             updated_dependencies.add(updated_dep)
 
         # Include the explicitly provided subchats in the set of dependencies if not already present.
         updated_dependencies_names = {dep.name for dep in updated_dependencies}
         remaining_subcharts = [
-            chart for chart in subcharts if chart.metadata.name not in updated_dependencies_names
+            chart for chart in subcharts if chart.info.name not in updated_dependencies_names
         ]
         for chart in remaining_subcharts:
             if chart.artifact:
@@ -176,16 +185,16 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) ->
                 )
             else:
                 dependency = HelmChartDependency(
-                    name=chart.metadata.name, version=chart.metadata.version
+                    name=chart.info.name, version=chart.info.version
                 )
             updated_dependencies.add(dependency)
 
         # Update metadata with the information about charts' dependencies.
-        metadata = dataclasses.replace(metadata, dependencies=tuple(updated_dependencies))
+        chart_info = dataclasses.replace(chart_info, dependencies=tuple(updated_dependencies))
 
     # Re-render the Chart.yaml file with the updated dependencies.
     metadata_digest, sources_without_metadata = await MultiGet(
-        Get(Digest, HelmChartMetadata, metadata),
+        Get(Digest, HelmChartMetadata, chart_info),
         Get(
             Digest,
             DigestSubset(
@@ -202,8 +211,8 @@ async def get_helm_chart(request: HelmChartRequest, subsystem: HelmSubsystem) ->
         Digest, MergeDigests([metadata_digest, sources_without_metadata, subcharts_digest])
     )
 
-    chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, metadata.name))
-    return HelmChart(address=request.field_set.address, metadata=metadata, snapshot=chart_snapshot)
+    chart_snapshot = await Get(Snapshot, AddPrefix(content_digest, chart_info.name))
+    return HelmChart(address=request.field_set.address, info=chart_info, snapshot=chart_snapshot)
 
 
 class MissingHelmDeploymentChartError(ValueError):
@@ -222,11 +231,14 @@ def __init__(self, address: Address) -> None:
 
 
 @dataclass(frozen=True)
-class FindHelmDeploymentChart:
+class FindHelmDeploymentChart(EngineAwareParameter):
     field_set: HelmDeploymentFieldSet
 
+    def debug_hint(self) -> str | None:
+        return self.field_set.address.spec
 
-@rule
+
+@rule(desc="Find Helm deployment's chart", level=LogLevel.DEBUG)
 async def find_chart_for_deployment(request: FindHelmDeploymentChart) -> HelmChartRequest:
     explicit_dependencies = await Get(
         ExplicitlyProvidedDependencies, DependenciesRequest(request.field_set.dependencies)