Generically compute dynamic defaults for Fields (pantsbuild#16206)

As discussed on pantsbuild#16175, we don't currently consume the "dynamic" defaults of field values for the purposes of `parametrize`. That is at least partially because there is no generic way to do so: a `Field` has no way to declare a dynamic default currently, because `Field`s cannot declare a dependency `@rule_helper` to compute their value (...yet? see pantsbuild#12934 (comment)).

This change adds a mechanism for generically declaring the default value of a `Field`. This is definitely not the most ergonomic API: over the next few versions, many dynamic `Field` defaults will hopefully move to `__defaults__`. And pantsbuild#12934 (comment) will hopefully allow for significantly cleaning up those that remain.

Fixes pantsbuild#16175.

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/java/bsp/rules.py

@@ -15,8 +15,6 @@
 from pants.bsp.util_rules.targets import (
     BSPBuildTargetsMetadataRequest,
     BSPBuildTargetsMetadataResult,
-    BSPResolveFieldFactoryRequest,
-    BSPResolveFieldFactoryResult,
 )
 from pants.engine.addresses import Addresses
 from pants.engine.fs import CreateDigest, DigestEntries
@@ -31,7 +29,6 @@
     FallibleClasspathEntry,
 )
 from pants.jvm.resolve.key import CoursierResolveKey
-from pants.jvm.subsystems import JvmSubsystem
 from pants.jvm.target_types import JvmResolveField
 
 LANGUAGE_ID = "java"
@@ -52,28 +49,15 @@ class JavaMetadataFieldSet(FieldSet):
     resolve: JvmResolveField
 
 
-class JavaBSPResolveFieldFactoryRequest(BSPResolveFieldFactoryRequest):
-    resolve_prefix = "jvm"
-
-
 class JavaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest):
     language_id = LANGUAGE_ID
     can_merge_metadata_from = ()
     field_set_type = JavaMetadataFieldSet
+
     resolve_prefix = "jvm"
     resolve_field = JvmResolveField
 
 
-@rule
-def bsp_resolve_field_factory(
-    request: JavaBSPResolveFieldFactoryRequest,
-    jvm: JvmSubsystem,
-) -> BSPResolveFieldFactoryResult:
-    return BSPResolveFieldFactoryResult(
-        lambda target: target.get(JvmResolveField).normalized_value(jvm)
-    )
-
-
 @rule
 async def bsp_resolve_java_metadata(
     _: JavaBSPBuildTargetsMetadataRequest,
@@ -195,7 +179,6 @@ def rules():
     return (
         *collect_rules(),
         UnionRule(BSPLanguageSupport, JavaBSPLanguageSupport),
-        UnionRule(BSPResolveFieldFactoryRequest, JavaBSPResolveFieldFactoryRequest),
         UnionRule(BSPBuildTargetsMetadataRequest, JavaBSPBuildTargetsMetadataRequest),
         UnionRule(BSPHandlerMapping, JavacOptionsHandlerMapping),
         UnionRule(BSPCompileRequest, JavaBSPCompileRequest),

src/python/pants/backend/java/target_types.py

@@ -15,6 +15,7 @@
     Target,
     TargetFilesGenerator,
 )
+from pants.jvm import target_types as jvm_target_types
 from pants.jvm.target_types import (
     JunitTestSourceField,
     JvmJdkField,
@@ -128,4 +129,7 @@ class JavaSourcesGeneratorTarget(TargetFilesGenerator):
 
 
 def rules():
-    return collect_rules()
+    return [
+        *collect_rules(),
+        *jvm_target_types.rules(),
+    ]

src/python/pants/backend/python/target_types_rules.py

@@ -51,6 +51,8 @@
     Dependencies,
     DependenciesRequest,
     ExplicitlyProvidedDependencies,
+    FieldDefaultFactoryRequest,
+    FieldDefaultFactoryResult,
     FieldSet,
     GeneratedTargets,
     GenerateTargetsRequest,
@@ -441,6 +443,23 @@ async def inject_python_distribution_dependencies(
     )
 
 
+# -----------------------------------------------------------------------------------------------
+# Dynamic Field defaults
+# -----------------------------------------------------------------------------------------------
+
+
+class PythonResolveFieldDefaultFactoryRequest(FieldDefaultFactoryRequest):
+    field_type = PythonResolveField
+
+
+@rule
+def python_resolve_field_default_factory(
+    request: PythonResolveFieldDefaultFactoryRequest,
+    python_setup: PythonSetup,
+) -> FieldDefaultFactoryResult:
+    return FieldDefaultFactoryResult(lambda f: f.normalized_value(python_setup))
+
+
 # -----------------------------------------------------------------------------------------------
 # Dependency validation
 # -----------------------------------------------------------------------------------------------
@@ -509,6 +528,7 @@ def rules():
     return (
         *collect_rules(),
         *import_rules(),
+        UnionRule(FieldDefaultFactoryRequest, PythonResolveFieldDefaultFactoryRequest),
         UnionRule(TargetFilesGeneratorSettingsRequest, PythonFilesGeneratorSettingsRequest),
         UnionRule(GenerateTargetsRequest, GenerateTargetsFromPexBinaries),
         UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency),

src/python/pants/backend/scala/bsp/rules.py

@@ -31,8 +31,6 @@
     BSPBuildTargetsMetadataResult,
     BSPDependencyModulesRequest,
     BSPDependencyModulesResult,
-    BSPResolveFieldFactoryRequest,
-    BSPResolveFieldFactoryResult,
 )
 from pants.engine.addresses import Addresses
 from pants.engine.fs import (
@@ -91,15 +89,14 @@ class ScalaMetadataFieldSet(FieldSet):
     resolve: JvmResolveField
 
 
-class ScalaBSPResolveFieldFactoryRequest(BSPResolveFieldFactoryRequest):
-    resolve_prefix = "jvm"
-
-
 class ScalaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest):
     language_id = LANGUAGE_ID
     can_merge_metadata_from = ("java",)
     field_set_type = ScalaMetadataFieldSet
 
+    resolve_prefix = "jvm"
+    resolve_field = JvmResolveField
+
 
 @dataclass(frozen=True)
 class ResolveScalaBSPBuildTargetRequest:
@@ -154,16 +151,6 @@ async def materialize_scala_runtime_jars(
     return MaterializeScalaRuntimeJarsResult(materialized_classpath)
 
 
-@rule
-def bsp_resolve_field_factory(
-    request: ScalaBSPResolveFieldFactoryRequest,
-    jvm: JvmSubsystem,
-) -> BSPResolveFieldFactoryResult:
-    return BSPResolveFieldFactoryResult(
-        lambda target: target.get(JvmResolveField).normalized_value(jvm)
-    )
-
-
 @rule
 async def bsp_resolve_scala_metadata(
     request: ScalaBSPBuildTargetsMetadataRequest,
@@ -470,7 +457,6 @@ def rules():
         *collect_rules(),
         UnionRule(BSPLanguageSupport, ScalaBSPLanguageSupport),
         UnionRule(BSPBuildTargetsMetadataRequest, ScalaBSPBuildTargetsMetadataRequest),
-        UnionRule(BSPResolveFieldFactoryRequest, ScalaBSPResolveFieldFactoryRequest),
         UnionRule(BSPHandlerMapping, ScalacOptionsHandlerMapping),
         UnionRule(BSPHandlerMapping, ScalaMainClassesHandlerMapping),
         UnionRule(BSPHandlerMapping, ScalaTestClassesHandlerMapping),

src/python/pants/backend/scala/target_types.py

@@ -20,6 +20,7 @@
     TargetFilesGeneratorSettingsRequest,
 )
 from pants.engine.unions import UnionRule
+from pants.jvm import target_types as jvm_target_types
 from pants.jvm.target_types import (
     JunitTestSourceField,
     JvmJdkField,
@@ -276,5 +277,6 @@ class ScalacPluginTarget(Target):
 def rules():
     return (
         *collect_rules(),
+        *jvm_target_types.rules(),
         UnionRule(TargetFilesGeneratorSettingsRequest, ScalaSettingsRequest),
     )

src/python/pants/bsp/util_rules/targets.py

@@ -10,7 +10,6 @@
 from typing import ClassVar, Generic, Sequence, Type, TypeVar
 
 import toml
-from typing_extensions import Protocol
 
 from pants.base.build_root import BuildRoot
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
@@ -40,11 +39,12 @@
 from pants.engine.internals.selectors import Get, MultiGet
 from pants.engine.rules import _uncacheable_rule, collect_rules, rule
 from pants.engine.target import (
+    Field,
+    FieldDefaults,
     FieldSet,
     SourcesField,
     SourcesPaths,
     SourcesPathsRequest,
-    Target,
     Targets,
 )
 from pants.engine.unions import UnionMembership, UnionRule, union
@@ -58,38 +58,6 @@
 _FS = TypeVar("_FS", bound=FieldSet)
 
 
-@union
-@dataclass(frozen=True)
-class BSPResolveFieldFactoryRequest(Generic[_FS]):
-    """Requests an implementation of `BSPResolveFieldFactory` which can filter resolve fields.
-
-    TODO: This is to work around the fact that Field value defaulting cannot have arbitrary
-    subsystem requirements, and so `JvmResolveField` and `PythonResolveField` have methods
-    which compute the true value of the field given a subsytem argument. Consumers need to
-    be type aware, and `@rules` cannot have dynamic requirements.
-
-    See https://github.com/pantsbuild/pants/issues/12934 about potentially allowing unions
-    (including Field registrations) to have `@rule_helper` methods, which would allow the
-    computation of an AsyncFields to directly require a subsystem.
-    """
-
-    resolve_prefix: ClassVar[str]
-
-
-# TODO: Workaround for https://github.com/python/mypy/issues/5485, because we cannot directly use
-# a Callable.
-class _ResolveFieldFactory(Protocol):
-    def __call__(self, target: Target) -> str | None:
-        pass
-
-
-@dataclass(frozen=True)
-class BSPResolveFieldFactoryResult:
-    """Computes the resolve field value for a Target, if applicable."""
-
-    resolve_field_value: _ResolveFieldFactory
-
-
 @union
 @dataclass(frozen=True)
 class BSPBuildTargetsMetadataRequest(Generic[_FS]):
@@ -99,6 +67,9 @@ class BSPBuildTargetsMetadataRequest(Generic[_FS]):
     can_merge_metadata_from: ClassVar[tuple[str, ...]]
     field_set_type: ClassVar[Type[_FS]]
 
+    resolve_prefix: ClassVar[str]
+    resolve_field: ClassVar[type[Field]]
+
     field_sets: tuple[_FS, ...]
 
 
@@ -255,6 +226,7 @@ async def resolve_bsp_build_target_identifier(
 async def resolve_bsp_build_target_addresses(
     bsp_target: BSPBuildTargetInternal,
     union_membership: UnionMembership,
+    field_defaults: FieldDefaults,
 ) -> Targets:
     targets = await Get(Targets, AddressSpecs, bsp_target.specs.address_specs)
     if bsp_target.definition.resolve_filter is None:
@@ -268,17 +240,19 @@ async def resolve_bsp_build_target_addresses(
             f"prefix like `$lang:$filter`, but the configured value: `{resolve_filter}` did not."
         )
 
-    # TODO: See `BSPResolveFieldFactoryRequest` re: this awkwardness.
-    factories = await MultiGet(
-        Get(BSPResolveFieldFactoryResult, BSPResolveFieldFactoryRequest, request())
-        for request in union_membership.get(BSPResolveFieldFactoryRequest)
-        if request.resolve_prefix == resolve_prefix
-    )
+    resolve_fields = {
+        impl.resolve_field
+        for impl in union_membership.get(BSPBuildTargetsMetadataRequest)
+        if impl.resolve_prefix == resolve_prefix
+    }
 
     return Targets(
         t
         for t in targets
-        if any((factory.resolve_field_value)(t) == resolve_value for factory in factories)
+        if any(
+            t.has_field(field) and field_defaults.value_or_default(t[field]) == resolve_value
+            for field in resolve_fields
+        )
     )
 
 

src/python/pants/bsp/util_rules/targets_test.py

@@ -4,6 +4,7 @@
 
 import pytest
 
+from pants.backend.java import target_types
 from pants.backend.java.bsp import rules as java_bsp_rules
 from pants.backend.java.compile import javac
 from pants.backend.java.target_types import JavaSourceTarget
@@ -29,6 +30,7 @@ def rule_runner() -> RuleRunner:
             *jvm_tool.rules(),
             *jvm_util_rules.rules(),
             *jdk_rules.rules(),
+            *target_types.rules(),
             QueryRule(BSPBuildTargets, ()),
             QueryRule(Targets, [BuildTargetIdentifier]),
         ],

src/python/pants/engine/internals/graph.py

@@ -57,6 +57,9 @@
     DependenciesRequest,
     ExplicitlyProvidedDependencies,
     Field,
+    FieldDefaultFactoryRequest,
+    FieldDefaultFactoryResult,
+    FieldDefaults,
     FieldSet,
     FieldSetsPerTarget,
     FieldSetsPerTargetRequest,
@@ -1066,6 +1069,7 @@ async def resolve_dependencies(
     target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests,
     union_membership: UnionMembership,
     subproject_roots: SubprojectRoots,
+    field_defaults: FieldDefaults,
 ) -> Addresses:
     wrapped_tgt, explicitly_provided = await MultiGet(
         Get(WrappedTarget, Address, request.field.address),
@@ -1119,7 +1123,7 @@ async def resolve_dependencies(
         )
 
         explicitly_provided_includes = [
-            parametrizations.get_subset(address, tgt).address
+            parametrizations.get_subset(address, tgt, field_defaults).address
             for address, parametrizations in zip(
                 explicitly_provided_includes, explicit_dependency_parametrizations
             )
@@ -1200,6 +1204,25 @@ async def resolve_unparsed_address_inputs(
     return Addresses(addresses)
 
 
+# -----------------------------------------------------------------------------------------------
+# Dynamic Field defaults
+# -----------------------------------------------------------------------------------------------
+
+
+@rule
+async def field_defaults(union_membership: UnionMembership) -> FieldDefaults:
+    requests = list(union_membership.get(FieldDefaultFactoryRequest))
+    factories = await MultiGet(
+        Get(FieldDefaultFactoryResult, FieldDefaultFactoryRequest, impl()) for impl in requests
+    )
+    return FieldDefaults(
+        FrozenDict(
+            (request.field_type, factory.default_factory)
+            for request, factory in zip(requests, factories)
+        )
+    )
+
+
 # -----------------------------------------------------------------------------------------------
 # Find applicable field sets
 # -----------------------------------------------------------------------------------------------