OptionsBootstrapper is constructed from uncacheable singleton Options…

…Inputs.

[ci skip-build-wheels]

[ci skip-rust]

src/python/pants/engine/internals/options_parsing.py

@@ -2,34 +2,52 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from dataclasses import dataclass
+from typing import cast
 
 from pants.build_graph.build_configuration import BuildConfiguration
-from pants.engine.rules import collect_rules, rule
+from pants.engine.rules import _uncacheable_rule, collect_rules, rule
 from pants.init.options_initializer import OptionsInitializer
 from pants.option.global_options import GlobalOptions
 from pants.option.options import Options
-from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs
 from pants.option.scope import Scope, ScopedOptions
 from pants.util.logging import LogLevel
+from pants.util.memo import memoized_property
 
 
 @dataclass(frozen=True)
 class _Options:
-    """A wrapper around bootstrapped options values: not for direct consumption."""
+    """A wrapper around bootstrapped options values: not for direct consumption.
 
-    options: Options
+    TODO: This odd indirection exists because the `Options` type does not have useful `eq`, but
+    OptionsBootstrapper and BuildConfiguration both do.
+    """
 
+    options_bootstrapper: OptionsBootstrapper
+    build_config: BuildConfiguration
 
-@rule
-def parse_options(
-    options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration
-) -> _Options:
-    # TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options
-    # remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make
-    # minimize the size of that value.
-    return _Options(
-        OptionsInitializer.create(options_bootstrapper, build_config, init_subsystems=False)
+    @memoized_property
+    def options(self) -> Options:
+        return cast(
+            Options,
+            OptionsInitializer.create(
+                self.options_bootstrapper, self.build_config, init_subsystems=False
+            ),
+        )
+
+
+# TODO: This rule is uncacheable because OptionsBootstrapper.create reads absolute files, which
+# we do not have intrinsics for yet. Once we have intrinsics for absolute file reads (relates
+# to #10769), this @rule should use them. On the other hand, computation of the OptionsInputs will
+# always be uncacheable.
+@_uncacheable_rule
+def parse_options(build_config: BuildConfiguration, options_inputs: OptionsInputs) -> _Options:
+    options_bootstrapper = OptionsBootstrapper.create(
+        args=options_inputs.args,
+        env=options_inputs.env,
+        allow_pantsrc=options_inputs.allow_pantsrc,
     )
+    return _Options(options_bootstrapper, build_config)
 
 
 @rule

src/python/pants/init/engine_initializer.py

@@ -2,9 +2,11 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import logging
+import os
+import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, ClassVar, Iterable, List, Optional, Set, Tuple, Type, cast
+from typing import Any, Callable, ClassVar, Iterable, List, Optional, Set, Tuple, Type, cast
 
 from pants.base.build_environment import get_buildroot
 from pants.base.build_root import BuildRoot
@@ -23,13 +25,13 @@
 from pants.engine.internals.selectors import Params
 from pants.engine.platform import create_platform_rules
 from pants.engine.process import InteractiveRunner
-from pants.engine.rules import QueryRule, collect_rules, rule
+from pants.engine.rules import QueryRule, _uncacheable_rule, collect_rules, rule
 from pants.engine.target import RegisteredTargetTypes
 from pants.engine.unions import UnionMembership
 from pants.init import specs_calculator
 from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer
 from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS, ExecutionOptions
-from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs
 from pants.option.subsystem import Subsystem
 from pants.scm.subsystems.changed import rules as changed_rules
 from pants.util.ordered_set import FrozenOrderedSet
@@ -177,8 +179,13 @@ def setup_graph(
         native = Native()
         build_root = get_buildroot()
         bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
+
+        def options_inputs_accessor() -> OptionsInputs:
+            return OptionsInputs(sys.argv, dict(os.environ), allow_pantsrc=True)
+
         return EngineInitializer.setup_graph_extended(
             options_bootstrapper,
+            options_inputs_accessor,
             build_configuration,
             ExecutionOptions.from_bootstrap_options(bootstrap_options),
             pants_ignore_patterns=OptionsInitializer.compute_pants_ignore(
@@ -197,6 +204,7 @@ def setup_graph(
     @staticmethod
     def setup_graph_extended(
         options_bootstrapper: OptionsBootstrapper,
+        options_inputs_accessor: Callable[[], OptionsInputs],
         build_configuration: BuildConfiguration,
         execution_options: ExecutionOptions,
         native: Native,
@@ -222,6 +230,11 @@ def setup_graph_extended(
         bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
         execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
 
+        @_uncacheable_rule
+        def options_inputs_singleton() -> OptionsInputs:
+            # Computed once per session: see OptionsInputs for more information.
+            return options_inputs_accessor()
+
         @rule
         def parser_singleton() -> Parser:
             return Parser(

src/python/pants/option/options_bootstrapper.py

@@ -20,6 +20,19 @@
 from pants.util.strutil import ensure_text
 
 
+@dataclass(frozen=True)
+class OptionsInputs:
+    """The inputs used to construct an OptionsBootstrapper.
+
+    This type is provided by an uncacheable singleton @rule in production usage, which forces it to
+    be re-computed once per session.
+    """
+
+    args: List[str]
+    env: Dict[str, str]
+    allow_pantsrc: bool
+
+
 @dataclass(frozen=True)
 class OptionsBootstrapper:
     """Holds the result of the first stage of options parsing, and assists with parsing full

src/python/pants/testutil/rule_runner.py

@@ -41,7 +41,7 @@
 from pants.engine.unions import UnionMembership
 from pants.init.engine_initializer import EngineInitializer
 from pants.option.global_options import ExecutionOptions, GlobalOptions
-from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs
 from pants.source import source_root
 from pants.testutil.option_util import create_options_bootstrapper
 from pants.util.collections import assert_single_element
@@ -106,8 +106,13 @@ def __init__(
         build_config_builder.register_target_types(target_types or ())
         self.build_config = build_config_builder.create()
 
+        options_inputs = OptionsInputs(
+            args=["--pants-config-files=[]"], env={}, allow_pantsrc=False
+        )
         options_bootstrapper = OptionsBootstrapper.create(
-            env={}, args=["--pants-config-files=[]"], allow_pantsrc=False
+            args=options_inputs.args,
+            env=options_inputs.env,
+            allow_pantsrc=options_inputs.allow_pantsrc,
         )
         global_options = options_bootstrapper.bootstrap_options.for_global_scope()
         local_store_dir = global_options.local_store_dir
@@ -122,6 +127,7 @@ def __init__(
             named_caches_dir=named_caches_dir,
             native=Native(),
             options_bootstrapper=options_bootstrapper,
+            options_inputs_accessor=lambda: options_inputs,
             build_root=self.build_root,
             build_configuration=self.build_config,
             execution_options=ExecutionOptions.from_bootstrap_options(global_options),

src/python/pants/testutil/test_base.py

@@ -42,7 +42,7 @@
 from pants.init.engine_initializer import EngineInitializer
 from pants.init.util import clean_global_runtime_state
 from pants.option.global_options import ExecutionOptions, GlobalOptions
-from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs
 from pants.option.subsystem import Subsystem
 from pants.source import source_root
 from pants.testutil.option_util import create_options_bootstrapper
@@ -300,8 +300,13 @@ def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
         if self._scheduler is not None:
             return
 
+        options_inputs = OptionsInputs(
+            args=["--pants-config-files=[]", *self.additional_options], env={}, allow_pantsrc=False
+        )
         options_bootstrapper = OptionsBootstrapper.create(
-            env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False
+            args=options_inputs.args,
+            env=options_inputs.env,
+            allow_pantsrc=options_inputs.allow_pantsrc,
         )
         global_options = options_bootstrapper.bootstrap_options.for_global_scope()
         local_store_dir = local_store_dir or global_options.local_store_dir
@@ -317,6 +322,7 @@ def _init_engine(self, local_store_dir: Optional[str] = None) -> None:
             ca_certs_path=global_options.ca_certs_path,
             native=Native(),
             options_bootstrapper=options_bootstrapper,
+            options_inputs_accessor=lambda: options_inputs,
             build_root=self.build_root,
             build_configuration=self.build_config(),
             execution_options=ExecutionOptions.from_bootstrap_options(global_options),