From 63f29c78adbf85cd469ce3a78c06a0ba808203ad Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Thu, 17 Sep 2020 11:56:01 -0700 Subject: [PATCH] OptionsBootstrapper is constructed from uncacheable singleton OptionsInputs. [ci skip-build-wheels] [ci skip-rust] --- .../pants/engine/internals/options_parsing.py | 44 +++++++++++++------ src/python/pants/init/engine_initializer.py | 19 ++++++-- .../pants/option/options_bootstrapper.py | 13 ++++++ src/python/pants/testutil/rule_runner.py | 10 ++++- src/python/pants/testutil/test_base.py | 10 ++++- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/src/python/pants/engine/internals/options_parsing.py b/src/python/pants/engine/internals/options_parsing.py index af932206ca55..42b6c28896a5 100644 --- a/src/python/pants/engine/internals/options_parsing.py +++ b/src/python/pants/engine/internals/options_parsing.py @@ -2,34 +2,52 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from dataclasses import dataclass +from typing import cast from pants.build_graph.build_configuration import BuildConfiguration -from pants.engine.rules import collect_rules, rule +from pants.engine.rules import _uncacheable_rule, collect_rules, rule from pants.init.options_initializer import OptionsInitializer from pants.option.global_options import GlobalOptions from pants.option.options import Options -from pants.option.options_bootstrapper import OptionsBootstrapper +from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs from pants.option.scope import Scope, ScopedOptions from pants.util.logging import LogLevel +from pants.util.memo import memoized_property @dataclass(frozen=True) class _Options: - """A wrapper around bootstrapped options values: not for direct consumption.""" + """A wrapper around bootstrapped options values: not for direct consumption. - options: Options + TODO: This odd indirection exists because the `Options` type does not have useful `eq`, but + OptionsBootstrapper and BuildConfiguration both do. + """ + options_bootstrapper: OptionsBootstrapper + build_config: BuildConfiguration -@rule -def parse_options( - options_bootstrapper: OptionsBootstrapper, build_config: BuildConfiguration -) -> _Options: - # TODO: Because _OptionsBootstapper is currently provided as a Param, this @rule relies on options - # remaining relatively stable in order to be efficient. See #6845 for a discussion of how to make - # minimize the size of that value. - return _Options( - OptionsInitializer.create(options_bootstrapper, build_config, init_subsystems=False) + @memoized_property + def options(self) -> Options: + return cast( + Options, + OptionsInitializer.create( + self.options_bootstrapper, self.build_config, init_subsystems=False + ), + ) + + +# TODO: This rule is uncacheable because OptionsBootstrapper.create reads absolute files, which +# we do not have intrinsics for yet. Once we have intrinsics for absolute file reads (relates +# to #10769), this @rule should use them. On the other hand, computation of the OptionsInputs will +# always be uncacheable. +@_uncacheable_rule +def parse_options(build_config: BuildConfiguration, options_inputs: OptionsInputs) -> _Options: + options_bootstrapper = OptionsBootstrapper.create( + args=options_inputs.args, + env=options_inputs.env, + allow_pantsrc=options_inputs.allow_pantsrc, ) + return _Options(options_bootstrapper, build_config) @rule diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index 70ab50de4fca..2eaf9d2de5c2 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -2,9 +2,11 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging +import os +import sys from dataclasses import dataclass from pathlib import Path -from typing import Any, ClassVar, Iterable, List, Optional, Set, Tuple, Type, cast +from typing import Any, Callable, ClassVar, Iterable, List, Optional, Set, Tuple, Type, cast from pants.base.build_environment import get_buildroot from pants.base.build_root import BuildRoot @@ -23,13 +25,13 @@ from pants.engine.internals.selectors import Params from pants.engine.platform import create_platform_rules from pants.engine.process import InteractiveRunner -from pants.engine.rules import QueryRule, collect_rules, rule +from pants.engine.rules import QueryRule, _uncacheable_rule, collect_rules, rule from pants.engine.target import RegisteredTargetTypes from pants.engine.unions import UnionMembership from pants.init import specs_calculator from pants.init.options_initializer import BuildConfigInitializer, OptionsInitializer from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS, ExecutionOptions -from pants.option.options_bootstrapper import OptionsBootstrapper +from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs from pants.option.subsystem import Subsystem from pants.scm.subsystems.changed import rules as changed_rules from pants.util.ordered_set import FrozenOrderedSet @@ -177,8 +179,13 @@ def setup_graph( native = Native() build_root = get_buildroot() bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() + + def options_inputs_accessor() -> OptionsInputs: + return OptionsInputs(sys.argv, dict(os.environ), allow_pantsrc=True) + return EngineInitializer.setup_graph_extended( options_bootstrapper, + options_inputs_accessor, build_configuration, ExecutionOptions.from_bootstrap_options(bootstrap_options), pants_ignore_patterns=OptionsInitializer.compute_pants_ignore( @@ -197,6 +204,7 @@ def setup_graph( @staticmethod def setup_graph_extended( options_bootstrapper: OptionsBootstrapper, + options_inputs_accessor: Callable[[], OptionsInputs], build_configuration: BuildConfiguration, execution_options: ExecutionOptions, native: Native, @@ -222,6 +230,11 @@ def setup_graph_extended( bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope() execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS + @_uncacheable_rule + def options_inputs_singleton() -> OptionsInputs: + # Computed once per session: see OptionsInputs for more information. + return options_inputs_accessor() + @rule def parser_singleton() -> Parser: return Parser( diff --git a/src/python/pants/option/options_bootstrapper.py b/src/python/pants/option/options_bootstrapper.py index 525f7c69fb5d..1b5f3da6804f 100644 --- a/src/python/pants/option/options_bootstrapper.py +++ b/src/python/pants/option/options_bootstrapper.py @@ -20,6 +20,19 @@ from pants.util.strutil import ensure_text +@dataclass(frozen=True) +class OptionsInputs: + """The inputs used to construct an OptionsBootstrapper. + + This type is provided by an uncacheable singleton @rule in production usage, which forces it to + be re-computed once per session. + """ + + args: List[str] + env: Dict[str, str] + allow_pantsrc: bool + + @dataclass(frozen=True) class OptionsBootstrapper: """Holds the result of the first stage of options parsing, and assists with parsing full diff --git a/src/python/pants/testutil/rule_runner.py b/src/python/pants/testutil/rule_runner.py index 9c5210b5f672..df1b3f397278 100644 --- a/src/python/pants/testutil/rule_runner.py +++ b/src/python/pants/testutil/rule_runner.py @@ -41,7 +41,7 @@ from pants.engine.unions import UnionMembership from pants.init.engine_initializer import EngineInitializer from pants.option.global_options import ExecutionOptions, GlobalOptions -from pants.option.options_bootstrapper import OptionsBootstrapper +from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs from pants.source import source_root from pants.testutil.option_util import create_options_bootstrapper from pants.util.collections import assert_single_element @@ -106,8 +106,13 @@ def __init__( build_config_builder.register_target_types(target_types or ()) self.build_config = build_config_builder.create() + options_inputs = OptionsInputs( + args=["--pants-config-files=[]"], env={}, allow_pantsrc=False + ) options_bootstrapper = OptionsBootstrapper.create( - env={}, args=["--pants-config-files=[]"], allow_pantsrc=False + args=options_inputs.args, + env=options_inputs.env, + allow_pantsrc=options_inputs.allow_pantsrc, ) global_options = options_bootstrapper.bootstrap_options.for_global_scope() local_store_dir = global_options.local_store_dir @@ -122,6 +127,7 @@ def __init__( named_caches_dir=named_caches_dir, native=Native(), options_bootstrapper=options_bootstrapper, + options_inputs_accessor=lambda: options_inputs, build_root=self.build_root, build_configuration=self.build_config, execution_options=ExecutionOptions.from_bootstrap_options(global_options), diff --git a/src/python/pants/testutil/test_base.py b/src/python/pants/testutil/test_base.py index c6896f75ac7d..5a85462f3596 100644 --- a/src/python/pants/testutil/test_base.py +++ b/src/python/pants/testutil/test_base.py @@ -42,7 +42,7 @@ from pants.init.engine_initializer import EngineInitializer from pants.init.util import clean_global_runtime_state from pants.option.global_options import ExecutionOptions, GlobalOptions -from pants.option.options_bootstrapper import OptionsBootstrapper +from pants.option.options_bootstrapper import OptionsBootstrapper, OptionsInputs from pants.option.subsystem import Subsystem from pants.source import source_root from pants.testutil.option_util import create_options_bootstrapper @@ -300,8 +300,13 @@ def _init_engine(self, local_store_dir: Optional[str] = None) -> None: if self._scheduler is not None: return + options_inputs = OptionsInputs( + args=["--pants-config-files=[]", *self.additional_options], env={}, allow_pantsrc=False + ) options_bootstrapper = OptionsBootstrapper.create( - env={}, args=["--pants-config-files=[]", *self.additional_options], allow_pantsrc=False + args=options_inputs.args, + env=options_inputs.env, + allow_pantsrc=options_inputs.allow_pantsrc, ) global_options = options_bootstrapper.bootstrap_options.for_global_scope() local_store_dir = local_store_dir or global_options.local_store_dir @@ -317,6 +322,7 @@ def _init_engine(self, local_store_dir: Optional[str] = None) -> None: ca_certs_path=global_options.ca_certs_path, native=Native(), options_bootstrapper=options_bootstrapper, + options_inputs_accessor=lambda: options_inputs, build_root=self.build_root, build_configuration=self.build_config(), execution_options=ExecutionOptions.from_bootstrap_options(global_options),