Allow turning off export of Python tools like black (Cherry-pick of

#15509) (#15511)

Allow turning off `export` of Python tools like `black` (#15509)

Closes #15502.

I considered adding an option to turn off export of _all_ tools via `[export].tools = false`, but the modeling didn't work out well. The generic `export` goal has zero idea what a "tool" is, and that's a Python abstraction. The requester of this feature (@kaos ) said he's happy with this to start.

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/python/goals/export.py

@@ -65,8 +65,13 @@ class ExportPythonToolSentinel:
 
 @dataclass(frozen=True)
 class ExportPythonTool(EngineAwareParameter):
+    """How to export a particular Python tool.
+
+    If `pex_request=None`, the tool will be skipped.
+    """
+
     resolve_name: str
-    pex_request: PexRequest
+    pex_request: PexRequest | None
 
     def debug_hint(self) -> str | None:
         return self.resolve_name
@@ -146,6 +151,8 @@ async def export_virtualenv(
 
 @rule
 async def export_tool(request: ExportPythonTool, pex_pex: PexPEX) -> ExportResult:
+    assert request.pex_request is not None
+
     # TODO: Unify export_virtualenv() and export_tool(), since their implementations mostly overlap.
     dest = os.path.join("python", "virtualenvs", "tools")
     pex = await Get(Pex, PexRequest, request.pex_request)
@@ -229,7 +236,9 @@ async def export_virtualenvs(
         for tool_export_type in tool_export_types
     )
     all_tool_results = await MultiGet(
-        Get(ExportResult, ExportPythonTool, request) for request in all_export_tool_requests
+        Get(ExportResult, ExportPythonTool, request)
+        for request in all_export_tool_requests
+        if request.pex_request is not None
     )
 
     return ExportResults(venvs + all_tool_results)

src/python/pants/backend/python/lint/autoflake/subsystem.py

@@ -6,7 +6,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
@@ -34,6 +34,7 @@ class Autoflake(PythonToolBase):
 
     skip = SkipOption("fmt", "lint")
     args = ArgsListOption(example="--target-version=py37 --quiet")
+    export = ExportToolOption()
 
 
 class AutoflakeLockfileSentinel(GenerateToolLockfileSentinel):
@@ -55,6 +56,8 @@ class AutoflakeExportSentinel(ExportPythonToolSentinel):
 
 @rule
 def autoflake_export(_: AutoflakeExportSentinel, autoflake: Autoflake) -> ExportPythonTool:
+    if not autoflake.export:
+        return ExportPythonTool(resolve_name=autoflake.options_scope, pex_request=None)
     return ExportPythonTool(
         resolve_name=autoflake.options_scope, pex_request=autoflake.to_pex_request()
     )

src/python/pants/backend/python/lint/bandit/subsystem.py

@@ -10,7 +10,7 @@
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.lint.bandit.skip_field import SkipBanditField
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     ConsoleScript,
@@ -64,6 +64,7 @@ class Bandit(PythonToolBase):
 
     skip = SkipOption("lint")
     args = ArgsListOption(example="--skip B101,B308 --confidence")
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -138,6 +139,8 @@ class BanditExportSentinel(ExportPythonToolSentinel):
 async def bandit_export(
     _: BanditExportSentinel, bandit: Bandit, python_setup: PythonSetup
 ) -> ExportPythonTool:
+    if not bandit.export:
+        return ExportPythonTool(resolve_name=bandit.options_scope, pex_request=None)
     constraints = await _bandit_interpreter_constraints(python_setup)
     return ExportPythonTool(
         resolve_name=bandit.options_scope,

src/python/pants/backend/python/lint/black/subsystem.py

@@ -10,7 +10,7 @@
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.lint.black.skip_field import SkipBlackField
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
@@ -43,6 +43,7 @@ class Black(PythonToolBase):
 
     skip = SkipOption("fmt", "lint")
     args = ArgsListOption(example="--target-version=py37 --quiet")
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -119,10 +120,12 @@ class BlackExportSentinel(ExportPythonToolSentinel):
     pass
 
 
-@rule(desc="Determine MyPy interpreter constraints (for `export` goal)", level=LogLevel.DEBUG)
+@rule(desc="Determine Black interpreter constraints (for `export` goal)", level=LogLevel.DEBUG)
 async def black_export(
     _: BlackExportSentinel, black: Black, python_setup: PythonSetup
 ) -> ExportPythonTool:
+    if not black.export:
+        return ExportPythonTool(resolve_name=black.options_scope, pex_request=None)
     constraints = await _black_interpreter_constraints(black, python_setup)
     return ExportPythonTool(
         resolve_name=black.options_scope,

src/python/pants/backend/python/lint/docformatter/subsystem.py

@@ -5,7 +5,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
@@ -33,6 +33,7 @@ class Docformatter(PythonToolBase):
 
     skip = SkipOption("fmt", "lint")
     args = ArgsListOption(example="--wrap-summaries=100 --pre-summary-newline")
+    export = ExportToolOption()
 
 
 class DocformatterLockfileSentinel(GenerateToolLockfileSentinel):
@@ -56,6 +57,8 @@ class DocformatterExportSentinel(ExportPythonToolSentinel):
 def docformatter_export(
     _: DocformatterExportSentinel, docformatter: Docformatter
 ) -> ExportPythonTool:
+    if not docformatter.export:
+        return ExportPythonTool(resolve_name=docformatter.options_scope, pex_request=None)
     return ExportPythonTool(
         resolve_name=docformatter.options_scope, pex_request=docformatter.to_pex_request()
     )

src/python/pants/backend/python/lint/flake8/subsystem.py

@@ -10,7 +10,7 @@
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.lint.flake8.skip_field import SkipFlake8Field
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     ConsoleScript,
@@ -80,6 +80,7 @@ class Flake8(PythonToolBase):
 
     skip = SkipOption("lint")
     args = ArgsListOption(example="--ignore E123,W456 --enable-extensions H111")
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -320,6 +321,8 @@ async def flake8_export(
     first_party_plugins: Flake8FirstPartyPlugins,
     python_setup: PythonSetup,
 ) -> ExportPythonTool:
+    if not flake8.export:
+        return ExportPythonTool(resolve_name=flake8.options_scope, pex_request=None)
     constraints = await _flake8_interpreter_constraints(first_party_plugins, python_setup)
     return ExportPythonTool(
         resolve_name=flake8.options_scope,

src/python/pants/backend/python/lint/isort/subsystem.py

@@ -9,7 +9,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
@@ -38,6 +38,7 @@ class Isort(PythonToolBase):
 
     skip = SkipOption("fmt", "lint")
     args = ArgsListOption(example="--case-sensitive --trailing-comma")
+    export = ExportToolOption()
     config = FileListOption(
         "--config",
         # TODO: Figure out how to deprecate this being a list in favor of a single string.
@@ -112,6 +113,8 @@ class IsortExportSentinel(ExportPythonToolSentinel):
 
 @rule
 def isort_export(_: IsortExportSentinel, isort: Isort) -> ExportPythonTool:
+    if not isort.export:
+        return ExportPythonTool(resolve_name=isort.options_scope, pex_request=None)
     return ExportPythonTool(resolve_name=isort.options_scope, pex_request=isort.to_pex_request())
 
 

src/python/pants/backend/python/lint/pylint/subsystem.py

@@ -12,7 +12,7 @@
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.lint.pylint.skip_field import SkipPylintField
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     ConsoleScript,
@@ -89,6 +89,7 @@ class Pylint(PythonToolBase):
 
     skip = SkipOption("lint")
     args = ArgsListOption(example="--ignore=foo.py,bar.py --disable=C0330,W0311")
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -330,6 +331,8 @@ async def pylint_export(
     first_party_plugins: PylintFirstPartyPlugins,
     python_setup: PythonSetup,
 ) -> ExportPythonTool:
+    if not pylint.export:
+        return ExportPythonTool(resolve_name=pylint.options_scope, pex_request=None)
     constraints = await _pylint_interpreter_constraints(first_party_plugins, python_setup)
     return ExportPythonTool(
         resolve_name=pylint.options_scope,

src/python/pants/backend/python/lint/pyupgrade/subsystem.py

@@ -6,7 +6,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
@@ -36,6 +36,7 @@ class PyUpgrade(PythonToolBase):
 
     skip = SkipOption("fmt", "lint")
     args = ArgsListOption(example="--py39-plus --keep-runtime-typing")
+    export = ExportToolOption()
 
 
 class PyUpgradeLockfileSentinel(GenerateToolLockfileSentinel):
@@ -57,6 +58,8 @@ class PyUpgradeExportSentinel(ExportPythonToolSentinel):
 
 @rule
 def pyupgrade_export(_: PyUpgradeExportSentinel, pyupgrade: PyUpgrade) -> ExportPythonTool:
+    if not pyupgrade.export:
+        return ExportPythonTool(resolve_name=pyupgrade.options_scope, pex_request=None)
     return ExportPythonTool(
         resolve_name=pyupgrade.options_scope, pex_request=pyupgrade.to_pex_request()
     )

src/python/pants/backend/python/lint/yapf/subsystem.py

@@ -9,7 +9,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import ConsoleScript
 from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
@@ -44,6 +44,7 @@ class Yapf(PythonToolBase):
         "`--parallel`, will be ignored because Pants takes care of finding "
         "all the relevant files and running the formatting in parallel.",
     )
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -107,6 +108,8 @@ class YapfExportSentinel(ExportPythonToolSentinel):
 
 @rule
 def yapf_export(_: YapfExportSentinel, yapf: Yapf) -> ExportPythonTool:
+    if not yapf.export:
+        return ExportPythonTool(resolve_name=yapf.options_scope, pex_request=None)
     return ExportPythonTool(resolve_name=yapf.options_scope, pex_request=yapf.to_pex_request())
 
 

src/python/pants/backend/python/subsystems/pytest.py

@@ -14,7 +14,7 @@
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
 from pants.backend.python.pip_requirement import PipRequirement
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     ConsoleScript,
@@ -145,6 +145,8 @@ class PyTest(PythonToolBase):
         ),
     )
 
+    export = ExportToolOption()
+
     @property
     def all_requirements(self) -> tuple[str, ...]:
         return (self.version, *self.extra_requirements)
@@ -251,6 +253,8 @@ class PytestExportSentinel(ExportPythonToolSentinel):
 async def pytest_export(
     _: PytestExportSentinel, pytest: PyTest, python_setup: PythonSetup
 ) -> ExportPythonTool:
+    if not pytest.export:
+        return ExportPythonTool(resolve_name=pytest.options_scope, pex_request=None)
     constraints = await _pytest_interpreter_constraints(python_setup)
     return ExportPythonTool(
         resolve_name=pytest.options_scope,

src/python/pants/backend/python/subsystems/python_tool_base.py

@@ -19,7 +19,7 @@
 from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest
 from pants.engine.fs import Digest, FileContent
 from pants.option.errors import OptionsError
-from pants.option.option_types import StrListOption, StrOption
+from pants.option.option_types import BoolOption, StrListOption, StrOption
 from pants.option.subsystem import Subsystem
 from pants.util.docutil import bin_name, doc_url
 from pants.util.strutil import softwrap
@@ -286,3 +286,25 @@ def to_pex_request(
             main=main or self.main,
             sources=sources,
         )
+
+
+class ExportToolOption(BoolOption):
+    """An `--export` option to toggle whether the `export` goal should include the tool."""
+
+    def __new__(cls):
+        return super().__new__(
+            cls,
+            "--export",
+            default=True,
+            help=(
+                lambda subsystem_cls: softwrap(
+                    f"""
+                    If true, export a virtual environment with {subsystem_cls.name} when running
+                    `{bin_name()} export`.
+
+                    This can be useful, for example, with IDE integrations to point your editor to
+                    the tool's binary.
+                    """
+                )
+            ),
+        )

src/python/pants/backend/python/typecheck/mypy/subsystem.py

@@ -11,7 +11,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.goals.export import ExportPythonTool, ExportPythonToolSentinel
 from pants.backend.python.goals.lockfile import GeneratePythonLockfile
-from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     ConsoleScript,
@@ -93,6 +93,7 @@ class MyPy(PythonToolBase):
 
     skip = SkipOption("check")
     args = ArgsListOption(example="--python-version 3.7 --disallow-any-expr")
+    export = ExportToolOption()
     config = FileOption(
         "--config",
         default=None,
@@ -355,6 +356,8 @@ async def mypy_export(
     python_setup: PythonSetup,
     first_party_plugins: MyPyFirstPartyPlugins,
 ) -> ExportPythonTool:
+    if not mypy.export:
+        return ExportPythonTool(resolve_name=mypy.options_scope, pex_request=None)
     constraints = await _mypy_interpreter_constraints(mypy, python_setup)
     return ExportPythonTool(
         resolve_name=mypy.options_scope,