Deprecate [python].only_binary and [python].no_binary in favor of…

… more powerful `[python].resolves_to_only_binary` and `[python].resolves_to_no_binary` (pantsbuild#16513)

Part of the per-resolve config project at https://docs.google.com/document/d/1HAvpSNvNAHreFfvTAXavZGka-A3WWvPuH0sMjGUCo48/edit. 

We already with multiple resolves allow you to have conflicting versions of the same requirement, e.g. Django 2 vs Django 3. So, it's useful to also allow those resolves to set different options for `--no-binary` and `--only-binary`, as you might only need it for certain versions of a project or for certain contexts.

This only works with Pex lockfiles, with similar reasoning to why we closed pantsbuild#16476.

This adds the options to the lockfile header, making progress on pantsbuild#12832.

src/python/pants/backend/python/goals/lockfile.py

@@ -159,21 +159,22 @@ class _PipArgsAndConstraintsSetup:
     args: tuple[str, ...]
     digest: Digest
     constraints: FrozenOrderedSet[PipRequirement]
+    only_binary: tuple[str, ...]
+    no_binary: tuple[str, ...]
 
 
 @rule_helper
-async def _setup_pip_args_and_constraints_file(
-    python_setup: PythonSetup, *, resolve_name: str
-) -> _PipArgsAndConstraintsSetup:
+async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAndConstraintsSetup:
     args = []
     digests = []
+    resolve_config = await Get(ResolvePexConfig, ResolvePexConfigRequest(resolve_name))
 
-    if python_setup.no_binary or python_setup.only_binary:
+    if resolve_config.no_binary or resolve_config.only_binary:
         pip_args_file = "__pip_args.txt"
         args.extend(["-r", pip_args_file])
         pip_args_file_content = "\n".join(
-            [f"--no-binary {pkg}" for pkg in python_setup.no_binary]
-            + [f"--only-binary {pkg}" for pkg in python_setup.only_binary]
+            [f"--no-binary {pkg}" for pkg in resolve_config.no_binary]
+            + [f"--only-binary {pkg}" for pkg in resolve_config.only_binary]
         )
         pip_args_digest = await Get(
             Digest, CreateDigest([FileContent(pip_args_file, pip_args_file_content.encode())])
@@ -188,7 +189,13 @@ async def _setup_pip_args_and_constraints_file(
         constraints = resolve_config.constraints_file.constraints
 
     input_digest = await Get(Digest, MergeDigests(digests))
-    return _PipArgsAndConstraintsSetup(tuple(args), input_digest, constraints)
+    return _PipArgsAndConstraintsSetup(
+        tuple(args),
+        input_digest,
+        constraints,
+        only_binary=resolve_config.only_binary,
+        no_binary=resolve_config.no_binary,
+    )
 
 
 @rule(desc="Generate Python lockfile", level=LogLevel.DEBUG)
@@ -200,12 +207,14 @@ async def generate_lockfile(
     python_setup: PythonSetup,
 ) -> GenerateLockfileResult:
     requirement_constraints: FrozenOrderedSet[PipRequirement] = FrozenOrderedSet()
+    only_binary: tuple[str, ...] = ()
+    no_binary: tuple[str, ...] = ()
 
     if req.use_pex:
-        pip_args_setup = await _setup_pip_args_and_constraints_file(
-            python_setup, resolve_name=req.resolve_name
-        )
+        pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name)
         requirement_constraints = pip_args_setup.constraints
+        only_binary = pip_args_setup.only_binary
+        no_binary = pip_args_setup.no_binary
 
         header_delimiter = "//"
         result = await Get(
@@ -309,6 +318,8 @@ async def generate_lockfile(
         valid_for_interpreter_constraints=req.interpreter_constraints,
         requirements={PipRequirement.parse(i) for i in req.requirements},
         requirement_constraints=set(requirement_constraints),
+        only_binary=set(only_binary),
+        no_binary=set(no_binary),
     )
     lockfile_with_header = metadata.add_header_to_lockfile(
         initial_lockfile_digest_contents[0].content,

src/python/pants/backend/python/goals/lockfile_test.py

@@ -14,7 +14,7 @@
 )
 from pants.backend.python.goals.lockfile import rules as lockfile_rules
 from pants.backend.python.goals.lockfile import setup_user_lockfile_requests
-from pants.backend.python.subsystems.setup import PythonSetup
+from pants.backend.python.subsystems.setup import RESOLVE_OPTION_KEY__DEFAULT, PythonSetup
 from pants.backend.python.target_types import PythonRequirementTarget
 from pants.backend.python.util_rules import pex
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
@@ -110,11 +110,12 @@ def test_poetry_lockfile_generation(rule_runner: RuleRunner) -> None:
 def test_pex_lockfile_generation(
     rule_runner: RuleRunner, no_binary: bool, only_binary: bool
 ) -> None:
-    args = []
+    args = ["--python-resolves={'test': 'foo.lock'}"]
+    no_binary_arg = f"{{'{RESOLVE_OPTION_KEY__DEFAULT}': ['ansicolors']}}"
     if no_binary:
-        args.append("--python-no-binary=ansicolors")
+        args.append(f"--python-resolves-to-no-binary={no_binary_arg}")
     if only_binary:
-        args.append("--python-only-binary=ansicolors")
+        args.append(f"--python-resolves-to-only-binary={no_binary_arg}")
     rule_runner.set_options(args, env_inherit=PYTHON_BOOTSTRAP_ENV)
 
     lock_entry = json.loads(_generate(rule_runner=rule_runner, use_pex=True))
@@ -161,7 +162,7 @@ def test_constraints_file(rule_runner: RuleRunner) -> None:
     rule_runner.set_options(
         [
             "--python-resolves={'test': 'foo.lock'}",
-            "--python-resolves-to-constraints-file={'test': 'constraints.txt'}",
+            f"--python-resolves-to-constraints-file={{'{RESOLVE_OPTION_KEY__DEFAULT}': 'constraints.txt'}}",
         ],
         env_inherit=PYTHON_BOOTSTRAP_ENV,
     )

src/python/pants/backend/python/subsystems/setup.py

@@ -6,7 +6,7 @@
 import enum
 import logging
 import os
-from typing import Iterable, Iterator, Optional, cast
+from typing import Iterable, Iterator, List, Optional, TypeVar, cast
 
 from pants.core.goals.generate_lockfiles import UnrecognizedResolveNamesError
 from pants.option.option_types import (
@@ -38,6 +38,11 @@ class LockfileGenerator(enum.Enum):
     POETRY = "poetry"
 
 
+RESOLVE_OPTION_KEY__DEFAULT = "__default__"
+
+_T = TypeVar("_T")
+
+
 class PythonSetup(Subsystem):
     options_scope = "python"
     help = "Options for Pants's Python backend."
@@ -215,7 +220,7 @@ class PythonSetup(Subsystem):
     )
     _resolves_to_constraints_file = DictOption[str](
         help=softwrap(
-            """
+            f"""
             When generating a resolve's lockfile, use a constraints file to pin the version of
             certain requirements. This is particularly useful to pin the versions of transitive
             dependencies of your direct requirements.
@@ -226,10 +231,63 @@ class PythonSetup(Subsystem):
             Expects a dictionary of resolve names from `[python].resolves` and Python tools (e.g.
             `black` and `pytest`) to file paths for
             constraints files. For example,
-            `{'data-science': '3rdparty/data-science-constraints.txt'}`.
+            `{{'data-science': '3rdparty/data-science-constraints.txt'}}`.
             If a resolve is not set in the dictionary, it will not use a constraints file.
 
-            You can use the key `__default__` to set a default value for all resolves.
+            You can use the key `{RESOLVE_OPTION_KEY__DEFAULT}` to set a default value for all
+            resolves.
+
+            Note: Only takes effect if you use Pex lockfiles. Use the default
+            `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal.
+            """
+        ),
+        advanced=True,
+    )
+    _resolves_to_no_binary = DictOption[List[str]](
+        help=softwrap(
+            f"""
+            When generating a resolve's lockfile, do not use binary packages (i.e. wheels) for
+            these 3rdparty project names.
+
+            Expects a dictionary of resolve names from `[python].resolves` and Python tools (e.g.
+            `black` and `pytest`) to lists of project names. For example,
+            `{{'data-science': ['requests', 'numpy']}}`. If a resolve is not set in the dictionary,
+            it will have no restrictions on binary packages.
+
+            You can use the key `{RESOLVE_OPTION_KEY__DEFAULT}` to set a default value for all
+            resolves.
+
+            For each resolve's value, you can use the value `:all:` to disable all binary packages.
+
+            Note that some packages are tricky to compile and may fail to install when this option
+            is used on them. See https://pip.pypa.io/en/stable/cli/pip_install/#install-no-binary
+            for details.
+
+            Note: Only takes effect if you use Pex lockfiles. Use the default
+            `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal.
+            """
+        ),
+        advanced=True,
+    )
+    _resolves_to_only_binary = DictOption[List[str]](
+        help=softwrap(
+            f"""
+            When generating a resolve's lockfile, do not use source packages (i.e. sdists) for
+            these 3rdparty project names, e.g `['django', 'requests']`.
+
+            Expects a dictionary of resolve names from `[python].resolves` and Python tools (e.g.
+            `black` and `pytest`) to lists of project names. For example,
+            `{{'data-science': ['requests', 'numpy']}}`. If a resolve is not set in the dictionary,
+            it will have no restrictions on source packages.
+
+            You can use the key `{RESOLVE_OPTION_KEY__DEFAULT}` to set a default value for all
+            resolves.
+
+            For each resolve's value, you can use the value `:all:` to disable all source packages.
+
+            Packages without binary distributions will fail to install when this option is used on
+            them. See https://pip.pypa.io/en/stable/cli/pip_install/#install-only-binary for
+            details.
 
             Note: Only takes effect if you use Pex lockfiles. Use the default
             `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal.
@@ -423,6 +481,15 @@ class PythonSetup(Subsystem):
             `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal.
             """
         ),
+        removal_version="2.15.0.dev0",
+        removal_hint=softwrap(
+            f"""
+            Use `[python].resolves_to_no_binary`, which allows you to set `--no-binary` on a
+            per-resolve basis for more flexibility. To keep this option's behavior, set
+            `[python].resolves_to_no_binary` with the key `{RESOLVE_OPTION_KEY__DEFAULT}` and the
+            value you used on this option.
+            """
+        ),
     )
     only_binary = StrListOption(
         help=softwrap(
@@ -439,6 +506,15 @@ class PythonSetup(Subsystem):
             `[python].lockfile_generator = "pex"` and run the `generate-lockfiles` goal.
             """
         ),
+        removal_version="2.15.0.dev0",
+        removal_hint=softwrap(
+            f"""
+            Use `[python].resolves_to_only_binary`, which allows you to set `--only-binary` on a
+            per-resolve basis for more flexibility. To keep this option's behavior, set
+            `[python].resolves_to_only_binary` with the key `{RESOLVE_OPTION_KEY__DEFAULT}` and the
+            value you used on this option.
+            """
+        ),
     )
     resolver_manylinux = StrOption(
         default="manylinux2014",
@@ -562,28 +638,91 @@ def resolves_to_interpreter_constraints(self) -> dict[str, tuple[str, ...]]:
             )
         return result
 
-    @memoized_method
-    def resolves_to_constraints_file(
-        self, all_python_tool_resolve_names: tuple[str, ...]
-    ) -> dict[str, str]:
+    def _resolves_to_option_helper(
+        self,
+        option_value: dict[str, _T],
+        option_name: str,
+        all_python_tool_resolve_names: tuple[str, ...],
+    ) -> dict[str, _T]:
         all_valid_resolves = {*self.resolves, *all_python_tool_resolve_names}
-        unrecognized_resolves = set(self._resolves_to_constraints_file.keys()) - {
-            "__default__",
+        unrecognized_resolves = set(option_value.keys()) - {
+            RESOLVE_OPTION_KEY__DEFAULT,
             *all_valid_resolves,
         }
         if unrecognized_resolves:
             raise UnrecognizedResolveNamesError(
                 sorted(unrecognized_resolves),
-                all_valid_resolves,
-                description_of_origin="the option `[python].resolves_to_constraints_file`",
+                {*all_valid_resolves, RESOLVE_OPTION_KEY__DEFAULT},
+                description_of_origin=f"the option `[python].{option_name}`",
             )
-        default_val = self._resolves_to_constraints_file.get("__default__")
+        default_val = option_value.get(RESOLVE_OPTION_KEY__DEFAULT)
         if not default_val:
-            return self._resolves_to_constraints_file
-        return {
-            resolve: self._resolves_to_constraints_file.get(resolve, default_val)
-            for resolve in all_valid_resolves
-        }
+            return option_value
+        return {resolve: option_value.get(resolve, default_val) for resolve in all_valid_resolves}
+
+    @memoized_method
+    def resolves_to_constraints_file(
+        self, all_python_tool_resolve_names: tuple[str, ...]
+    ) -> dict[str, str]:
+        return self._resolves_to_option_helper(
+            self._resolves_to_constraints_file,
+            "resolves_to_constraints_file",
+            all_python_tool_resolve_names,
+        )
+
+    @memoized_method
+    def resolves_to_no_binary(
+        self, all_python_tool_resolve_names: tuple[str, ...]
+    ) -> dict[str, list[str]]:
+        if self.no_binary:
+            if self._resolves_to_no_binary:
+                raise ValueError(
+                    softwrap(
+                        """
+                        Conflicting options used. You used the new, preferred
+                        `[python].resolves_to_no_binary`, but also used the deprecated
+                        `[python].no_binary`.
+
+                        Please use only one of these (preferably `[python].resolves_to_no_binary`).
+                        """
+                    )
+                )
+            return {
+                resolve: list(self.no_binary)
+                for resolve in {*self.resolves, *all_python_tool_resolve_names}
+            }
+        return self._resolves_to_option_helper(
+            self._resolves_to_no_binary,
+            "resolves_to_no_binary",
+            all_python_tool_resolve_names,
+        )
+
+    @memoized_method
+    def resolves_to_only_binary(
+        self, all_python_tool_resolve_names: tuple[str, ...]
+    ) -> dict[str, list[str]]:
+        if self.only_binary:
+            if self._resolves_to_only_binary:
+                raise ValueError(
+                    softwrap(
+                        """
+                        Conflicting options used. You used the new, preferred
+                        `[python].resolves_to_only_binary`, but also used the deprecated
+                        `[python].only_binary`.
+
+                        Please use only one of these (preferably `[python].resolves_to_only_binary`).
+                        """
+                    )
+                )
+            return {
+                resolve: list(self.only_binary)
+                for resolve in {*self.resolves, *all_python_tool_resolve_names}
+            }
+        return self._resolves_to_option_helper(
+            self._resolves_to_only_binary,
+            "resolves_to_only_binary",
+            all_python_tool_resolve_names,
+        )
 
     def resolve_all_constraints_was_set_explicitly(self) -> bool:
         return not self.options.is_default("resolve_all_constraints")

src/python/pants/backend/python/subsystems/setup_test.py

@@ -39,3 +39,38 @@ def create(resolves_to_constraints_file: dict[str, str]) -> dict[str, str]:
     }
     with pytest.raises(UnrecognizedResolveNamesError):
         create({"fake": "c.txt"})
+
+
+def test_resolves_to_no_binary_and_only_binary() -> None:
+    def create(
+        resolves_to_projects: dict[str, list[str]], deprecated_options: list[str] | None = None
+    ) -> dict[str, list[str]]:
+        subsystem = create_subsystem(
+            PythonSetup,
+            resolves={"a": "a.lock"},
+            resolves_to_no_binary=resolves_to_projects,
+            resolves_to_only_binary=resolves_to_projects,
+            only_binary=deprecated_options or [],
+            no_binary=deprecated_options or [],
+        )
+        only_binary = subsystem.resolves_to_only_binary(
+            all_python_tool_resolve_names=("tool1", "tool2")
+        )
+        no_binary = subsystem.resolves_to_no_binary(
+            all_python_tool_resolve_names=("tool1", "tool2")
+        )
+        assert only_binary == no_binary
+        return only_binary
+
+    assert create({"a": ["p1"], "tool1": ["p2"]}) == {"a": ["p1"], "tool1": ["p2"]}
+    assert create({"__default__": ["p1"], "tool2": ["override"]}) == {
+        "a": ["p1"],
+        "tool1": ["p1"],
+        "tool2": ["override"],
+    }
+    with pytest.raises(UnrecognizedResolveNamesError):
+        create({"fake": []})
+
+    assert create({}, deprecated_options=["p1"]) == {"a": ["p1"], "tool1": ["p1"], "tool2": ["p1"]}
+    with pytest.raises(ValueError):
+        create({"a": ["p1"]}, deprecated_options=["p2"])