From 0ba8f1a63bfa9cde50e75f519a52c3a9634c3c9a Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sun, 8 May 2022 18:31:05 -0700 Subject: [PATCH 1/5] Support building dists using mypyc. Handles setting up mypy and any necessary config, requirements and type stubs, so that a preexisting setup.py can call mypycify(). Does not handle supporting mypyc in generated setup.py files. That will be provided in a future change. Also fixes a bug where we did not set the sys.path correctly when running a preexisting setup.py that imported code from a different source root. Also switches a test to use provides=python_artifact(), which is what we document, instead of the older alias setup_py(). [ci skip-rust] [ci skip-build-wheels] --- .../pants/backend/python/goals/setup_py.py | 71 +++++++++++++++- .../python/subsystems/setuptools_test.py | 22 ++--- .../backend/python/typecheck/mypy/mypyc.py | 83 +++++++++++++++++++ .../backend/python/typecheck/mypy/register.py | 3 +- .../pants/backend/python/util_rules/dists.py | 13 ++- 5 files changed, 173 insertions(+), 19 deletions(-) create mode 100644 src/python/pants/backend/python/typecheck/mypy/mypyc.py diff --git a/src/python/pants/backend/python/goals/setup_py.py b/src/python/pants/backend/python/goals/setup_py.py index 0dd9382a78e..72a5b4b0570 100644 --- a/src/python/pants/backend/python/goals/setup_py.py +++ b/src/python/pants/backend/python/goals/setup_py.py @@ -12,6 +12,7 @@ from collections import defaultdict from dataclasses import dataclass from functools import partial +from pathlib import PurePath from typing import Any, DefaultDict, Dict, List, Mapping, Tuple, cast from pants.backend.python.macros.python_artifact import PythonArtifact @@ -41,6 +42,7 @@ ) from pants.backend.python.util_rules.dists import rules as dists_rules from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.pex import Pex from pants.backend.python.util_rules.pex_requirements import PexRequirements from pants.backend.python.util_rules.python_sources import ( PythonSourceFiles, @@ -80,6 +82,7 @@ from pants.engine.unions import UnionMembership, UnionRule, union from pants.option.option_types import BoolOption, EnumOption from pants.option.subsystem import Subsystem +from pants.source.source_root import SourceRootsRequest, SourceRootsResult from pants.util.docutil import doc_url from pants.util.frozendict import FrozenDict from pants.util.logging import LogLevel @@ -370,10 +373,31 @@ class NoDistTypeSelected(ValueError): pass +@union +@dataclass(frozen=True) +class DistBuildEnvironmentRequest: + target_addresses: tuple[Address, ...] + interpreter_constraints: InterpreterConstraints + + @classmethod + def is_applicable(cls, tgt: Target) -> bool: + # Union members should override. + return False + + +@dataclass(frozen=True) +class DistBuildEnvironment: + """Various extra information that might be needed to build a dist.""" + + extra_build_time_requirements: tuple[Pex, ...] + extra_build_time_inputs: Digest + + @rule async def package_python_dist( field_set: PythonDistributionFieldSet, python_setup: PythonSetup, + union_membership: UnionMembership, ) -> BuiltPackage: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])) exported_target = ExportedTarget(transitive_targets.roots[0]) @@ -401,13 +425,52 @@ async def package_python_dist( ), ) + # Find the source roots for the build-time 1stparty deps (e.g., deps of setup.py). + source_roots_result = await Get( + SourceRootsResult, + SourceRootsRequest( + files=[], dirs={PurePath(tgt.address.spec_path) for tgt in transitive_targets.closure} + ), + ) + source_roots = tuple(sorted({sr.path for sr in source_roots_result.path_to_root.values()})) + + # Get any extra build-time environment (e.g., native extension requirements). + build_env_requests = [] + build_env_request_types = union_membership[DistBuildEnvironmentRequest] + for build_env_request_type in build_env_request_types: + if build_env_request_type.is_applicable(dist_tgt): + build_env_requests.append( + build_env_request_type( + tuple(tt.address for tt in transitive_targets.closure), interpreter_constraints + ) + ) + + build_envs = await MultiGet( + [ + Get(DistBuildEnvironment, DistBuildEnvironmentRequest, build_env_request) + for build_env_request in build_env_requests + ] + ) + extra_build_time_requirements = tuple( + itertools.chain.from_iterable( + build_env.extra_build_time_requirements for build_env in build_envs + ) + ) + input_digest = await Get( + Digest, + MergeDigests( + [chroot.digest, *(build_env.extra_build_time_inputs for build_env in build_envs)] + ), + ) + # We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture # any changes setup made within it without also capturing other artifacts of the pex # process invocation. chroot_prefix = "chroot" working_directory = os.path.join(chroot_prefix, chroot.working_directory) - prefixed_chroot = await Get(Digest, AddPrefix(chroot.digest, chroot_prefix)) - build_system = await Get(BuildSystem, BuildSystemRequest(prefixed_chroot, working_directory)) + prefixed_input = await Get(Digest, AddPrefix(input_digest, chroot_prefix)) + build_system = await Get(BuildSystem, BuildSystemRequest(prefixed_input, working_directory)) + setup_py_result = await Get( DistBuildResult, DistBuildRequest( @@ -415,11 +478,13 @@ async def package_python_dist( interpreter_constraints=interpreter_constraints, build_wheel=wheel, build_sdist=sdist, - input=prefixed_chroot, + input=prefixed_input, working_directory=working_directory, + build_time_source_roots=source_roots, target_address_spec=exported_target.target.address.spec, wheel_config_settings=wheel_config_settings, sdist_config_settings=sdist_config_settings, + extra_build_time_requirements=extra_build_time_requirements, ), ) dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output) diff --git a/src/python/pants/backend/python/subsystems/setuptools_test.py b/src/python/pants/backend/python/subsystems/setuptools_test.py index bbd63463210..6dec7ac8bd5 100644 --- a/src/python/pants/backend/python/subsystems/setuptools_test.py +++ b/src/python/pants/backend/python/subsystems/setuptools_test.py @@ -23,7 +23,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: QueryRule(GeneratePythonLockfile, [SetuptoolsLockfileSentinel]), ], target_types=[PythonSourcesGeneratorTarget, PythonDistribution], - objects={"setup_py": PythonArtifact}, + objects={"python_artifact": PythonArtifact}, ) global_constraint = "==3.9.*" @@ -49,7 +49,7 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist", dependencies=[":lib"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), @@ -62,7 +62,7 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist", dependencies=[":lib"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), @@ -75,7 +75,7 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist", dependencies=[":lib"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), @@ -94,14 +94,14 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist1", dependencies=[":lib1"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) python_sources(name="lib2", interpreter_constraints=["==3.5.*"]) python_distribution( name="dist2", dependencies=[":lib2"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), @@ -114,14 +114,14 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist1", dependencies=[":lib1"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) python_sources(name="lib2", interpreter_constraints=[">=3.5"]) python_distribution( name="dist2", dependencies=[":lib2"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), @@ -134,21 +134,21 @@ def assert_ics(build_file: str, expected: list[str]) -> None: python_distribution( name="dist1", dependencies=[":lib1"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) python_sources(name="lib2", interpreter_constraints=["==2.7.*"]) python_distribution( name="dist2", dependencies=[":lib2"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) python_sources(name="lib3", interpreter_constraints=[">=3.6"]) python_distribution( name="dist3", dependencies=[":lib3"], - provides=setup_py(name="dist"), + provides=python_artifact(name="dist"), ) """ ), diff --git a/src/python/pants/backend/python/typecheck/mypy/mypyc.py b/src/python/pants/backend/python/typecheck/mypy/mypyc.py new file mode 100644 index 00000000000..567c134feb4 --- /dev/null +++ b/src/python/pants/backend/python/typecheck/mypy/mypyc.py @@ -0,0 +1,83 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from dataclasses import dataclass + +from pants.backend.python.goals.setup_py import DistBuildEnvironment, DistBuildEnvironmentRequest +from pants.backend.python.target_types import PythonDistribution +from pants.backend.python.typecheck.mypy.subsystem import ( + MyPy, + MyPyConfigFile, + MyPyFirstPartyPlugins, +) +from pants.backend.python.util_rules import pex_from_targets +from pants.backend.python.util_rules.pex import Pex, PexRequest +from pants.backend.python.util_rules.pex_from_targets import RequirementsPexRequest +from pants.backend.python.util_rules.pex_requirements import PexRequirements +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import BoolField, Target +from pants.engine.unions import UnionRule + + +class UsesMyPycField(BoolField): + alias = "uses_mypyc" + default = False + help = "If true, this distribution is built using mypyc." + + +@dataclass(frozen=True) +class MyPycDistBuildEnvironmentRequest(DistBuildEnvironmentRequest): + @classmethod + def is_applicable(cls, tgt: Target) -> bool: + return tgt.get(UsesMyPycField).value + + +@rule(desc="Get mypyc build environment") +async def get_mypyc_build_environment( + request: MyPycDistBuildEnvironmentRequest, + first_party_plugins: MyPyFirstPartyPlugins, + mypy_config_file: MyPyConfigFile, + mypy: MyPy, +) -> DistBuildEnvironment: + mypy_pex_get = Get( + Pex, + PexRequest, + mypy.to_pex_request( + interpreter_constraints=request.interpreter_constraints, + extra_requirements=first_party_plugins.requirement_strings, + ), + ) + requirements_pex_get = Get( + Pex, + RequirementsPexRequest( + addresses=request.target_addresses, + hardcoded_interpreter_constraints=request.interpreter_constraints, + ), + ) + extra_type_stubs_pex_get = Get( + Pex, + PexRequest( + output_filename="extra_type_stubs.pex", + internal_only=True, + requirements=PexRequirements(mypy.extra_type_stubs), + interpreter_constraints=request.interpreter_constraints, + ), + ) + (mypy_pex, requirements_pex, extra_type_stubs_pex) = await MultiGet( + mypy_pex_get, requirements_pex_get, extra_type_stubs_pex_get + ) + return DistBuildEnvironment( + extra_build_time_requirements=(mypy_pex, requirements_pex, extra_type_stubs_pex), + extra_build_time_inputs=mypy_config_file.digest, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(DistBuildEnvironmentRequest, MyPycDistBuildEnvironmentRequest), + PythonDistribution.register_plugin_field(UsesMyPycField), + *pex_from_targets.rules(), + ] diff --git a/src/python/pants/backend/python/typecheck/mypy/register.py b/src/python/pants/backend/python/typecheck/mypy/register.py index 7ef1bff252d..119898bab1f 100644 --- a/src/python/pants/backend/python/typecheck/mypy/register.py +++ b/src/python/pants/backend/python/typecheck/mypy/register.py @@ -7,9 +7,10 @@ https://mypy.readthedocs.io/en/stable/. """ +from pants.backend.python.typecheck.mypy import mypyc from pants.backend.python.typecheck.mypy import rules as mypy_rules from pants.backend.python.typecheck.mypy import skip_field, subsystem def rules(): - return (*mypy_rules.rules(), *skip_field.rules(), *subsystem.rules()) + return (*mypy_rules.rules(), *mypyc.rules(), *skip_field.rules(), *subsystem.rules()) diff --git a/src/python/pants/backend/python/util_rules/dists.py b/src/python/pants/backend/python/util_rules/dists.py index 1f249be5b40..ef987673d45 100644 --- a/src/python/pants/backend/python/util_rules/dists.py +++ b/src/python/pants/backend/python/util_rules/dists.py @@ -14,7 +14,7 @@ from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.subsystems.setuptools import Setuptools from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess +from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexProcess from pants.backend.python.util_rules.pex import rules as pex_rules from pants.backend.python.util_rules.pex_requirements import EntireLockfile, PexRequirements from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior @@ -116,11 +116,14 @@ class DistBuildRequest: build_sdist: bool input: Digest working_directory: str # Relpath within the input digest. + build_time_source_roots: tuple[str, ...] # Source roots for 1st party build-time deps. target_address_spec: str | None = None # Only needed for logging etc. wheel_config_settings: FrozenDict[str, tuple[str, ...]] | None = None sdist_config_settings: FrozenDict[str, tuple[str, ...]] | None = None + extra_build_time_requirements: tuple[Pex, ...] = tuple() + @dataclass(frozen=True) class DistBuildResult: @@ -191,6 +194,7 @@ async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) output_filename="build_backend.pex", internal_only=True, requirements=request.build_system.requires, + pex_path=request.extra_build_time_requirements, interpreter_constraints=request.interpreter_constraints, ), ) @@ -209,10 +213,11 @@ async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) merged_digest = await Get(Digest, MergeDigests((request.input, backend_shim_digest))) + extra_env = { + "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots), + } if python_setup.macos_big_sur_compatibility and is_macos_big_sur(): - extra_env = {"MACOSX_DEPLOYMENT_TARGET": "10.16"} - else: - extra_env = {} + extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16" result = await Get( ProcessResult, From ed9d5b931dbac0f495162488c074fc03861b0905 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Mon, 9 May 2022 20:51:28 -0700 Subject: [PATCH 2/5] Add a test # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/typecheck/mypy/BUILD | 8 ++++ .../backend/python/typecheck/mypy/mypyc.py | 2 +- .../typecheck/mypy/mypyc_integration_test.py | 45 +++++++++++++++++++ testprojects/src/python/BUILD | 2 + testprojects/src/python/mypyc_fib/BUILD | 15 +++++++ .../python/mypyc_fib/mypyc_fib/__init__.py | 0 .../src/python/mypyc_fib/mypyc_fib/fib.py | 22 +++++++++ .../src/python/mypyc_fib/pyproject.toml | 7 +++ testprojects/src/python/mypyc_fib/setup.py | 18 ++++++++ 9 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py create mode 100644 testprojects/src/python/mypyc_fib/BUILD create mode 100644 testprojects/src/python/mypyc_fib/mypyc_fib/__init__.py create mode 100644 testprojects/src/python/mypyc_fib/mypyc_fib/fib.py create mode 100644 testprojects/src/python/mypyc_fib/pyproject.toml create mode 100644 testprojects/src/python/mypyc_fib/setup.py diff --git a/src/python/pants/backend/python/typecheck/mypy/BUILD b/src/python/pants/backend/python/typecheck/mypy/BUILD index c09c07a8267..df3562b61ba 100644 --- a/src/python/pants/backend/python/typecheck/mypy/BUILD +++ b/src/python/pants/backend/python/typecheck/mypy/BUILD @@ -12,3 +12,11 @@ python_tests( # We want to make sure the default lockfile works for both macOS and Linux. tags=["platform_specific_behavior"], ) +python_tests( + name="mypyc_integration_test", + sources=["mypyc_integration_test.py"], + dependencies=["testprojects/src/python:mypyc_fib_directory"], + timeout=120, + # We want to make sure we can build with mypyc on both macOS and Linux. + tags=["platform_specific_behavior"], +) diff --git a/src/python/pants/backend/python/typecheck/mypy/mypyc.py b/src/python/pants/backend/python/typecheck/mypy/mypyc.py index 567c134feb4..fba7c874116 100644 --- a/src/python/pants/backend/python/typecheck/mypy/mypyc.py +++ b/src/python/pants/backend/python/typecheck/mypy/mypyc.py @@ -1,4 +1,4 @@ -# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations diff --git a/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py b/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py new file mode 100644 index 00000000000..6e95cd136a4 --- /dev/null +++ b/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py @@ -0,0 +1,45 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import os +import subprocess +import sys +import venv +from tempfile import TemporaryDirectory + +from pants.testutil.pants_integration_test import run_pants + + +def test_mypyc_build() -> None: + dist_dir = "dist" + pyver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + + pants_run = run_pants( + [ + "--backend-packages=['pants.backend.python','pants.backend.python.typecheck.mypy']", + f"--python-interpreter-constraints=['=={pyver}']", + "package", + "testprojects/src/python/mypyc_fib:dist", + ], + extra_env={"PYTHON": sys.executable}, + ) + pants_run.assert_success() + wheels = os.listdir(dist_dir) + assert len(wheels) == 1 + wheel = os.path.join(dist_dir, wheels[0]) + + with TemporaryDirectory() as venvdir: + venv.create(venvdir, with_pip=True, clear=True, symlinks=True) + subprocess.run([os.path.join(venvdir, "bin", "pip"), "install", wheel], check=True) + proc = subprocess.run( + [ + os.path.join(venvdir, "bin", "python"), + "-c", + "import mypyc_fib.fib", + ], + check=True, + capture_output=True, + ) + assert proc.stdout.splitlines(keepends=False)[0] == b"compiled" diff --git a/testprojects/src/python/BUILD b/testprojects/src/python/BUILD index 13eb5fb8824..6515ebe2f03 100644 --- a/testprojects/src/python/BUILD +++ b/testprojects/src/python/BUILD @@ -9,6 +9,8 @@ files(name="coordinated_runs_directory", sources=["coordinated_runs/**/*"]) files(name="hello_directory", sources=["hello/**/*"]) +files(name="mypyc_fib_directory", sources=["mypyc_fib/**/*"]) + files(name="native_directory", sources=["native/**/*"]) files(name="no_build_file_directory", sources=["no_build_file/**/*"]) diff --git a/testprojects/src/python/mypyc_fib/BUILD b/testprojects/src/python/mypyc_fib/BUILD new file mode 100644 index 00000000000..5ebc7eea28f --- /dev/null +++ b/testprojects/src/python/mypyc_fib/BUILD @@ -0,0 +1,15 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_sources(name="lib", sources=["setup.py", "mypyc_fib/*.py"]) + +resources(name="pyproject", sources=["pyproject.toml"]) + +python_distribution( + name="dist", + dependencies=[":pyproject", ":lib"], + generate_setup=False, + provides=python_artifact(name="mypyc_fib", version="2.3.4"), + sdist=False, + uses_mypyc=True, +) diff --git a/testprojects/src/python/mypyc_fib/mypyc_fib/__init__.py b/testprojects/src/python/mypyc_fib/mypyc_fib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testprojects/src/python/mypyc_fib/mypyc_fib/fib.py b/testprojects/src/python/mypyc_fib/mypyc_fib/fib.py new file mode 100644 index 00000000000..1a56e523656 --- /dev/null +++ b/testprojects/src/python/mypyc_fib/mypyc_fib/fib.py @@ -0,0 +1,22 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# Adapted from https://mypyc.readthedocs.io/en/latest/getting_started.html#example-program + +import time + + +def fib(n: int) -> int: + if n <= 1: + return n + else: + return fib(n - 2) + fib(n - 1) + + +t0 = time.time() +fib(32) +if "__file__" in locals(): + print("interpreted") +else: + print("compiled") +print(time.time() - t0) diff --git a/testprojects/src/python/mypyc_fib/pyproject.toml b/testprojects/src/python/mypyc_fib/pyproject.toml new file mode 100644 index 00000000000..af2bad2bb7a --- /dev/null +++ b/testprojects/src/python/mypyc_fib/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] + +requires = [ + "setuptools==57.5.0", + "wheel==0.37.0", +] +build-backend = "setuptools.build_meta" diff --git a/testprojects/src/python/mypyc_fib/setup.py b/testprojects/src/python/mypyc_fib/setup.py new file mode 100644 index 00000000000..2508ac4de3f --- /dev/null +++ b/testprojects/src/python/mypyc_fib/setup.py @@ -0,0 +1,18 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +# Note that we don't want to infer a dep on mypyc, as it's provided by the dist build system due +# to uses_mypyc=True in the BUILD file. We therefore also ignore typechecking in this file. + +# type: ignore + +from mypyc.build import mypycify # pants: no-infer-dep +from setuptools import setup + +setup( + name="mypyc_fib", + version="2.3.4", + packages=["mypyc_fib"], + ext_modules=mypycify(["mypyc_fib/__init__.py", "mypyc_fib/fib.py"]), + description="Proof that mypyc compilation works", +) From 07c3e415595b02538f1ffabbda6c613fe4c4ac3b Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Mon, 9 May 2022 22:12:30 -0700 Subject: [PATCH 3/5] Fix tests # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/goals/setup_py.py | 2 +- src/python/pants/backend/python/subsystems/setuptools_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/goals/setup_py.py b/src/python/pants/backend/python/goals/setup_py.py index 72a5b4b0570..b68d38f7cd4 100644 --- a/src/python/pants/backend/python/goals/setup_py.py +++ b/src/python/pants/backend/python/goals/setup_py.py @@ -436,7 +436,7 @@ async def package_python_dist( # Get any extra build-time environment (e.g., native extension requirements). build_env_requests = [] - build_env_request_types = union_membership[DistBuildEnvironmentRequest] + build_env_request_types = union_membership.get(DistBuildEnvironmentRequest) for build_env_request_type in build_env_request_types: if build_env_request_type.is_applicable(dist_tgt): build_env_requests.append( diff --git a/src/python/pants/backend/python/subsystems/setuptools_test.py b/src/python/pants/backend/python/subsystems/setuptools_test.py index 6dec7ac8bd5..62ae988b869 100644 --- a/src/python/pants/backend/python/subsystems/setuptools_test.py +++ b/src/python/pants/backend/python/subsystems/setuptools_test.py @@ -40,7 +40,7 @@ def assert_ics(build_file: str, expected: list[str]) -> None: assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) # If no dependencies for python_distribution, fall back to global [python] constraints. - assert_ics("python_distribution(provides=setup_py(name='dist'))", [global_constraint]) + assert_ics("python_distribution(provides=python_artifact(name='dist'))", [global_constraint]) assert_ics( dedent( From fe201ec0b5761ef2b9b37b00504c1acde6311a3a Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 10 May 2022 12:10:28 -0700 Subject: [PATCH 4/5] Fix test. Also make the old native code setup.py test run on macos CI. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/goals/BUILD | 2 ++ .../pants/backend/python/goals/setup_py_integration_test.py | 3 +++ .../backend/python/typecheck/mypy/mypyc_integration_test.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/src/python/pants/backend/python/goals/BUILD b/src/python/pants/backend/python/goals/BUILD index a66634e274e..3d3150aa960 100644 --- a/src/python/pants/backend/python/goals/BUILD +++ b/src/python/pants/backend/python/goals/BUILD @@ -52,6 +52,8 @@ python_tests( name="setup_py_integration", sources=["setup_py_integration_test.py"], dependencies=["testprojects/src/python:native_directory"], + # We want to make sure the native builds work for both macOS and Linux. + tags=["platform_specific_behavior"], timeout=180, ) diff --git a/src/python/pants/backend/python/goals/setup_py_integration_test.py b/src/python/pants/backend/python/goals/setup_py_integration_test.py index 14eafa62637..198b60fd0e4 100644 --- a/src/python/pants/backend/python/goals/setup_py_integration_test.py +++ b/src/python/pants/backend/python/goals/setup_py_integration_test.py @@ -12,10 +12,13 @@ from tempfile import TemporaryDirectory from textwrap import dedent +import pytest + from pants.testutil.pants_integration_test import run_pants, setup_tmpdir from pants.util.dirutil import safe_rmtree +@pytest.mark.platform_specific_behavior def test_native_code() -> None: dist_dir = "dist" pyver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" diff --git a/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py b/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py index 6e95cd136a4..0e515e6b724 100644 --- a/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/mypyc_integration_test.py @@ -9,9 +9,12 @@ import venv from tempfile import TemporaryDirectory +import pytest + from pants.testutil.pants_integration_test import run_pants +@pytest.mark.platform_specific_behavior def test_mypyc_build() -> None: dist_dir = "dist" pyver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" From 1a409c9c6cd0cf8d4e82cb928717d9ebdcf090ed Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Tue, 10 May 2022 13:28:32 -0700 Subject: [PATCH 5/5] Improve help message # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/typecheck/mypy/mypyc.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/typecheck/mypy/mypyc.py b/src/python/pants/backend/python/typecheck/mypy/mypyc.py index fba7c874116..f41eb13adfa 100644 --- a/src/python/pants/backend/python/typecheck/mypy/mypyc.py +++ b/src/python/pants/backend/python/typecheck/mypy/mypyc.py @@ -19,12 +19,25 @@ from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import BoolField, Target from pants.engine.unions import UnionRule +from pants.util.strutil import softwrap class UsesMyPycField(BoolField): alias = "uses_mypyc" default = False - help = "If true, this distribution is built using mypyc." + help = softwrap( + """ + If true, this distribution is built using mypyc. + + In this case, Pants will build the distribution in an environment that includes + mypy, as configured in the `[mypy]` subsystem, including plugins, config files, + extra type stubs, and the distribution's own requirements (which normally would not + be needed at build time, but in this case may provide necessary type annotations). + + You will typically set this field on distributions whose setup.py uses + mypyc.build.mypycify(). See https://mypyc.readthedocs.io/en/latest/index.html . + """ + ) @dataclass(frozen=True)