From b3ae750ca77eb57d633c6757cad469e02ef90349 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 14 Jan 2023 10:06:17 -0800 Subject: [PATCH] Clarify runtime vs. complete_platforms for serverless. A common pitfall is 1st using `runtime`, encountering issues, migrating to `complete_platforms` and still running into issue due to not deleting `runtime`. Hopefully this helps clarify you generally want to use these two fields as XOR. --- .../pants/backend/awslambda/python/rules.py | 16 +++++++++++++--- .../backend/awslambda/python/target_types.py | 19 ++++++++++++++++++- .../awslambda/python/target_types_test.py | 13 +++++-------- .../google_cloud_function/python/rules.py | 16 +++++++++++++--- .../python/target_types.py | 19 ++++++++++++++++++- .../python/target_types_test.py | 13 +++++-------- .../pants/backend/python/util_rules/pex.py | 15 ++++++++++++--- 7 files changed, 84 insertions(+), 27 deletions(-) diff --git a/src/python/pants/backend/awslambda/python/rules.py b/src/python/pants/backend/awslambda/python/rules.py index 196a900bd93..9b3e5bfc0a7 100644 --- a/src/python/pants/backend/awslambda/python/rules.py +++ b/src/python/pants/backend/awslambda/python/rules.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pants.backend.awslambda.python.target_types import ( + PythonAwsLambdaCompletePlatforms, PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, PythonAwsLambdaRuntime, @@ -14,7 +15,6 @@ ResolvePythonAwsHandlerRequest, ) from pants.backend.python.subsystems.lambdex import Lambdex -from pants.backend.python.target_types import PexCompletePlatformsField from pants.backend.python.util_rules import pex_from_targets from pants.backend.python.util_rules.pex import ( CompletePlatforms, @@ -32,6 +32,7 @@ PackageFieldSet, ) from pants.core.target_types import FileSourceField +from pants.engine.addresses import UnparsedAddressInputs from pants.engine.platform import Platform from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule @@ -55,10 +56,19 @@ class PythonAwsLambdaFieldSet(PackageFieldSet): handler: PythonAwsLambdaHandlerField include_requirements: PythonAwsLambdaIncludeRequirements runtime: PythonAwsLambdaRuntime - complete_platforms: PexCompletePlatformsField + complete_platforms: PythonAwsLambdaCompletePlatforms output_path: OutputPathField +@rule +async def digest_complete_platforms( + complete_platforms: PythonAwsLambdaCompletePlatforms, +) -> CompletePlatforms: + return await Get( + CompletePlatforms, UnparsedAddressInputs, complete_platforms.to_unparsed_address_inputs() + ) + + @rule(desc="Create Python AWS Lambda", level=LogLevel.DEBUG) async def package_python_awslambda( field_set: PythonAwsLambdaFieldSet, @@ -110,7 +120,7 @@ async def package_python_awslambda( ) complete_platforms = await Get( - CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms + CompletePlatforms, PythonAwsLambdaCompletePlatforms, field_set.complete_platforms ) pex_request = PexFromTargetsRequest( diff --git a/src/python/pants/backend/awslambda/python/target_types.py b/src/python/pants/backend/awslambda/python/target_types.py index 61eede89c0b..23e5dc57c07 100644 --- a/src/python/pants/backend/awslambda/python/target_types.py +++ b/src/python/pants/backend/awslambda/python/target_types.py @@ -246,6 +246,11 @@ class PythonAwsLambdaRuntime(StringField): """ The identifier of the AWS Lambda runtime to target (pythonX.Y). See https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html. + + In general you'll want to define either a `runtime` or one `complete_platforms` but not + both. Specifying a `runtime` is simpler, but less accurate. If you have issues either + packaging the AWS Lambda PEX or running it as a deployed AWS Lambda function, you should try + using `complete_platforms` instead. """ ) @@ -273,6 +278,18 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]: return int(mo.group("major")), int(mo.group("minor")) +class PythonAwsLambdaCompletePlatforms(PexCompletePlatformsField): + help = softwrap( + f""" + {PexCompletePlatformsField.help} + + N.B.: If specifying `complete_platforms` to work around packaging failures encountered when + using the `runtime` field, ensure you delete the `runtime` field from your + `python_awslambda` target. + """ + ) + + class PythonAWSLambda(Target): alias = "python_awslambda" core_fields = ( @@ -282,7 +299,7 @@ class PythonAWSLambda(Target): PythonAwsLambdaHandlerField, PythonAwsLambdaIncludeRequirements, PythonAwsLambdaRuntime, - PexCompletePlatformsField, + PythonAwsLambdaCompletePlatforms, PythonResolveField, ) help = softwrap( diff --git a/src/python/pants/backend/awslambda/python/target_types_test.py b/src/python/pants/backend/awslambda/python/target_types_test.py index c5aa43fe150..e90bffc47de 100644 --- a/src/python/pants/backend/awslambda/python/target_types_test.py +++ b/src/python/pants/backend/awslambda/python/target_types_test.py @@ -9,6 +9,7 @@ from pants.backend.awslambda.python.target_types import ( InferPythonLambdaHandlerDependency, PythonAWSLambda, + PythonAwsLambdaCompletePlatforms, PythonAwsLambdaHandlerField, PythonAwsLambdaRuntime, PythonLambdaHandlerDependencyInferenceFieldSet, @@ -16,11 +17,7 @@ ResolvePythonAwsHandlerRequest, ) from pants.backend.awslambda.python.target_types import rules as target_type_rules -from pants.backend.python.target_types import ( - PexCompletePlatformsField, - PythonRequirementTarget, - PythonSourcesGeneratorTarget, -) +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.build_graph.address import Address from pants.core.target_types import FileTarget @@ -280,17 +277,17 @@ def test_at_least_one_target_platform(rule_runner: RuleRunner) -> None: runtime = rule_runner.get_target(Address("project", target_name="runtime")) assert "python3.7" == runtime[PythonAwsLambdaRuntime].value - assert runtime[PexCompletePlatformsField].value is None + assert runtime[PythonAwsLambdaCompletePlatforms].value is None complete_platforms = rule_runner.get_target( Address("project", target_name="complete_platforms") ) assert complete_platforms[PythonAwsLambdaRuntime].value is None - assert (":python37",) == complete_platforms[PexCompletePlatformsField].value + assert (":python37",) == complete_platforms[PythonAwsLambdaCompletePlatforms].value both = rule_runner.get_target(Address("project", target_name="both")) assert "python3.7" == both[PythonAwsLambdaRuntime].value - assert (":python37",) == both[PexCompletePlatformsField].value + assert (":python37",) == both[PythonAwsLambdaCompletePlatforms].value with pytest.raises( ExecutionError, diff --git a/src/python/pants/backend/google_cloud_function/python/rules.py b/src/python/pants/backend/google_cloud_function/python/rules.py index f38ef2170ea..ca5315e06d2 100644 --- a/src/python/pants/backend/google_cloud_function/python/rules.py +++ b/src/python/pants/backend/google_cloud_function/python/rules.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from pants.backend.google_cloud_function.python.target_types import ( + PythonGoogleCloudFunctionCompletePlatforms, PythonGoogleCloudFunctionHandlerField, PythonGoogleCloudFunctionRuntime, PythonGoogleCloudFunctionType, @@ -14,7 +15,6 @@ ResolvePythonGoogleHandlerRequest, ) from pants.backend.python.subsystems.lambdex import Lambdex -from pants.backend.python.target_types import PexCompletePlatformsField from pants.backend.python.util_rules import pex_from_targets from pants.backend.python.util_rules.pex import ( CompletePlatforms, @@ -32,6 +32,7 @@ PackageFieldSet, ) from pants.core.target_types import FileSourceField +from pants.engine.addresses import UnparsedAddressInputs from pants.engine.platform import Platform from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule @@ -53,11 +54,20 @@ class PythonGoogleCloudFunctionFieldSet(PackageFieldSet): handler: PythonGoogleCloudFunctionHandlerField runtime: PythonGoogleCloudFunctionRuntime - complete_platforms: PexCompletePlatformsField + complete_platforms: PythonGoogleCloudFunctionCompletePlatforms type: PythonGoogleCloudFunctionType output_path: OutputPathField +@rule +async def digest_complete_platforms( + complete_platforms: PythonGoogleCloudFunctionCompletePlatforms, +) -> CompletePlatforms: + return await Get( + CompletePlatforms, UnparsedAddressInputs, complete_platforms.to_unparsed_address_inputs() + ) + + @rule(desc="Create Python Google Cloud Function", level=LogLevel.DEBUG) async def package_python_google_cloud_function( field_set: PythonGoogleCloudFunctionFieldSet, @@ -102,7 +112,7 @@ async def package_python_google_cloud_function( ) complete_platforms = await Get( - CompletePlatforms, PexCompletePlatformsField, field_set.complete_platforms + CompletePlatforms, PythonGoogleCloudFunctionCompletePlatforms, field_set.complete_platforms ) pex_request = PexFromTargetsRequest( diff --git a/src/python/pants/backend/google_cloud_function/python/target_types.py b/src/python/pants/backend/google_cloud_function/python/target_types.py index f77f0921081..b3c6cd02fc2 100644 --- a/src/python/pants/backend/google_cloud_function/python/target_types.py +++ b/src/python/pants/backend/google_cloud_function/python/target_types.py @@ -231,6 +231,11 @@ class PythonGoogleCloudFunctionRuntime(StringField): """ The identifier of the Google Cloud Function runtime to target (pythonXY). See https://cloud.google.com/functions/docs/concepts/python-runtime. + + In general you'll want to define either a `runtime` or one `complete_platforms` but not + both. Specifying a `runtime` is simpler, but less accurate. If you have issues either + packaging the Google Cloud Function PEX or running it as a deployed Google Cloud Function, + you should try using `complete_platforms` instead. """ ) @@ -254,6 +259,18 @@ def to_interpreter_version(self) -> Optional[Tuple[int, int]]: return int(mo.group("major")), int(mo.group("minor")) +class PythonGoogleCloudFunctionCompletePlatforms(PexCompletePlatformsField): + help = softwrap( + f""" + {PexCompletePlatformsField.help} + + N.B.: If specifying `complete_platforms` to work around packaging failures encountered when + using the `runtime` field, ensure you delete the `runtime` field from your + `python_google_cloud_function` target. + """ + ) + + class GoogleCloudFunctionTypes(Enum): EVENT = "event" HTTP = "http" @@ -281,7 +298,7 @@ class PythonGoogleCloudFunction(Target): PythonGoogleCloudFunctionDependencies, PythonGoogleCloudFunctionHandlerField, PythonGoogleCloudFunctionRuntime, - PexCompletePlatformsField, + PythonGoogleCloudFunctionCompletePlatforms, PythonGoogleCloudFunctionType, PythonResolveField, ) diff --git a/src/python/pants/backend/google_cloud_function/python/target_types_test.py b/src/python/pants/backend/google_cloud_function/python/target_types_test.py index 9df51beaf72..1abcfb15262 100644 --- a/src/python/pants/backend/google_cloud_function/python/target_types_test.py +++ b/src/python/pants/backend/google_cloud_function/python/target_types_test.py @@ -10,17 +10,14 @@ InferPythonCloudFunctionHandlerDependency, PythonCloudFunctionHandlerInferenceFieldSet, PythonGoogleCloudFunction, + PythonGoogleCloudFunctionCompletePlatforms, PythonGoogleCloudFunctionHandlerField, PythonGoogleCloudFunctionRuntime, ResolvedPythonGoogleHandler, ResolvePythonGoogleHandlerRequest, ) from pants.backend.google_cloud_function.python.target_types import rules as target_type_rules -from pants.backend.python.target_types import ( - PexCompletePlatformsField, - PythonRequirementTarget, - PythonSourcesGeneratorTarget, -) +from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.backend.python.target_types_rules import rules as python_target_types_rules from pants.build_graph.address import Address from pants.core.target_types import FileTarget @@ -298,17 +295,17 @@ def test_at_least_one_target_platform(rule_runner: RuleRunner) -> None: runtime = rule_runner.get_target(Address("project", target_name="runtime")) assert "python37" == runtime[PythonGoogleCloudFunctionRuntime].value - assert runtime[PexCompletePlatformsField].value is None + assert runtime[PythonGoogleCloudFunctionCompletePlatforms].value is None complete_platforms = rule_runner.get_target( Address("project", target_name="complete_platforms") ) assert complete_platforms[PythonGoogleCloudFunctionRuntime].value is None - assert (":python37",) == complete_platforms[PexCompletePlatformsField].value + assert (":python37",) == complete_platforms[PythonGoogleCloudFunctionCompletePlatforms].value both = rule_runner.get_target(Address("project", target_name="both")) assert "python37" == both[PythonGoogleCloudFunctionRuntime].value - assert (":python37",) == both[PexCompletePlatformsField].value + assert (":python37",) == both[PythonGoogleCloudFunctionCompletePlatforms].value with pytest.raises( ExecutionError, diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index c2dbd487030..02b6d1ff400 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -103,13 +103,13 @@ def generate_pex_arg_list(self) -> Iterator[str]: @rule -async def digest_complete_platforms( - complete_platforms: PexCompletePlatformsField, +async def digest_complete_platform_addresses( + addresses: UnparsedAddressInputs, ) -> CompletePlatforms: original_file_targets = await Get( Targets, UnparsedAddressInputs, - complete_platforms.to_unparsed_address_inputs(), + addresses, ) original_files_sources = await MultiGet( Get( @@ -124,6 +124,15 @@ async def digest_complete_platforms( return CompletePlatforms.from_snapshot(snapshot) +@rule +async def digest_complete_platforms( + complete_platforms: PexCompletePlatformsField, +) -> CompletePlatforms: + return await Get( + CompletePlatforms, UnparsedAddressInputs, complete_platforms.to_unparsed_address_inputs() + ) + + @frozen_after_init @dataclass(unsafe_hash=True) class PexRequest(EngineAwareParameter):