From 12a4148df50eeb4ca83e3a080c278883ca77ac83 Mon Sep 17 00:00:00 2001 From: SamRemis Date: Wed, 13 Nov 2024 14:10:17 -0500 Subject: [PATCH 1/6] initial commit --- botocore/endpoint_provider.py | 1 + botocore/model.py | 4 ++ botocore/regions.py | 15 ++++++++ botocore/utils.py | 2 +- .../endpoints/test-cases/array-index.json | 24 ++++++++++++ .../endpoints/valid-rules/array-index.json | 37 +++++++++++++++++++ tests/unit/test_endpoint_provider.py | 4 ++ 7 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/unit/data/endpoints/test-cases/array-index.json create mode 100644 tests/unit/data/endpoints/valid-rules/array-index.json diff --git a/botocore/endpoint_provider.py b/botocore/endpoint_provider.py index 0fbd802b8d..55563c3033 100644 --- a/botocore/endpoint_provider.py +++ b/botocore/endpoint_provider.py @@ -577,6 +577,7 @@ class ParameterType(Enum): string = str boolean = bool + stringarray = tuple class ParameterDefinition: diff --git a/botocore/model.py b/botocore/model.py index 70d20f8cca..efd15e90e7 100644 --- a/botocore/model.py +++ b/botocore/model.py @@ -624,6 +624,10 @@ def context_parameters(self): and 'name' in shape.metadata['contextParam'] ] + @CachedProperty + def operation_context_parameters(self): + return self._operation_model.get('operationContextParams', []) + @CachedProperty def request_compression(self): return self._operation_model.get('requestcompression') diff --git a/botocore/regions.py b/botocore/regions.py index 4c23737270..9f1a641873 100644 --- a/botocore/regions.py +++ b/botocore/regions.py @@ -20,6 +20,8 @@ import copy import logging import re +import jmespath + from enum import Enum from botocore import UNSIGNED, xform_name @@ -578,6 +580,11 @@ def _resolve_param_from_context( ) if dynamic is not None: return dynamic + operation_context_params = self._resolve_param_as_operation_context_param( + param_name, operation_model, call_args + ) + if operation_context_params is not None: + return operation_context_params return self._resolve_param_as_client_context_param(param_name) def _resolve_param_as_static_context_param( @@ -600,6 +607,14 @@ def _resolve_param_as_client_context_param(self, param_name): client_ctx_varname = client_ctx_params[param_name] return self._client_context.get(client_ctx_varname) + def _resolve_param_as_operation_context_param(self, param_name, operation_model, + call_args): + operation_ctx_params = operation_model.operation_context_parameters + if param_name in operation_ctx_params: + #TODO confirm this will always exist + path = operation_ctx_params[param_name]['path'] + return tuple(jmespath.search(path, call_args)) + def _resolve_param_as_builtin(self, builtin_name, builtins): if builtin_name not in EndpointResolverBuiltins.__members__.values(): raise UnknownEndpointResolutionBuiltInName(name=builtin_name) diff --git a/botocore/utils.py b/botocore/utils.py index a54f4a39d3..fe704ff274 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -1479,7 +1479,7 @@ def lru_cache_weakref(*cache_args, **cache_kwargs): functools implementation which offers ``max_size`` and ``typed`` properties. lru_cache is a global cache even when used on a method. The cache's - reference to ``self`` will prevent garbace collection of the object. This + reference to ``self`` will prevent garbage collection of the object. This wrapper around functools.lru_cache replaces the reference to ``self`` with a weak reference to not interfere with garbage collection. """ diff --git a/tests/unit/data/endpoints/test-cases/array-index.json b/tests/unit/data/endpoints/test-cases/array-index.json new file mode 100644 index 0000000000..bb90e1cb6f --- /dev/null +++ b/tests/unit/data/endpoints/test-cases/array-index.json @@ -0,0 +1,24 @@ +{ + "version": "1.0", + "testCases": [ + { + "documentation": "Access an array index at index 0", + "params": { + "Input": ["first index"] + }, + "expect": { + "endpoint": { + "url": "https://www.example.com", + "headers": { + "x-uri": [ + "https://www.example.com" + ], + "x-arn-region": [ + "us-east-2" + ] + } + } + } + } + ] +} diff --git a/tests/unit/data/endpoints/valid-rules/array-index.json b/tests/unit/data/endpoints/valid-rules/array-index.json new file mode 100644 index 0000000000..3bb2c464ef --- /dev/null +++ b/tests/unit/data/endpoints/valid-rules/array-index.json @@ -0,0 +1,37 @@ +{ + "version": "1.3", + "parameters": { + "ResourceList": { + "type": "stringArray" + } + }, + "rules": [ + { + "documentation": "Array is set, retrieve index 0", + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "ResourceList" + } + ] + }, + { + "fn": "getAttr", + "argv": [ + { + "ref": "bucketArn" + }, + "resourceId[2]" + ], + "assign": "outpostId" + } + ], + "endpoint": { + "url": "https://{firstResourceId}.example.com" + }, + "type": "endpoint" + } + ] +} diff --git a/tests/unit/test_endpoint_provider.py b/tests/unit/test_endpoint_provider.py index 8bc8c429d0..8a54cd22f9 100644 --- a/tests/unit/test_endpoint_provider.py +++ b/tests/unit/test_endpoint_provider.py @@ -137,6 +137,7 @@ def endpoint_rule(): def ruleset_testcases(): filenames = [ + "array-index", "aws-region", "default-values", "eventbridge", @@ -161,6 +162,9 @@ def ruleset_testcases(): for test in tests["testCases"]: input_params = test["params"] + for key, value in input_params.items(): + if type(value) == list: + input_params[key] = tuple(value) expected_object = test["expect"] if "error" in expected_object: error_cases.append( From 49cf3409dd6955ae5859828f9c136f2b658965e0 Mon Sep 17 00:00:00 2001 From: SamRemis Date: Wed, 13 Nov 2024 14:21:20 -0500 Subject: [PATCH 2/6] Fixed test --- .../unit/data/endpoints/test-cases/array-index.json | 12 ++---------- .../unit/data/endpoints/valid-rules/array-index.json | 8 ++++---- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/unit/data/endpoints/test-cases/array-index.json b/tests/unit/data/endpoints/test-cases/array-index.json index bb90e1cb6f..5926bd1906 100644 --- a/tests/unit/data/endpoints/test-cases/array-index.json +++ b/tests/unit/data/endpoints/test-cases/array-index.json @@ -4,19 +4,11 @@ { "documentation": "Access an array index at index 0", "params": { - "Input": ["first index"] + "ResourceList": ["resource"] }, "expect": { "endpoint": { - "url": "https://www.example.com", - "headers": { - "x-uri": [ - "https://www.example.com" - ], - "x-arn-region": [ - "us-east-2" - ] - } + "url": "https://www.resource.example.com" } } } diff --git a/tests/unit/data/endpoints/valid-rules/array-index.json b/tests/unit/data/endpoints/valid-rules/array-index.json index 3bb2c464ef..563a33329b 100644 --- a/tests/unit/data/endpoints/valid-rules/array-index.json +++ b/tests/unit/data/endpoints/valid-rules/array-index.json @@ -21,15 +21,15 @@ "fn": "getAttr", "argv": [ { - "ref": "bucketArn" + "ref": "ResourceList" }, - "resourceId[2]" + "[0]" ], - "assign": "outpostId" + "assign": "resourceid" } ], "endpoint": { - "url": "https://{firstResourceId}.example.com" + "url": "https://www.{resourceid}.example.com" }, "type": "endpoint" } From b3cb3452225877f7fa823ab488d86bacd3211be9 Mon Sep 17 00:00:00 2001 From: Alessandra Romero Date: Thu, 14 Nov 2024 14:09:59 -0500 Subject: [PATCH 3/6] Fix error message and add fallback test for unset array --- botocore/endpoint_provider.py | 2 +- tests/unit/data/endpoints/test-cases/array-index.json | 11 +++++++++++ .../unit/data/endpoints/valid-rules/array-index.json | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/botocore/endpoint_provider.py b/botocore/endpoint_provider.py index 55563c3033..d76f9ac245 100644 --- a/botocore/endpoint_provider.py +++ b/botocore/endpoint_provider.py @@ -601,7 +601,7 @@ def __init__( except AttributeError: raise EndpointResolutionError( msg=f"Unknown parameter type: {parameter_type}. " - "A parameter must be of type string or boolean." + "A parameter must be of type string, boolean, or stringarray." ) self.documentation = documentation self.builtin = builtIn diff --git a/tests/unit/data/endpoints/test-cases/array-index.json b/tests/unit/data/endpoints/test-cases/array-index.json index 5926bd1906..22bab2de9e 100644 --- a/tests/unit/data/endpoints/test-cases/array-index.json +++ b/tests/unit/data/endpoints/test-cases/array-index.json @@ -11,6 +11,17 @@ "url": "https://www.resource.example.com" } } + }, + { + "documentation": "Fallback when array is unset", + "params": { + "ResourceList": [] + }, + "expect": { + "endpoint": { + "url": "https://www.example.com" + } + } } ] } diff --git a/tests/unit/data/endpoints/valid-rules/array-index.json b/tests/unit/data/endpoints/valid-rules/array-index.json index 563a33329b..941513d8d3 100644 --- a/tests/unit/data/endpoints/valid-rules/array-index.json +++ b/tests/unit/data/endpoints/valid-rules/array-index.json @@ -32,6 +32,14 @@ "url": "https://www.{resourceid}.example.com" }, "type": "endpoint" + }, + { + "documentation": "Fallback when array is unset", + "conditions": [], + "endpoint": { + "url": "https://www.example.com" + }, + "type": "endpoint" } ] } From 4356c63b66e020e4d72395c0e678fd107b5ee2e0 Mon Sep 17 00:00:00 2001 From: Alessandra Romero Date: Thu, 14 Nov 2024 14:20:17 -0500 Subject: [PATCH 4/6] clean up with pre-commit --- botocore/regions.py | 16 +++++++++------- tests/unit/test_endpoint_provider.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/botocore/regions.py b/botocore/regions.py index 9f1a641873..569830ae0d 100644 --- a/botocore/regions.py +++ b/botocore/regions.py @@ -20,10 +20,10 @@ import copy import logging import re -import jmespath - from enum import Enum +import jmespath + from botocore import UNSIGNED, xform_name from botocore.auth import AUTH_TYPE_MAPS, HAS_CRT from botocore.crt import CRT_SUPPORTED_AUTH_TYPES @@ -580,8 +580,10 @@ def _resolve_param_from_context( ) if dynamic is not None: return dynamic - operation_context_params = self._resolve_param_as_operation_context_param( - param_name, operation_model, call_args + operation_context_params = ( + self._resolve_param_as_operation_context_param( + param_name, operation_model, call_args + ) ) if operation_context_params is not None: return operation_context_params @@ -607,11 +609,11 @@ def _resolve_param_as_client_context_param(self, param_name): client_ctx_varname = client_ctx_params[param_name] return self._client_context.get(client_ctx_varname) - def _resolve_param_as_operation_context_param(self, param_name, operation_model, - call_args): + def _resolve_param_as_operation_context_param( + self, param_name, operation_model, call_args + ): operation_ctx_params = operation_model.operation_context_parameters if param_name in operation_ctx_params: - #TODO confirm this will always exist path = operation_ctx_params[param_name]['path'] return tuple(jmespath.search(path, call_args)) diff --git a/tests/unit/test_endpoint_provider.py b/tests/unit/test_endpoint_provider.py index 8a54cd22f9..83752f43de 100644 --- a/tests/unit/test_endpoint_provider.py +++ b/tests/unit/test_endpoint_provider.py @@ -163,7 +163,7 @@ def ruleset_testcases(): for test in tests["testCases"]: input_params = test["params"] for key, value in input_params.items(): - if type(value) == list: + if isinstance(value, list): input_params[key] = tuple(value) expected_object = test["expect"] if "error" in expected_object: From 85adf28687d75fa482a4b9d3c0d8383a9eb800d5 Mon Sep 17 00:00:00 2001 From: Alessandra Romero Date: Thu, 21 Nov 2024 10:21:36 -0500 Subject: [PATCH 5/6] convert lists to tuples --- botocore/regions.py | 2 +- botocore/utils.py | 3 +++ tests/unit/test_endpoint_provider.py | 3 --- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/botocore/regions.py b/botocore/regions.py index 569830ae0d..d3b2705727 100644 --- a/botocore/regions.py +++ b/botocore/regions.py @@ -615,7 +615,7 @@ def _resolve_param_as_operation_context_param( operation_ctx_params = operation_model.operation_context_parameters if param_name in operation_ctx_params: path = operation_ctx_params[param_name]['path'] - return tuple(jmespath.search(path, call_args)) + return jmespath.search(path, call_args) def _resolve_param_as_builtin(self, builtin_name, builtins): if builtin_name not in EndpointResolverBuiltins.__members__.values(): diff --git a/botocore/utils.py b/botocore/utils.py index fe704ff274..2d55e8a8dc 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -1491,6 +1491,9 @@ def func_with_weakref(weakref_to_self, *args, **kwargs): @functools.wraps(func) def inner(self, *args, **kwargs): + for kwarg_key, kwarg_value in kwargs.items(): + if isinstance(kwarg_value, list): + kwargs[kwarg_key] = tuple(kwarg_value) return func_with_weakref(weakref.ref(self), *args, **kwargs) inner.cache_info = func_with_weakref.cache_info diff --git a/tests/unit/test_endpoint_provider.py b/tests/unit/test_endpoint_provider.py index 83752f43de..3d6bf81e32 100644 --- a/tests/unit/test_endpoint_provider.py +++ b/tests/unit/test_endpoint_provider.py @@ -162,9 +162,6 @@ def ruleset_testcases(): for test in tests["testCases"]: input_params = test["params"] - for key, value in input_params.items(): - if isinstance(value, list): - input_params[key] = tuple(value) expected_object = test["expect"] if "error" in expected_object: error_cases.append( From c7b1f6a1fc43654c69f6dabffc3c456dc98fccb2 Mon Sep 17 00:00:00 2001 From: SamRemis Date: Wed, 4 Dec 2024 12:34:47 -0500 Subject: [PATCH 6/6] Add default test --- .../unit/data/endpoints/test-cases/array-index.json | 12 +++++++++++- .../unit/data/endpoints/valid-rules/array-index.json | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/data/endpoints/test-cases/array-index.json b/tests/unit/data/endpoints/test-cases/array-index.json index 22bab2de9e..aa73d460d3 100644 --- a/tests/unit/data/endpoints/test-cases/array-index.json +++ b/tests/unit/data/endpoints/test-cases/array-index.json @@ -13,7 +13,7 @@ } }, { - "documentation": "Fallback when array is unset", + "documentation": "Resolved value when array is explictly set to empty", "params": { "ResourceList": [] }, @@ -22,6 +22,16 @@ "url": "https://www.example.com" } } + }, + { + "documentation": "Resolved value to default if array is unset", + "params": { + }, + "expect": { + "endpoint": { + "url": "https://www.default1.example.com" + } + } } ] } diff --git a/tests/unit/data/endpoints/valid-rules/array-index.json b/tests/unit/data/endpoints/valid-rules/array-index.json index 941513d8d3..3d426b09aa 100644 --- a/tests/unit/data/endpoints/valid-rules/array-index.json +++ b/tests/unit/data/endpoints/valid-rules/array-index.json @@ -2,6 +2,8 @@ "version": "1.3", "parameters": { "ResourceList": { + "required": true, + "default": ["default1", "default2"], "type": "stringArray" } },