diff --git a/CHANGES.rst b/CHANGES.rst index db375c72..24466373 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ Changes ------- +2.16.0 (2024-12-16) +^^^^^^^^^^^^^^^^^^^ +* bump botocore dependency specification + 2.15.2 (2024-10-09) ^^^^^^^^^^^^^^^^^^^ * relax botocore dependency specification diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index 30334bd3..48aa744f 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1 +1 @@ -__version__ = '2.15.2' +__version__ = '2.16.0' diff --git a/aiobotocore/handlers.py b/aiobotocore/handlers.py index c060ab94..1c0b0df6 100644 --- a/aiobotocore/handlers.py +++ b/aiobotocore/handlers.py @@ -1,13 +1,14 @@ from botocore.handlers import ( ETree, - XMLParseError, _get_cross_region_presigned_url, _get_presigned_url_source_and_destination_regions, + _looks_like_special_case_error, logger, ) async def check_for_200_error(response, **kwargs): + """This function has been deprecated, but is kept for backwards compatibility.""" # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html # There are two opportunities for a copy request to return an error. One # can occur when Amazon S3 receives the copy request and the other can @@ -28,7 +29,9 @@ async def check_for_200_error(response, **kwargs): # trying to retrieve the response. See Endpoint._get_response(). return http_response, parsed = response - if await _looks_like_special_case_error(http_response): + if _looks_like_special_case_error( + http_response.status_code, await http_response.content + ): logger.debug( "Error found for response with 200 status code, " "errors: %s, changing status code to " @@ -38,24 +41,6 @@ async def check_for_200_error(response, **kwargs): http_response.status_code = 500 -async def _looks_like_special_case_error(http_response): - if http_response.status_code == 200: - try: - parser = ETree.XMLParser( - target=ETree.TreeBuilder(), encoding='utf-8' - ) - parser.feed(await http_response.content) - root = parser.close() - except XMLParseError: - # In cases of network disruptions, we may end up with a partial - # streamed response from S3. We need to treat these cases as - # 500 Service Errors and try again. - return True - if root.tag == 'Error': - return True - return False - - async def inject_presigned_url_ec2(params, request_signer, model, **kwargs): # The customer can still provide this, so we should pass if they do. if 'PresignedUrl' in params['body']: diff --git a/aiobotocore/hooks.py b/aiobotocore/hooks.py index eaae300e..79acb072 100644 --- a/aiobotocore/hooks.py +++ b/aiobotocore/hooks.py @@ -1,4 +1,3 @@ -from botocore.handlers import check_for_200_error as boto_check_for_200_error from botocore.handlers import ( inject_presigned_url_ec2 as boto_inject_presigned_url_ec2, ) @@ -9,6 +8,9 @@ parse_get_bucket_location as boto_parse_get_bucket_location, ) from botocore.hooks import HierarchicalEmitter, logger +from botocore.signers import ( + add_dsql_generate_db_auth_token_methods as boto_add_dsql_generate_db_auth_token_methods, +) from botocore.signers import ( add_generate_db_auth_token as boto_add_generate_db_auth_token, ) @@ -21,12 +23,12 @@ from ._helpers import resolve_awaitable from .handlers import ( - check_for_200_error, inject_presigned_url_ec2, inject_presigned_url_rds, parse_get_bucket_location, ) from .signers import ( + add_dsql_generate_db_auth_token_methods, add_generate_db_auth_token, add_generate_presigned_post, add_generate_presigned_url, @@ -39,7 +41,7 @@ boto_add_generate_presigned_post: add_generate_presigned_post, boto_add_generate_db_auth_token: add_generate_db_auth_token, boto_parse_get_bucket_location: parse_get_bucket_location, - boto_check_for_200_error: check_for_200_error, + boto_add_dsql_generate_db_auth_token_methods: add_dsql_generate_db_auth_token_methods, } diff --git a/aiobotocore/signers.py b/aiobotocore/signers.py index ba2679e2..5471d93d 100644 --- a/aiobotocore/signers.py +++ b/aiobotocore/signers.py @@ -2,7 +2,7 @@ import botocore import botocore.auth -from botocore.exceptions import UnknownClientMethodError +from botocore.exceptions import ParamValidationError, UnknownClientMethodError from botocore.signers import ( RequestSigner, S3PostPresigner, @@ -200,6 +200,15 @@ def add_generate_db_auth_token(class_attributes, **kwargs): class_attributes['generate_db_auth_token'] = generate_db_auth_token +def add_dsql_generate_db_auth_token_methods(class_attributes, **kwargs): + class_attributes['generate_db_connect_auth_token'] = ( + dsql_generate_db_connect_auth_token + ) + class_attributes['generate_db_connect_admin_auth_token'] = ( + dsql_generate_db_connect_admin_auth_token + ) + + async def generate_db_auth_token( self, DBHostname, Port, DBUsername, Region=None ): @@ -256,6 +265,86 @@ async def generate_db_auth_token( return presigned_url[len(scheme) :] +async def _dsql_generate_db_auth_token( + self, Hostname, Action, Region=None, ExpiresIn=900 +): + """Generate a DSQL database token for an arbitrary action. + :type Hostname: str + :param Hostname: The DSQL endpoint host name. + :type Action: str + :param Action: Action to perform on the cluster (DbConnectAdmin or DbConnect). + :type Region: str + :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used. + :type ExpiresIn: int + :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds). + :return: A presigned url which can be used as an auth token. + """ + possible_actions = ("DbConnect", "DbConnectAdmin") + + if Action not in possible_actions: + raise ParamValidationError( + report=f"Received {Action} for action but expected one of: {', '.join(possible_actions)}" + ) + + if Region is None: + Region = self.meta.region_name + + request_dict = { + 'url_path': '/', + 'query_string': '', + 'headers': {}, + 'body': { + 'Action': Action, + }, + 'method': 'GET', + } + scheme = 'https://' + endpoint_url = f'{scheme}{Hostname}' + prepare_request_dict(request_dict, endpoint_url) + presigned_url = await self._request_signer.generate_presigned_url( + operation_name=Action, + request_dict=request_dict, + region_name=Region, + expires_in=ExpiresIn, + signing_name='dsql', + ) + return presigned_url[len(scheme) :] + + +async def dsql_generate_db_connect_auth_token( + self, Hostname, Region=None, ExpiresIn=900 +): + """Generate a DSQL database token for the "DbConnect" action. + :type Hostname: str + :param Hostname: The DSQL endpoint host name. + :type Region: str + :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used. + :type ExpiresIn: int + :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds). + :return: A presigned url which can be used as an auth token. + """ + return await _dsql_generate_db_auth_token( + self, Hostname, "DbConnect", Region, ExpiresIn + ) + + +async def dsql_generate_db_connect_admin_auth_token( + self, Hostname, Region=None, ExpiresIn=900 +): + """Generate a DSQL database token for the "DbConnectAdmin" action. + :type Hostname: str + :param Hostname: The DSQL endpoint host name. + :type Region: str + :param Region: The AWS region where the DSQL Cluster is hosted. If None, the client region will be used. + :type ExpiresIn: int + :param ExpiresIn: The token expiry duration in seconds (default is 900 seconds). + :return: A presigned url which can be used as an auth token. + """ + return await _dsql_generate_db_auth_token( + self, Hostname, "DbConnectAdmin", Region, ExpiresIn + ) + + class AioS3PostPresigner(S3PostPresigner): async def generate_presigned_post( self, diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index a540de69..bf472538 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -461,6 +461,10 @@ async def redirect_from_error( 0 ].status_code in (301, 302, 307) is_permanent_redirect = error_code == 'PermanentRedirect' + is_opt_in_region_redirect = ( + error_code == 'IllegalLocationConstraintException' + and operation.name != 'CreateBucket' + ) if not any( [ is_special_head_object, @@ -468,6 +472,7 @@ async def redirect_from_error( is_permanent_redirect, is_special_head_bucket, is_redirect_status, + is_opt_in_region_redirect, ] ): return diff --git a/pyproject.toml b/pyproject.toml index 371a68a5..c60ee4e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ dynamic = ["version", "readme"] dependencies = [ - "botocore >=1.35.16, <1.35.37", # NOTE: When updating, always keep `project.optional-dependencies` aligned + "botocore >=1.35.74, <1.35.82", # NOTE: When updating, always keep `project.optional-dependencies` aligned "aiohttp >=3.9.2, <4.0.0", "wrapt >=1.10.10, <2.0.0", "aioitertools >=0.5.1, <1.0.0", @@ -38,10 +38,10 @@ dependencies = [ [project.optional-dependencies] awscli = [ - "awscli >=1.34.16, <1.35.3", + "awscli >=1.36.15, <1.36.23", ] boto3 = [ - "boto3 >=1.35.16, <1.35.37", + "boto3 >=1.35.74, <1.35.82", ] [project.urls] diff --git a/requirements-dev.in b/requirements-dev.in index 1567ea46..955d0f86 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -22,4 +22,5 @@ docker~=7.1 moto[server,s3,sqs,awslambda,dynamodb,cloudformation,sns,batch,ec2,rds]~=4.2.9 pre-commit~=3.5.0 pytest-asyncio~=0.23.8 +time-machine~=2.15.0 tomli; python_version < "3.11" # Requirement for tests/test_version.py diff --git a/tests/boto_tests/__init__.py b/tests/boto_tests/__init__.py index e69de29b..f6378c8f 100644 --- a/tests/boto_tests/__init__.py +++ b/tests/boto_tests/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from botocore.compat import parse_qs, urlparse + + +def _urlparse(url): + if isinstance(url, bytes): + # Not really necessary, but it helps to reduce noise on Python 2.x + url = url.decode('utf8') + return urlparse(url) + + +def assert_url_equal(url1, url2): + parts1 = _urlparse(url1) + parts2 = _urlparse(url2) + + # Because the query string ordering isn't relevant, we have to parse + # every single part manually and then handle the query string. + assert parts1.scheme == parts2.scheme + assert parts1.netloc == parts2.netloc + assert parts1.path == parts2.path + assert parts1.params == parts2.params + assert parts1.fragment == parts2.fragment + assert parts1.username == parts2.username + assert parts1.password == parts2.password + assert parts1.hostname == parts2.hostname + assert parts1.port == parts2.port + assert parse_qs(parts1.query) == parse_qs(parts2.query) diff --git a/tests/boto_tests/unit/test_handlers.py b/tests/boto_tests/unit/test_handlers.py new file mode 100644 index 00000000..eea52b82 --- /dev/null +++ b/tests/boto_tests/unit/test_handlers.py @@ -0,0 +1,53 @@ +# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +from unittest import mock + +from aiobotocore import handlers + + +class TestHandlers: + async def test_500_status_code_set_for_200_response(self): + http_response = mock.Mock() + http_response.status_code = 200 + + async def content(): + return """ + + AccessDenied + Access Denied + id + hostid + + """ + + http_response.content = content() + await handlers.check_for_200_error((http_response, {})) + assert http_response.status_code == 500 + + async def test_200_response_with_no_error_left_untouched(self): + http_response = mock.Mock() + http_response.status_code = 200 + + async def content(): + return "" + + http_response.content = content() + await handlers.check_for_200_error((http_response, {})) + # We don't touch the status code since there are no errors present. + assert http_response.status_code == 200 + + async def test_500_response_can_be_none(self): + # A 500 response can raise an exception, which means the response + # object is None. We need to handle this case. + await handlers.check_for_200_error(None) diff --git a/tests/boto_tests/unit/test_signers.py b/tests/boto_tests/unit/test_signers.py index 1447b306..bcdcdf92 100644 --- a/tests/boto_tests/unit/test_signers.py +++ b/tests/boto_tests/unit/test_signers.py @@ -1,10 +1,28 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. import datetime from datetime import timezone from unittest import mock +import pytest +from botocore.exceptions import ParamValidationError + import aiobotocore.credentials import aiobotocore.session import aiobotocore.signers +from tests.boto_tests import assert_url_equal + +DATE = datetime.datetime(2024, 11, 7, 17, 39, 33, tzinfo=timezone.utc) async def test_signers_generate_db_auth_token(rds_client): @@ -31,3 +49,98 @@ async def test_signers_generate_db_auth_token(rds_client): assert result2.startswith( 'prod-instance.us-east-1.rds.amazonaws.com:3306/?AWSAccessKeyId=xxx&' ) + + +class TestDSQLGenerateDBAuthToken: + @pytest.fixture(scope="session") + def hostname(self): + return 'test.dsql.us-east-1.on.aws' + + @pytest.fixture(scope="session") + def action(self): + return 'DbConnect' + + @pytest.fixture + async def client(self, session): + async with session.create_client( + 'dsql', + region_name='us-east-1', + aws_access_key_id='ACCESS_KEY', + aws_secret_access_key='SECRET_KEY', + aws_session_token="SESSION_TOKEN", + ) as client: + yield client + + async def test_dsql_generate_db_auth_token( + self, client, hostname, action, time_machine + ): + time_machine.move_to(DATE, tick=False) + + result = await aiobotocore.signers._dsql_generate_db_auth_token( + client, hostname, action + ) + + expected_result = ( + 'test.dsql.us-east-1.on.aws/?Action=DbConnect' + '&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=' + 'ACCESS_KEY%2F20241107%2Fus-east-1%2Fdsql%2Faws4_request' + '&X-Amz-Date=20241107T173933Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host' + '&X-Amz-Security-Token=SESSION_TOKEN&X-Amz-Signature=' + '57fe03e060348aaa21405c239bf02572bbc911076e94dcd65c12ae569dd8fcf4' + ) + + # A scheme needs to be appended to the beginning or urlsplit may fail + # on certain systems. + assert_url_equal('https://' + result, 'https://' + expected_result) + + async def test_dsql_generate_db_connect_auth_token( + self, client, hostname, time_machine + ): + time_machine.move_to(DATE, tick=False) + + result = await aiobotocore.signers.dsql_generate_db_connect_auth_token( + client, hostname + ) + + expected_result = ( + 'test.dsql.us-east-1.on.aws/?Action=DbConnect' + '&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=' + 'ACCESS_KEY%2F20241107%2Fus-east-1%2Fdsql%2Faws4_request' + '&X-Amz-Date=20241107T173933Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host' + '&X-Amz-Security-Token=SESSION_TOKEN&X-Amz-Signature=' + '57fe03e060348aaa21405c239bf02572bbc911076e94dcd65c12ae569dd8fcf4' + ) + + # A scheme needs to be appended to the beginning or urlsplit may fail + # on certain systems. + assert_url_equal('https://' + result, 'https://' + expected_result) + + async def test_dsql_generate_db_connect_admin_auth_token( + self, client, hostname, time_machine + ): + time_machine.move_to(DATE, tick=False) + + result = await aiobotocore.signers.dsql_generate_db_connect_admin_auth_token( + client, hostname + ) + + expected_result = ( + 'test.dsql.us-east-1.on.aws/?Action=DbConnectAdmin' + '&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=' + 'ACCESS_KEY%2F20241107%2Fus-east-1%2Fdsql%2Faws4_request' + '&X-Amz-Date=20241107T173933Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host' + '&X-Amz-Security-Token=SESSION_TOKEN&X-Amz-Signature=' + '5ac084bc7cabccc19a52a5d1b5c24b50d3ce143f43b659bd484c91aaf555e190' + ) + + # A scheme needs to be appended to the beginning or urlsplit may fail + # on certain systems. + assert_url_equal('https://' + result, 'https://' + expected_result) + + async def test_dsql_generate_db_auth_token_invalid_action( + self, client, hostname + ): + with pytest.raises(ParamValidationError): + await aiobotocore.signers._dsql_generate_db_auth_token( + client, hostname, "FooBar" + ) diff --git a/tests/test_patches.py b/tests/test_patches.py index 5401db79..b234c529 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -42,7 +42,6 @@ ) from botocore.eventstream import EventStream from botocore.handlers import ( - _looks_like_special_case_error, check_for_200_error, inject_presigned_url_ec2, inject_presigned_url_rds, @@ -78,9 +77,13 @@ from botocore.signers import ( RequestSigner, S3PostPresigner, + _dsql_generate_db_auth_token, + add_dsql_generate_db_auth_token_methods, add_generate_db_auth_token, add_generate_presigned_post, add_generate_presigned_url, + dsql_generate_db_connect_admin_auth_token, + dsql_generate_db_connect_auth_token, generate_db_auth_token, generate_presigned_post, generate_presigned_url, @@ -202,6 +205,7 @@ Config.merge: {'c3dd8c3ffe0da86953ceba4a35267dfb79c6a2c8'}, Config: { '823f8d031fc7218a600a56268a369aaa878f46c8', + 'b1bd1c2cb9a20afa98db306c803617543ffecbf4', }, # credentials.py create_mfa_serial_refresher: {'9b5e98782fcacdcea5899a6d0d29d1b9de348bb0'}, @@ -487,14 +491,28 @@ 'd03631d6810e2453b8874bc76619927b694a4207', }, S3PostPresigner.generate_presigned_post: { - '269efc9af054a2fd2728d5b0a27db82c48053d7f' + '269efc9af054a2fd2728d5b0a27db82c48053d7f', + '48418dc6c9b04fdc8689c7cb5b6eb987321a84e3', }, add_generate_presigned_post: {'e30360f2bd893fabf47f5cdb04b0de420ccd414d'}, generate_presigned_post: { 'a3a834a08be2cf76c20ea137ba6b28e7a12f58ed', + 'd93240c58dcda7b63cf2b7144ee0fea110f0e762', }, add_generate_db_auth_token: {'f61014e6fac4b5c7ee7ac2d2bec15fb16fa9fbe5'}, generate_db_auth_token: {'1f37e1e5982d8528841ce6b79f229b3e23a18959'}, + add_dsql_generate_db_auth_token_methods: { + '95c68a1aac8ee549e11b5dc010b6bb03f9ea00ea', + }, + _dsql_generate_db_auth_token: { + '53034b0475122209509db59fbd79a4ead70836cf', + }, + dsql_generate_db_connect_auth_token: { + '29b5919b695113c55452f2325d0ff66dd719a647' + }, + dsql_generate_db_connect_admin_auth_token: { + 'd7e7a4899b8fd3a544dd1df95196517e2cfd5c84' + }, # tokens.py create_token_resolver: {'b287f4879235a4292592a49b201d2b0bc2dbf401'}, DeferredRefreshableToken.__init__: { @@ -569,8 +587,7 @@ 'e7e5a8ce541110eb79bf98414171d3a1c137e32b' }, S3RegionRedirectorv2.redirect_from_error: { - 'ac37ca2ca48f7bde42d9659c01d5bd5bc08a78f9', - 'bc01047b596b1d1113df7cc2481af9cca2a49267', + '8e3003ec881c7eab0945fe4b6e021ca488fbcd78', }, S3RegionRedirectorv2.get_bucket_region: { 'b5bbc8b010576668dc2812d657c4b48af79e8f99' @@ -623,9 +640,8 @@ inject_presigned_url_rds: {'b5d45b339686346e81b255d4e8c36e76d3fe6a78'}, inject_presigned_url_ec2: {'48e09a5e4e95577e716be30f2d2706949261a07f'}, parse_get_bucket_location: {'64ffbf5c6aa6ebd083f49371000fa046d0de1fc6'}, - check_for_200_error: {'ded7f3aaef7b1a5d047c4dac86692ab55cbd7a13'}, - _looks_like_special_case_error: { - '86946722d10a72b593483fca0abf30100c609178' + check_for_200_error: { + '3a00f0bea409528f8457d6569aecf05998094386', }, # httpsession.py URLLib3Session: {