From 50f7d4d02e2c8e932737486eb27a24af2f4e243a Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 03:46:39 -0700 Subject: [PATCH 1/8] bypass await for moto --- aiobotocore/endpoint.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 81965b54..047c9ee4 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -1,4 +1,5 @@ import asyncio +from inspect import isawaitable from botocore.endpoint import ( DEFAULT_TIMEOUT, @@ -13,6 +14,7 @@ logger, ) from botocore.hooks import first_non_none_response +from botocore.utils import lowercase_dict from urllib3.response import HTTPHeaderDict from aiobotocore.httpchecksum import handle_checksum_body @@ -37,30 +39,27 @@ async def convert_to_response_dict(http_response, operation_model): """ response_dict = { - # botocore converts keys to str, so make sure that they are in - # the expected case. See detailed discussion here: - # https://github.com/aio-libs/aiobotocore/pull/116 - # aiohttp's CIMultiDict camel cases the headers :( - 'headers': HTTPHeaderDict( - { - k.decode('utf-8').lower(): v.decode('utf-8') - for k, v in http_response.raw.raw_headers - } - ), + 'headers': HTTPHeaderDict(lowercase_dict(http_response.headers)), 'status_code': http_response.status_code, 'context': { 'operation_name': operation_model.name, }, } if response_dict['status_code'] >= 300: - response_dict['body'] = await http_response.content + if isawaitable(http_response.content): + response_dict['body'] = await http_response.content + else: + response_dict['body'] = http_response.content elif operation_model.has_event_stream_output: response_dict['body'] = http_response.raw elif operation_model.has_streaming_output: length = response_dict['headers'].get('content-length') response_dict['body'] = StreamingBody(http_response.raw, length) else: - response_dict['body'] = await http_response.content + if isawaitable(http_response.content): + response_dict['body'] = await http_response.content + else: + response_dict['body'] = http_response.content return response_dict From 8c622779052b8a2ef90179586e1b202b1c2aa5cd Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 12:15:10 -0700 Subject: [PATCH 2/8] add tests for moto integration --- tests/test_response.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_response.py b/tests/test_response.py index 5a3bae26..c3870d1e 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,7 +1,12 @@ import io +import os +from unittest.mock import patch +import boto3 import pytest +import s3fs from botocore.exceptions import IncompleteReadError +from moto import mock_s3 from aiobotocore import response @@ -187,3 +192,44 @@ async def test_streaming_line_empty_body(): content_length=0, ) await assert_lines(stream.iter_lines(), []) + + +@pytest.fixture +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + + +@pytest.fixture +def s3(aws_credentials): + with mock_s3(): + conn = boto3.resource("s3") + conn.create_bucket(Bucket='testbucket') + yield conn + + +def test_moto_ok(s3): + for i in range(3): + s3.Bucket('testbucket').put_object( + Key=f'glob_{i}.txt', Body=f"test glob file {i}" + ) + path = 's3://testbucket/glob_*.txt' + fs = s3fs.S3FileSystem() + files = fs.glob(path) + assert len(files) == 3 + + +@patch('s3fs.core.aiobotocore.endpoint.isawaitable', return_value=True) +def test_moto_fail(mock_inspect, s3): + with pytest.raises(TypeError) as e: + for i in range(3): + s3.Bucket('testbucket').put_object( + Key=f'glob_{i}.txt', Body=f"test glob file {i}" + ) + path = 's3://testbucket/glob_*.txt' + fs = s3fs.S3FileSystem() + fs.glob(path) + assert "can't be used in 'await' expression" in str(e) From 37c598f36f545aabfe587be9750f169a51d37c4b Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 12:19:14 -0700 Subject: [PATCH 3/8] update CHANGES.rst --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e40529bd..d91a9d87 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,9 @@ Changes ------- +2.5.1 (2023-04-05) +^^^^^^^^^^^^^^^^^^ +* integrate moto + 2.5.0 (2023-03-06) ^^^^^^^^^^^^^^^^^^ * bump botocore to 1.29.76 (thanks @jakob-keller #999) From 9f20dc24d79906c405d15950bd90fc3086064433 Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 15:58:12 -0700 Subject: [PATCH 4/8] change order of tests --- tests/test_response.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_response.py b/tests/test_response.py index c3870d1e..9f4ea583 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -211,17 +211,6 @@ def s3(aws_credentials): yield conn -def test_moto_ok(s3): - for i in range(3): - s3.Bucket('testbucket').put_object( - Key=f'glob_{i}.txt', Body=f"test glob file {i}" - ) - path = 's3://testbucket/glob_*.txt' - fs = s3fs.S3FileSystem() - files = fs.glob(path) - assert len(files) == 3 - - @patch('s3fs.core.aiobotocore.endpoint.isawaitable', return_value=True) def test_moto_fail(mock_inspect, s3): with pytest.raises(TypeError) as e: @@ -233,3 +222,14 @@ def test_moto_fail(mock_inspect, s3): fs = s3fs.S3FileSystem() fs.glob(path) assert "can't be used in 'await' expression" in str(e) + + +def test_moto_ok(s3): + for i in range(3): + s3.Bucket('testbucket').put_object( + Key=f'glob_{i}.txt', Body=f"test glob file {i}" + ) + path = 's3://testbucket/glob_*.txt' + fs = s3fs.S3FileSystem() + files = fs.glob(path) + assert len(files) == 3 From 37e12f58ed7b07185350a17ace82e7942ca1fdfd Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 16:05:49 -0700 Subject: [PATCH 5/8] Add s3fs to dev env for tests --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 91456ff2..1c278505 100644 --- a/Pipfile +++ b/Pipfile @@ -18,6 +18,7 @@ pytest = "==6.2.4" pytest-cov = "==2.11.1" pytest-asyncio = "==0.14.0" pytest-xdist = "==2.2.1" +s3fs = "==2023.3.0" # this is needed for test_patches dill = "==0.3.3" From 5e2fc16e459f6bc4d3703a8429962d506b613ea5 Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 22:13:50 -0700 Subject: [PATCH 6/8] Fix to continue using raw_headers --- aiobotocore/endpoint.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/aiobotocore/endpoint.py b/aiobotocore/endpoint.py index 047c9ee4..7c1a8fb0 100644 --- a/aiobotocore/endpoint.py +++ b/aiobotocore/endpoint.py @@ -15,6 +15,7 @@ ) from botocore.hooks import first_non_none_response from botocore.utils import lowercase_dict +from moto.core.botocore_stubber import MockRawResponse from urllib3.response import HTTPHeaderDict from aiobotocore.httpchecksum import handle_checksum_body @@ -39,12 +40,28 @@ async def convert_to_response_dict(http_response, operation_model): """ response_dict = { - 'headers': HTTPHeaderDict(lowercase_dict(http_response.headers)), 'status_code': http_response.status_code, 'context': { 'operation_name': operation_model.name, }, } + + if isinstance(http_response.raw, MockRawResponse): + # patch to integrate with moto + response_dict['headers'] = HTTPHeaderDict( + lowercase_dict(http_response.headers) + ) + else: + # botocore converts keys to str, so make sure that they are in + # the expected case. See detailed discussion here: + # https://github.com/aio-libs/aiobotocore/pull/116 + # aiohttp's CIMultiDict camel cases the headers :( + response_dict['headers'] = HTTPHeaderDict( + { + k.decode('utf-8').lower(): v.decode('utf-8') + for k, v in http_response.raw.raw_headers + } + ) if response_dict['status_code'] >= 300: if isawaitable(http_response.content): response_dict['body'] = await http_response.content From b47ee262002c76765a7381e0f2989aed711fe0ff Mon Sep 17 00:00:00 2001 From: akshara08 Date: Wed, 5 Apr 2023 22:14:15 -0700 Subject: [PATCH 7/8] Remove s3fs dependency for testing --- Pipfile | 1 - tests/conftest.py | 8 +++++ tests/test_response.py | 72 +++++++++++++++++++----------------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/Pipfile b/Pipfile index 1c278505..91456ff2 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,6 @@ pytest = "==6.2.4" pytest-cov = "==2.11.1" pytest-asyncio = "==0.14.0" pytest-xdist = "==2.2.1" -s3fs = "==2023.3.0" # this is needed for test_patches dill = "==0.3.3" diff --git a/tests/conftest.py b/tests/conftest.py index 62581876..57d8d17a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ # Third Party import pytest +from moto import mock_s3 import aiobotocore.session from aiobotocore.config import AioConfig @@ -583,4 +584,11 @@ async def exit_stack(): yield es +@pytest.fixture +async def moto_client(session): + with mock_s3(): + async with session.create_client('s3') as client: + yield client + + pytest_plugins = ['tests.mock_server'] diff --git a/tests/test_response.py b/tests/test_response.py index 9f4ea583..20c69b75 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -1,12 +1,10 @@ import io -import os from unittest.mock import patch -import boto3 import pytest -import s3fs from botocore.exceptions import IncompleteReadError -from moto import mock_s3 +from moto.core.botocore_stubber import MockRawResponse +from urllib3.response import HTTPHeaderDict from aiobotocore import response @@ -194,42 +192,38 @@ async def test_streaming_line_empty_body(): await assert_lines(stream.iter_lines(), []) -@pytest.fixture -def aws_credentials(): - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURITY_TOKEN"] = "testing" - os.environ["AWS_SESSION_TOKEN"] = "testing" +@patch('aiobotocore.endpoint.isawaitable', return_value=True) +@pytest.mark.asyncio +@pytest.mark.moto +async def test_moto_fail(mock_awaitable, moto_client): + with pytest.raises(TypeError) as e: + await moto_client.create_bucket(Bucket='testbucket') + assert "can't be used in 'await' expression" in str(e) -@pytest.fixture -def s3(aws_credentials): - with mock_s3(): - conn = boto3.resource("s3") - conn.create_bucket(Bucket='testbucket') - yield conn +@patch('aiobotocore.endpoint.convert_to_response_dict') +@pytest.mark.asyncio +@pytest.mark.moto +async def test_moto_mockrawresponse(mock_response_dict, moto_client): + mock_response_dict.return_value = { + 'status_code': 200, + 'context': {'operation_name': 'CreateBucket'}, + 'headers': HTTPHeaderDict( + { + 'x-amzn-requestid': 'waEAf9728rSOyzXaRaiURPOBgJVY206J3ZEhRXEdhhxH5ZFY1TnU' + } + ), + 'body': b'' + b'testbucket' + b'', + } + await moto_client.create_bucket(Bucket='testbucket') + args, kwargs = mock_response_dict.call_args + assert isinstance(args[0].raw, MockRawResponse) -@patch('s3fs.core.aiobotocore.endpoint.isawaitable', return_value=True) -def test_moto_fail(mock_inspect, s3): - with pytest.raises(TypeError) as e: - for i in range(3): - s3.Bucket('testbucket').put_object( - Key=f'glob_{i}.txt', Body=f"test glob file {i}" - ) - path = 's3://testbucket/glob_*.txt' - fs = s3fs.S3FileSystem() - fs.glob(path) - assert "can't be used in 'await' expression" in str(e) - - -def test_moto_ok(s3): - for i in range(3): - s3.Bucket('testbucket').put_object( - Key=f'glob_{i}.txt', Body=f"test glob file {i}" - ) - path = 's3://testbucket/glob_*.txt' - fs = s3fs.S3FileSystem() - files = fs.glob(path) - assert len(files) == 3 +@pytest.mark.asyncio +@pytest.mark.moto +async def test_moto_ok(moto_client): + response = await moto_client.create_bucket(Bucket='testbucket') + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 From e9c670c7186ed3ab673950b08b13e5b403b321db Mon Sep 17 00:00:00 2001 From: akshara08 Date: Thu, 6 Apr 2023 11:10:13 -0700 Subject: [PATCH 8/8] Added one more test for raw_headers fail --- tests/test_response.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_response.py b/tests/test_response.py index 20c69b75..3d29dce0 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -195,12 +195,21 @@ async def test_streaming_line_empty_body(): @patch('aiobotocore.endpoint.isawaitable', return_value=True) @pytest.mark.asyncio @pytest.mark.moto -async def test_moto_fail(mock_awaitable, moto_client): +async def test_moto_fail_await(mock_awaitable, moto_client): with pytest.raises(TypeError) as e: await moto_client.create_bucket(Bucket='testbucket') assert "can't be used in 'await' expression" in str(e) +@patch('aiobotocore.endpoint.isinstance', return_value=False) +@pytest.mark.asyncio +@pytest.mark.moto +async def test_moto_fail_raw_headers(mock_isinstance, moto_client): + with pytest.raises(AttributeError) as e: + await moto_client.create_bucket(Bucket='testbucket') + assert "object has no attribute 'raw_headers" in str(e) + + @patch('aiobotocore.endpoint.convert_to_response_dict') @pytest.mark.asyncio @pytest.mark.moto