Moto integration

CHANGES.rst

@@ -1,5 +1,9 @@
 Changes
 -------
+2.5.1 (2023-04-05)
+^^^^^^^^^^^^^^^^^^
+* integrate moto
+
 2.5.0 (2023-03-06)
 ^^^^^^^^^^^^^^^^^^
 * bump botocore to 1.29.76 (thanks @jakob-keller #999)

Pipfile

@@ -18,6 +18,7 @@ pytest = "==6.2.4"
 pytest-cov = "==2.11.1"
 pytest-asyncio = "==0.14.0"
 pytest-xdist = "==2.2.1"
+s3fs = "==2023.3.0"
 
 # this is needed for test_patches
 dill = "==0.3.3"

aiobotocore/endpoint.py

@@ -1,4 +1,5 @@
 import asyncio
+from inspect import isawaitable
 
 from botocore.endpoint import (
     DEFAULT_TIMEOUT,
@@ -13,6 +14,7 @@
     logger,
 )
 from botocore.hooks import first_non_none_response
+from botocore.utils import lowercase_dict
 from urllib3.response import HTTPHeaderDict
 
 from aiobotocore.httpchecksum import handle_checksum_body
@@ -37,30 +39,27 @@ async def convert_to_response_dict(http_response, operation_model):
 
     """
     response_dict = {
-        # botocore converts keys to str, so make sure that they are in
-        # the expected case. See detailed discussion here:
-        # https://github.com/aio-libs/aiobotocore/pull/116
-        # aiohttp's CIMultiDict camel cases the headers :(
-        'headers': HTTPHeaderDict(
-            {
-                k.decode('utf-8').lower(): v.decode('utf-8')
-                for k, v in http_response.raw.raw_headers
-            }
-        ),
+        'headers': HTTPHeaderDict(lowercase_dict(http_response.headers)),
         'status_code': http_response.status_code,
         'context': {
             'operation_name': operation_model.name,
         },
     }
     if response_dict['status_code'] >= 300:
-        response_dict['body'] = await http_response.content
+        if isawaitable(http_response.content):
+            response_dict['body'] = await http_response.content
+        else:
+            response_dict['body'] = http_response.content
     elif operation_model.has_event_stream_output:
         response_dict['body'] = http_response.raw
     elif operation_model.has_streaming_output:
         length = response_dict['headers'].get('content-length')
         response_dict['body'] = StreamingBody(http_response.raw, length)
     else:
-        response_dict['body'] = await http_response.content
+        if isawaitable(http_response.content):
+            response_dict['body'] = await http_response.content
+        else:
+            response_dict['body'] = http_response.content
     return response_dict
 
 

tests/test_response.py

@@ -1,7 +1,12 @@
 import io
+import os
+from unittest.mock import patch
 
+import boto3
 import pytest
+import s3fs
 from botocore.exceptions import IncompleteReadError
+from moto import mock_s3
 
 from aiobotocore import response
 
@@ -187,3 +192,44 @@ async def test_streaming_line_empty_body():
         content_length=0,
     )
     await assert_lines(stream.iter_lines(), [])
+
+
+@pytest.fixture
+def aws_credentials():
+    """Mocked AWS Credentials for moto."""
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+
+
+@pytest.fixture
+def s3(aws_credentials):
+    with mock_s3():
+        conn = boto3.resource("s3")
+        conn.create_bucket(Bucket='testbucket')
+        yield conn
+
+
+@patch('s3fs.core.aiobotocore.endpoint.isawaitable', return_value=True)
+def test_moto_fail(mock_inspect, s3):
+    with pytest.raises(TypeError) as e:
+        for i in range(3):
+            s3.Bucket('testbucket').put_object(
+                Key=f'glob_{i}.txt', Body=f"test glob file {i}"
+            )
+        path = 's3://testbucket/glob_*.txt'
+        fs = s3fs.S3FileSystem()
+        fs.glob(path)
+    assert "can't be used in 'await' expression" in str(e)
+
+
+def test_moto_ok(s3):
+    for i in range(3):
+        s3.Bucket('testbucket').put_object(
+            Key=f'glob_{i}.txt', Body=f"test glob file {i}"
+        )
+    path = 's3://testbucket/glob_*.txt'
+    fs = s3fs.S3FileSystem()
+    files = fs.glob(path)
+    assert len(files) == 3