aiobotocore AttributeError - missing raw_headers

[giacomomasoncma]

I'm using moto to test functions that write tabular data to s3. I use tox + pytest. Here is the py37 environment that tox builds to run pytest:

py37 installed: 
aiobotocore==1.1.0,
aiohttp==3.6.2,
aioitertools==0.7.0,
async-timeout==3.0.1,
attrs==20.1.0,
aws-sam-translator==1.26.0,
aws-xray-sdk==2.6.0,
boto==2.49.0,
boto3==1.14.48,
botocore==1.17.48,
Cerberus==1.3.2,
certifi==2020.6.20,
cffi==1.14.2,
cfn-lint==0.35.0,
chardet==3.0.4,
cryptography==3.0,
decorator==4.4.2,
docker==4.3.1,
docutils==0.15.2,
ecdsa==0.14.1,
fsspec==0.8.0,
future==0.18.2,
idna==2.8,
importlib-metadata==1.7.0,
iniconfig==1.0.1,
Jinja2==2.11.2,
jmespath==0.10.0,
jsondiff==1.1.2,
jsonpatch==1.26,
jsonpickle==1.4.1,
jsonpointer==2.0,
jsonschema==3.2.0,
junit-xml==1.9,
MarkupSafe==1.1.1,
mock==4.0.2,
more-itertools==8.4.0,
moto==1.3.14,
multidict==4.7.6,
networkx==2.5,
numpy==1.19.1,
packaging==20.4,
pandas==1.1.1,
pluggy==0.13.1,
py==1.9.0,
pyarrow==1.0.1,
pyasn1==0.4.8,
pycparser==2.20,
pyparsing==2.4.7,
pyrsistent==0.16.0,
pytest==6.0.1,
python-dateutil==2.8.1,
python-jose==3.2.0,
pytz==2020.1,
PyYAML==5.3.1,
requests==2.24.0,
responses==0.11.0,
rsa==4.6,
s3fs==0.5.0,
s3transfer==0.3.3,
six==1.15.0,
sshpubkeys==3.1.0,
toml==0.10.1,
typing-extensions==3.7.4.3,
urllib3==1.25.10,
websocket-client==0.57.0,
Werkzeug==1.0.1,
wrapt==1.12.1,
xmltodict==0.12.0,
yarl==1.5.1,
zipp==3.1.0

This is the relevant snipped of the function I'm trying to test:

def write_data(df, key, bucket=BUCKET, tags=None, verbose=True, **kwargs):
    [...]
    if key_extension == ".parquet":
        df.to_parquet(full_path, **kwargs)
    elif key_extension == ".csv":
        df.to_csv(full_path, **kwargs)
    else:
        raise NotImplementedError(f"Extension {key_extension} not supported yet.")
    [...]

I've recently been getting a new error from aiobotocore when calling the to_csv method in pandas to write to my mock s3 instance:

def test_write_read_data(s3_moto, mock_bucket, data_example_df, data_example_tags):
    [...]
        # writing to new file with tags, and reading back
        supported_extensions = ["csv", "parquet"]
        for ext in supported_extensions:
            lz.s3.write_data(
                data_example_df,
                f"new_file.{ext}",
                bucket=mock_bucket,
                tags=data_example_tags,
>               index=False,
            )
tests/aws/s3/test_data.py:44: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.tox/py37/lib/python3.7/site-packages/ledatoolz/aws/s3/data.py:63: in write_data
    df.to_csv(full_path, **kwargs)
.tox/py37/lib/python3.7/site-packages/pandas/core/generic.py:3165: in to_csv
    decimal=decimal,
.tox/py37/lib/python3.7/site-packages/pandas/io/formats/csvs.py:67: in __init__
    path_or_buf, encoding=encoding, compression=compression, mode=mode
.tox/py37/lib/python3.7/site-packages/pandas/io/common.py:222: in get_filepath_or_buffer
    filepath_or_buffer, mode=mode or "rb", **(storage_options or {})
.tox/py37/lib/python3.7/site-packages/fsspec/core.py:399: in open
    **kwargs
.tox/py37/lib/python3.7/site-packages/fsspec/core.py:254: in open_files
    [fs.makedirs(parent, exist_ok=True) for parent in parents]
.tox/py37/lib/python3.7/site-packages/fsspec/core.py:254: in <listcomp>
    [fs.makedirs(parent, exist_ok=True) for parent in parents]
.tox/py37/lib/python3.7/site-packages/s3fs/core.py:460: in makedirs
    self.mkdir(path, create_parents=True)
.tox/py37/lib/python3.7/site-packages/fsspec/asyn.py:100: in wrapper
    return maybe_sync(func, self, *args, **kwargs)
.tox/py37/lib/python3.7/site-packages/fsspec/asyn.py:80: in maybe_sync
    return sync(loop, func, *args, **kwargs)
.tox/py37/lib/python3.7/site-packages/fsspec/asyn.py:51: in sync
    raise exc.with_traceback(tb)
.tox/py37/lib/python3.7/site-packages/fsspec/asyn.py:35: in f
    result[0] = await future
.tox/py37/lib/python3.7/site-packages/s3fs/core.py:446: in _mkdir
    await self.s3.create_bucket(**params)
.tox/py37/lib/python3.7/site-packages/aiobotocore/client.py:121: in _make_api_call
    operation_model, request_dict, request_context)
.tox/py37/lib/python3.7/site-packages/aiobotocore/client.py:140: in _make_request
    return await self._endpoint.make_request(operation_model, request_dict)
.tox/py37/lib/python3.7/site-packages/aiobotocore/endpoint.py:87: in _send_request
    request, operation_model, context)
.tox/py37/lib/python3.7/site-packages/aiobotocore/endpoint.py:120: in _get_response
    request, operation_model)
.tox/py37/lib/python3.7/site-packages/aiobotocore/endpoint.py:170: in _do_get_response
    operation_model)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
http_response = <botocore.awsrequest.AWSResponse object at 0x7f9affb32550>
operation_model = OperationModel(name=CreateBucket)
    async def convert_to_response_dict(http_response, operation_model):
        """Convert an HTTP response object to a request dict.
    
        This converts the requests library's HTTP response object to
        a dictionary.
    
        :type http_response: botocore.vendored.requests.model.Response
        :param http_response: The HTTP response from an AWS service request.
    
        :rtype: dict
        :return: A response dictionary which will contain the following keys:
            * headers (dict)
            * status_code (int)
            * body (string or file-like object)
    
        """
        response_dict = {
            # botocore converts keys to str, so make sure that they are in
            # the expected case. See detailed discussion here:
            # https://github.com/aio-libs/aiobotocore/pull/116
            # aiohttp's CIMultiDict camel cases the headers :(
            'headers': HTTPHeaderDict(
                {k.decode('utf-8').lower(): v.decode('utf-8')
>                for k, v in http_response.raw_headers}),
            'status_code': http_response.status_code,
            'context': {
                'operation_name': operation_model.name,
            }
        }
E       AttributeError: 'AWSResponse' object has no attribute 'raw_headers'
.tox/py37/lib/python3.7/site-packages/aiobotocore/endpoint.py:43: AttributeError

[bblommers]

Hi @giacomomasoncma, this is a aiobotocore specific issue - not related to moto.
Looks to be a duplicate of #2706 and aio-libs/aiobotocore#755

Closing this.