From ccc5e071d5ee58ff8fbc72b65f41783293bf095b Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 16 Sep 2019 16:15:47 -0700 Subject: [PATCH 1/4] Revert "Always use raw response data. (#87)" This reverts commit 2b9ffc813b47912b5609985db9dc5d9e4936310a. --- google/resumable_media/_download.py | 47 ++----- google/resumable_media/requests/_helpers.py | 8 +- google/resumable_media/requests/download.py | 104 +++++++++++---- noxfile.py | 2 +- tests/system/requests/test_download.py | 41 +++++- tests/unit/requests/test__helpers.py | 10 ++ tests/unit/requests/test_download.py | 137 +++++++++++++------- tests/unit/test__download.py | 77 +++-------- 8 files changed, 248 insertions(+), 178 deletions(-) diff --git a/google/resumable_media/_download.py b/google/resumable_media/_download.py index 5d2d10d1..9d194c47 100644 --- a/google/resumable_media/_download.py +++ b/google/resumable_media/_download.py @@ -349,41 +349,20 @@ def _process_response(self, response): return _helpers.require_status_code( - response, - _ACCEPTABLE_STATUS_CODES, - self._get_status_code, - callback=self._make_invalid, - ) - headers = self._get_headers(response) + response, _ACCEPTABLE_STATUS_CODES, + self._get_status_code, callback=self._make_invalid) + content_length = _helpers.header_required( + response, u'content-length', self._get_headers, + callback=self._make_invalid) + num_bytes = int(content_length) + _, end_byte, total_bytes = get_range_info( + response, self._get_headers, callback=self._make_invalid) response_body = self._get_body(response) - - start_byte, end_byte, total_bytes = get_range_info( - response, self._get_headers, callback=self._make_invalid - ) - - transfer_encoding = headers.get(u"transfer-encoding") - - if transfer_encoding is None: - content_length = _helpers.header_required( - response, - u"content-length", - self._get_headers, - callback=self._make_invalid, - ) - num_bytes = int(content_length) - if len(response_body) != num_bytes: - self._make_invalid() - raise common.InvalidResponse( - response, - u"Response is different size than content-length", - u"Expected", - num_bytes, - u"Received", - len(response_body), - ) - else: - # 'content-length' header not allowed with chunked encoding. - num_bytes = end_byte - start_byte + 1 + if len(response_body) != num_bytes: + self._make_invalid() + raise common.InvalidResponse( + response, u'Response is different size than content-length', + u'Expected', num_bytes, u'Received', len(response_body)) # First update ``bytes_downloaded``. self._bytes_downloaded += num_bytes diff --git a/google/resumable_media/requests/_helpers.py b/google/resumable_media/requests/_helpers.py index 80cf4542..b7343ce5 100644 --- a/google/resumable_media/requests/_helpers.py +++ b/google/resumable_media/requests/_helpers.py @@ -25,7 +25,6 @@ _DEFAULT_RETRY_STRATEGY = common.RetryStrategy() -_SINGLE_GET_CHUNK_SIZE = 8192 # The number of seconds to wait to establish a connection # (connect() call on socket). Avoid setting this to a multiple of 3 to not # Align with TCP Retransmission timing. (typically 2.5-3s) @@ -76,12 +75,7 @@ def _get_body(response): Returns: bytes: The body of the ``response``. """ - if response._content is False: - response._content = b"".join( - response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False) - ) - response._content_consumed = True - return response._content + return response.content def http_request( diff --git a/google/resumable_media/requests/download.py b/google/resumable_media/requests/download.py index a41ff623..d63c58ba 100644 --- a/google/resumable_media/requests/download.py +++ b/google/resumable_media/requests/download.py @@ -18,13 +18,16 @@ import hashlib import logging +import urllib3.response + from google.resumable_media import _download from google.resumable_media import common from google.resumable_media.requests import _helpers _LOGGER = logging.getLogger(__name__) -_HASH_HEADER = u"x-goog-hash" +_SINGLE_GET_CHUNK_SIZE = 8192 +_HASH_HEADER = u'x-goog-hash' _MISSING_MD5 = u"""\ No MD5 checksum was returned from the service while downloading {} (which happens for composite objects), so client-side content integrity @@ -113,13 +116,12 @@ def _write_to_stream(self, response): with response: # NOTE: This might "donate" ``md5_hash`` to the decoder and replace # it with a ``_DoNothingHash``. - body_iter = response.raw.stream( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + local_hash = _add_decoder(response.raw, md5_hash) + body_iter = response.iter_content( + chunk_size=_SINGLE_GET_CHUNK_SIZE, decode_unicode=False) for chunk in body_iter: self._stream.write(chunk) - md5_hash.update(chunk) - response._content_consumed = True + local_hash.update(chunk) if expected_md5_hash is None: return @@ -155,15 +157,16 @@ def consume(self, transport): """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. - response = _helpers.http_request( - transport, - method, - url, - data=payload, - headers=headers, - retry_strategy=self._retry_strategy, - stream=True, - ) + request_kwargs = { + u'data': payload, + u'headers': headers, + u'retry_strategy': self._retry_strategy, + } + if self._stream is not None: + request_kwargs[u'stream'] = True + + result = _helpers.http_request( + transport, method, url, **request_kwargs) self._process_response(response) @@ -216,17 +219,11 @@ def consume_next_chunk(self, transport): """ method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. - response = _helpers.http_request( - transport, - method, - url, - data=payload, - headers=headers, - retry_strategy=self._retry_strategy, - stream=True, - ) - self._process_response(response) - return response + result = _helpers.http_request( + transport, method, url, data=payload, headers=headers, + retry_strategy=self._retry_strategy) + self._process_response(result) + return result def _parse_md5_header(header_value, response): @@ -294,3 +291,58 @@ def update(self, unused_chunk): Args: unused_chunk (bytes): A chunk of data. """ + + +def _add_decoder(response_raw, md5_hash): + """Patch the ``_decoder`` on a ``urllib3`` response. + + This is so that we can intercept the compressed bytes before they are + decoded. + + Only patches if the content encoding is ``gzip``. + + Args: + response_raw (urllib3.response.HTTPResponse): The raw response for + an HTTP request. + md5_hash (Union[_DoNothingHash, hashlib.md5]): A hash function which + will get updated when it encounters compressed bytes. + + Returns: + Union[_DoNothingHash, hashlib.md5]: Either the original ``md5_hash`` + if ``_decoder`` is not patched. Otherwise, returns a ``_DoNothingHash`` + since the caller will no longer need to hash to decoded bytes. + """ + encoding = response_raw.headers.get(u'content-encoding', u'').lower() + if encoding != u'gzip': + return md5_hash + + response_raw._decoder = _GzipDecoder(md5_hash) + return _DoNothingHash() + + +class _GzipDecoder(urllib3.response.GzipDecoder): + """Custom subclass of ``urllib3`` decoder for ``gzip``-ed bytes. + + Allows an MD5 hash function to see the compressed bytes before they are + decoded. This way the hash of the compressed value can be computed. + + Args: + md5_hash (Union[_DoNothingHash, hashlib.md5]): A hash function which + will get updated when it encounters compressed bytes. + """ + + def __init__(self, md5_hash): + super(_GzipDecoder, self).__init__() + self._md5_hash = md5_hash + + def decompress(self, data): + """Decompress the bytes. + + Args: + data (bytes): The compressed bytes to be decompressed. + + Returns: + bytes: The decompressed bytes from ``data``. + """ + self._md5_hash.update(data) + return super(_GzipDecoder, self).decompress(data) diff --git a/noxfile.py b/noxfile.py index 22d86cd3..b419f445 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,7 +24,7 @@ GOOGLE_AUTH = 'google-auth >= 0.10.0' -@nox.session(python=['2.7', '3.4', '3.5', '3.6', '3.7']) +@nox.session(python=['2,7', '3.4', '3.5', '3.6', '3.7']) def unit_tests(session): """Run the unit test suite.""" diff --git a/tests/system/requests/test_download.py b/tests/system/requests/test_download.py index 9314dcb6..cf99230b 100644 --- a/tests/system/requests/test_download.py +++ b/tests/system/requests/test_download.py @@ -25,9 +25,8 @@ from six.moves import http_client from google import resumable_media -from google.resumable_media import requests as resumable_requests -from google.resumable_media.requests import download as download_mod -from google.resumable_media.requests import _helpers +import google.resumable_media.requests as resumable_requests +import google.resumable_media.requests.download as download_mod from tests.system import utils @@ -57,6 +56,7 @@ slice(-256, None, None), # obj[-256:] slice(262144, None, None), # obj[262144:] ), +<<<<<<< HEAD }, { u"path": os.path.realpath(os.path.join(DATA_DIR, u"file.txt")), @@ -70,6 +70,23 @@ u"checksum": u"KHRs/+ZSrc/FuuR4qz/PZQ==", u"slices": (), u"metadata": {u"contentEncoding": u"gzip"}, +======= + }, { + u'path': os.path.realpath(os.path.join(DATA_DIR, u'file.txt')), + u'content_type': PLAIN_TEXT, + u'checksum': u'KHRs/+ZSrc/FuuR4qz/PZQ==', + u'slices': (), + }, { + u'path': os.path.realpath(os.path.join(DATA_DIR, u'gzipped.txt.gz')), + u'uncompressed': + os.path.realpath(os.path.join(DATA_DIR, u'gzipped.txt')), + u'content_type': PLAIN_TEXT, + u'checksum': u'KHRs/+ZSrc/FuuR4qz/PZQ==', + u'slices': (), + u'metadata': { + u'contentEncoding': u'gzip', + }, +>>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) }, ) ENCRYPTED_ERR = b"The target object is encrypted by a customer-supplied encryption key." @@ -126,13 +143,22 @@ def _get_contents_for_upload(info): def _get_contents(info): +<<<<<<< HEAD full_path = info[u"path"] with open(full_path, u"rb") as file_obj: +======= + full_path = info.get(u'uncompressed', info[u'path']) + with open(full_path, u'rb') as file_obj: +>>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) return file_obj.read() def _get_blob_name(info): +<<<<<<< HEAD full_path = info[u"path"] +======= + full_path = info.get(u'uncompressed', info[u'path']) +>>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) return os.path.basename(full_path) @@ -179,12 +205,15 @@ def check_tombstoned(download, transport): assert exc_info.match(u"Download has finished.") +<<<<<<< HEAD def read_raw_content(response): return b"".join( response.raw.stream(_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False) ) +======= +>>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) def test_download_full(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) @@ -196,7 +225,7 @@ def test_download_full(add_files, authorized_transport): # Consume the resource. response = download.consume(authorized_transport) assert response.status_code == http_client.OK - assert read_raw_content(response) == actual_contents + assert response.content == actual_contents check_tombstoned(download, authorized_transport) @@ -221,6 +250,7 @@ def test_download_to_stream(add_files, authorized_transport): check_tombstoned(download, authorized_transport) +@pytest.mark.xfail # See: #76 def test_corrupt_download(add_files, corrupting_transport): for info in ALL_FILES: blob_name = _get_blob_name(info) @@ -396,7 +426,8 @@ def consume_chunks(download, authorized_transport, total_bytes, actual_contents) return num_responses, response -def test_chunked_download_full(add_files, authorized_transport): +@pytest.mark.xfail # See issue #56 +def test_chunked_download(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) blob_name = _get_blob_name(info) diff --git a/tests/unit/requests/test__helpers.py b/tests/unit/requests/test__helpers.py index 81d6a50e..9029243c 100644 --- a/tests/unit/requests/test__helpers.py +++ b/tests/unit/requests/test__helpers.py @@ -27,6 +27,7 @@ def test__get_status_code(self): assert status_code == _helpers.RequestsMixin._get_status_code(response) def test__get_headers(self): +<<<<<<< HEAD headers = {u"fruit": u"apple"} response = mock.Mock(headers=headers, spec=["headers"]) assert headers == _helpers.RequestsMixin._get_headers(response) @@ -44,6 +45,15 @@ def test__get_body_wo_content_consumed(self): def test__get_body_w_content_consumed(self): body = b"This is the payload." response = mock.Mock(_content=body, spec=["_content"]) +======= + headers = {u'fruit': u'apple'} + response = mock.Mock(headers=headers, spec=[u'headers']) + assert headers == _helpers.RequestsMixin._get_headers(response) + + def test__get_body(self): + body = b'This is the payload.' + response = mock.Mock(content=body, spec=[u'content']) +>>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) assert body == _helpers.RequestsMixin._get_body(response) diff --git a/tests/unit/requests/test_download.py b/tests/unit/requests/test_download.py index 54a14c28..3e724a73 100644 --- a/tests/unit/requests/test_download.py +++ b/tests/unit/requests/test_download.py @@ -19,8 +19,7 @@ from six.moves import http_client from google.resumable_media import common -from google.resumable_media.requests import download as download_mod -from google.resumable_media.requests import _helpers +import google.resumable_media.requests.download as download_mod EXAMPLE_URL = ( @@ -31,7 +30,8 @@ class TestDownload(object): - @mock.patch("google.resumable_media.requests.download._LOGGER") + + @mock.patch(u'google.resumable_media.requests.download._LOGGER') def test__get_expected_md5_present(self, _LOGGER): download = download_mod.Download(EXAMPLE_URL) @@ -44,7 +44,7 @@ def test__get_expected_md5_present(self, _LOGGER): assert expected_md5_hash == checksum _LOGGER.info.assert_not_called() - @mock.patch("google.resumable_media.requests.download._LOGGER") + @mock.patch(u'google.resumable_media.requests.download._LOGGER') def test__get_expected_md5_missing(self, _LOGGER): download = download_mod.Download(EXAMPLE_URL) @@ -72,9 +72,9 @@ def test__write_to_stream_no_hash_check(self): # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) def test__write_to_stream_with_hash_check_success(self): stream = io.BytesIO() @@ -95,9 +95,9 @@ def test__write_to_stream_with_hash_check_success(self): # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) def test__write_to_stream_with_hash_check_fail(self): stream = io.BytesIO() @@ -128,17 +128,16 @@ def test__write_to_stream_with_hash_check_fail(self): # Check mocks. response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) def _consume_helper( self, stream=None, end=65536, headers=None, chunks=(), response_headers=None ): download = download_mod.Download( - EXAMPLE_URL, stream=stream, end=end, headers=headers - ) - transport = mock.Mock(spec=["request"]) + EXAMPLE_URL, stream=stream, end=end, headers=headers) + transport = mock.Mock(spec=[u'request']) transport.request.return_value = _mock_response( chunks=chunks, headers=response_headers ) @@ -147,17 +146,12 @@ def _consume_helper( ret_val = download.consume(transport) assert ret_val is transport.request.return_value + called_kwargs = {u'data': None, u'headers': download._headers} if chunks: assert stream is not None - + called_kwargs[u'stream'] = True transport.request.assert_called_once_with( - u"GET", - EXAMPLE_URL, - data=None, - headers=download._headers, - stream=True, - timeout=EXPECTED_TIMEOUT, - ) + u'GET', EXAMPLE_URL, timeout=EXPECTED_TIMEOUT, **called_kwargs) range_bytes = u"bytes={:d}-{:d}".format(0, end) assert download._headers[u"range"] == range_bytes @@ -179,9 +173,9 @@ def test_consume_with_stream(self): response = transport.request.return_value response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) def test_consume_with_stream_hash_check_success(self): stream = io.BytesIO() @@ -198,9 +192,9 @@ def test_consume_with_stream_hash_check_success(self): response = transport.request.return_value response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) - response.raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) + response.iter_content.assert_called_once_with( + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, + decode_unicode=False) def test_consume_with_stream_hash_check_fail(self): stream = io.BytesIO() @@ -210,8 +204,9 @@ def test_consume_with_stream_hash_check_fail(self): bad_checksum = u"anVzdCBub3QgdGhpcyAxLA==" header_value = u"crc32c=V0FUPw==,md5={}".format(bad_checksum) headers = {download_mod._HASH_HEADER: header_value} - transport = mock.Mock(spec=["request"]) - transport.request.return_value = _mock_response(chunks=chunks, headers=headers) + transport = mock.Mock(spec=[u'request']) + transport.request.return_value = _mock_response( + chunks=chunks, headers=headers) assert not download.finished with pytest.raises(common.DataCorruption) as exc_info: @@ -267,10 +262,14 @@ def _mock_response( ): response_headers = self._response_headers(start_byte, end_byte, total_bytes) return mock.Mock( - _content=content, + content=content, headers=response_headers, status_code=status_code, - spec=[u"_content", u"headers", u"status_code"], + spec=[ + u'content', + u'headers', + u'status_code', + ], ) def test_consume_next_chunk_already_finished(self): @@ -279,8 +278,8 @@ def test_consume_next_chunk_already_finished(self): with pytest.raises(ValueError): download.consume_next_chunk(None) - def _mock_transport(self, start, chunk_size, total_bytes, content=b""): - transport = mock.Mock(spec=["request"]) + def _mock_transport(self, start, chunk_size, total_bytes, content=b''): + transport = mock.Mock(spec=[u'request']) assert len(content) == chunk_size transport.request.return_value = self._mock_response( start, @@ -313,13 +312,8 @@ def test_consume_next_chunk(self): range_bytes = u"bytes={:d}-{:d}".format(start, start + chunk_size - 1) download_headers = {u"range": range_bytes} transport.request.assert_called_once_with( - u"GET", - EXAMPLE_URL, - data=None, - headers=download_headers, - stream=True, - timeout=EXPECTED_TIMEOUT, - ) + u'GET', EXAMPLE_URL, data=None, headers=download_headers, + timeout=EXPECTED_TIMEOUT) assert stream.getvalue() == data # Go back and check the internal state after consuming the chunk. assert not download.finished @@ -379,26 +373,75 @@ def test__DoNothingHash(): assert return_value is None +class Test__add_decoder(object): + + def test_non_gzipped(self): + response_raw = mock.Mock(headers={}, spec=[u'headers']) + md5_hash = download_mod._add_decoder( + response_raw, mock.sentinel.md5_hash) + + assert md5_hash is mock.sentinel.md5_hash + + def test_gzipped(self): + headers = {u'content-encoding': u'gzip'} + response_raw = mock.Mock( + headers=headers, spec=[u'headers', u'_decoder']) + md5_hash = download_mod._add_decoder( + response_raw, mock.sentinel.md5_hash) + + assert md5_hash is not mock.sentinel.md5_hash + assert isinstance(md5_hash, download_mod._DoNothingHash) + assert isinstance(response_raw._decoder, download_mod._GzipDecoder) + assert response_raw._decoder._md5_hash is mock.sentinel.md5_hash + + +class Test_GzipDecoder(object): + + def test_constructor(self): + decoder = download_mod._GzipDecoder(mock.sentinel.md5_hash) + assert decoder._md5_hash is mock.sentinel.md5_hash + + def test_decompress(self): + md5_hash = mock.Mock(spec=['update']) + decoder = download_mod._GzipDecoder(md5_hash) + + data = b'\x1f\x8b\x08\x08' + result = decoder.decompress(data) + + assert result == b'' + md5_hash.update.assert_called_once_with(data) + + def _mock_response(status_code=http_client.OK, chunks=(), headers=None): if headers is None: headers = {} if chunks: - mock_raw = mock.Mock(headers=headers, spec=["headers", "stream"]) - mock_raw.stream.return_value = iter(chunks) + mock_raw = mock.Mock(headers=headers, spec=[u'headers']) response = mock.MagicMock( headers=headers, status_code=int(status_code), raw=mock_raw, - spec=["__enter__", "__exit__", "raw", "status_code", "headers", "raw"], + spec=[ + u'__enter__', + u'__exit__', + u'iter_content', + u'status_code', + u'headers', + u'raw', + ], ) # i.e. context manager returns ``self``. response.__enter__.return_value = response response.__exit__.return_value = None + response.iter_content.return_value = iter(chunks) return response else: return mock.Mock( headers=headers, status_code=int(status_code), - spec=["status_code", "headers"], + spec=[ + u'status_code', + u'headers', + ], ) diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index bfad49c1..edda8fe1 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -128,7 +128,8 @@ def test__process_response(self): # Make sure **not finished** before. assert not download.finished - response = mock.Mock(status_code=int(http_client.OK), spec=["status_code"]) + response = mock.Mock( + status_code=int(http_client.OK), spec=[u'status_code']) ret_val = download._process_response(response) assert ret_val is None # Make sure **finished** after. @@ -141,8 +142,7 @@ def test__process_response_bad_status(self): # Make sure **not finished** before. assert not download.finished response = mock.Mock( - status_code=int(http_client.NOT_FOUND), spec=["status_code"] - ) + status_code=int(http_client.NOT_FOUND), spec=[u'status_code']) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -260,11 +260,8 @@ def _mock_response( ): response_headers = self._response_headers(start_byte, end_byte, total_bytes) return mock.Mock( - content=content, - headers=response_headers, - status_code=status_code, - spec=["content", "headers", "status_code"], - ) + content=content, headers=response_headers, status_code=status_code, + spec=[u'content', u'headers', u'status_code']) def test__prepare_request_already_finished(self): download = _download.ChunkedDownload(EXAMPLE_URL, 64, None) @@ -322,38 +319,7 @@ def test__make_invalid(self): assert download.invalid def test__process_response(self): - data = b"1234xyztL" * 37 - chunk_size = len(data) - stream = io.BytesIO() - download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) - _fix_up_virtual(download) - - already = 22 - download._bytes_downloaded = already - total_bytes = 4444 - - # Check internal state before. - assert not download.finished - assert download.bytes_downloaded == already - assert download.total_bytes is None - # Actually call the method to update. - response = self._mock_response( - already, - already + chunk_size - 1, - total_bytes, - content=data, - status_code=int(http_client.PARTIAL_CONTENT), - ) - download._process_response(response) - # Check internal state after. - assert not download.finished - assert download.bytes_downloaded == already + chunk_size - assert download.total_bytes == total_bytes - assert stream.getvalue() == data - - def test__process_response_transfer_encoding(self): - data = b"1234xyztL" * 37 - chunk_size = len(data) + chunk_size = 333 stream = io.BytesIO() download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) _fix_up_virtual(download) @@ -366,8 +332,8 @@ def test__process_response_transfer_encoding(self): assert not download.finished assert download.bytes_downloaded == already assert download.total_bytes is None - assert not download.invalid # Actually call the method to update. + data = b'1234xyztL' * 37 # 9 * 37 == 33 response = self._mock_response( already, already + chunk_size - 1, @@ -375,8 +341,6 @@ def test__process_response_transfer_encoding(self): content=data, status_code=int(http_client.PARTIAL_CONTENT), ) - response.headers[u"transfer-encoding"] = "chunked" - del response.headers[u"content-length"] download._process_response(response) # Check internal state after. assert not download.finished @@ -386,8 +350,9 @@ def test__process_response_transfer_encoding(self): def test__process_response_bad_status(self): chunk_size = 384 - stream = mock.Mock(spec=["write"]) - download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) + stream = mock.Mock(spec=[u'write']) + download = _download.ChunkedDownload( + EXAMPLE_URL, chunk_size, stream) _fix_up_virtual(download) total_bytes = 300 @@ -426,13 +391,9 @@ def test__process_response_missing_content_length(self): assert download.total_bytes is None assert not download.invalid # Actually call the method to update. - headers = {u"content-range": u"bytes 0-99/99"} response = mock.Mock( - headers=headers, - status_code=int(http_client.PARTIAL_CONTENT), - content=b"DEADBEEF", - spec=["headers", "status_code", "content"], - ) + headers={}, status_code=int(http_client.PARTIAL_CONTENT), + spec=[u'headers', u'status_code']) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -465,8 +426,7 @@ def test__process_response_bad_content_range(self): content=data, headers=headers, status_code=int(http_client.PARTIAL_CONTENT), - spec=["content", "headers", "status_code"], - ) + spec=[u'content', u'headers', u'status_code']) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -482,8 +442,9 @@ def test__process_response_bad_content_range(self): def test__process_response_body_wrong_length(self): chunk_size = 10 - stream = mock.Mock(spec=["write"]) - download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) + stream = mock.Mock(spec=[u'write']) + download = _download.ChunkedDownload( + EXAMPLE_URL, chunk_size, stream) _fix_up_virtual(download) total_bytes = 100 @@ -638,8 +599,8 @@ def test_start_as_offset(self): class Test_get_range_info(object): @staticmethod def _make_response(content_range): - headers = {u"content-range": content_range} - return mock.Mock(headers=headers, spec=["headers"]) + headers = {u'content-range': content_range} + return mock.Mock(headers=headers, spec=[u'headers']) def _success_helper(self, **kwargs): content_range = u"Bytes 7-11/42" @@ -679,7 +640,7 @@ def test_failure_with_callback(self): callback.assert_called_once_with() def _missing_header_helper(self, **kwargs): - response = mock.Mock(headers={}, spec=["headers"]) + response = mock.Mock(headers={}, spec=[u'headers']) with pytest.raises(common.InvalidResponse) as exc_info: _download.get_range_info(response, _get_headers, **kwargs) From 42cb9fff8e8afa3ebea15e343a13aafcb21840af Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 16 Sep 2019 16:17:47 -0700 Subject: [PATCH 2/4] missed merge --- tests/system/requests/test_download.py | 34 -------------------------- tests/unit/requests/test__helpers.py | 20 --------------- 2 files changed, 54 deletions(-) diff --git a/tests/system/requests/test_download.py b/tests/system/requests/test_download.py index cf99230b..9a7cb3b2 100644 --- a/tests/system/requests/test_download.py +++ b/tests/system/requests/test_download.py @@ -56,21 +56,6 @@ slice(-256, None, None), # obj[-256:] slice(262144, None, None), # obj[262144:] ), -<<<<<<< HEAD - }, - { - u"path": os.path.realpath(os.path.join(DATA_DIR, u"file.txt")), - u"content_type": PLAIN_TEXT, - u"checksum": u"XHSHAr/SpIeZtZbjgQ4nGw==", - u"slices": (), - }, - { - u"path": os.path.realpath(os.path.join(DATA_DIR, u"gzipped.txt.gz")), - u"content_type": PLAIN_TEXT, - u"checksum": u"KHRs/+ZSrc/FuuR4qz/PZQ==", - u"slices": (), - u"metadata": {u"contentEncoding": u"gzip"}, -======= }, { u'path': os.path.realpath(os.path.join(DATA_DIR, u'file.txt')), u'content_type': PLAIN_TEXT, @@ -86,7 +71,6 @@ u'metadata': { u'contentEncoding': u'gzip', }, ->>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) }, ) ENCRYPTED_ERR = b"The target object is encrypted by a customer-supplied encryption key." @@ -143,22 +127,13 @@ def _get_contents_for_upload(info): def _get_contents(info): -<<<<<<< HEAD - full_path = info[u"path"] - with open(full_path, u"rb") as file_obj: -======= full_path = info.get(u'uncompressed', info[u'path']) with open(full_path, u'rb') as file_obj: ->>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) return file_obj.read() def _get_blob_name(info): -<<<<<<< HEAD - full_path = info[u"path"] -======= full_path = info.get(u'uncompressed', info[u'path']) ->>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) return os.path.basename(full_path) @@ -205,15 +180,6 @@ def check_tombstoned(download, transport): assert exc_info.match(u"Download has finished.") -<<<<<<< HEAD -def read_raw_content(response): - return b"".join( - response.raw.stream(_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False) - ) - - -======= ->>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) def test_download_full(add_files, authorized_transport): for info in ALL_FILES: actual_contents = _get_contents(info) diff --git a/tests/unit/requests/test__helpers.py b/tests/unit/requests/test__helpers.py index 9029243c..0e7b37c2 100644 --- a/tests/unit/requests/test__helpers.py +++ b/tests/unit/requests/test__helpers.py @@ -27,25 +27,6 @@ def test__get_status_code(self): assert status_code == _helpers.RequestsMixin._get_status_code(response) def test__get_headers(self): -<<<<<<< HEAD - headers = {u"fruit": u"apple"} - response = mock.Mock(headers=headers, spec=["headers"]) - assert headers == _helpers.RequestsMixin._get_headers(response) - - def test__get_body_wo_content_consumed(self): - body = b"This is the payload." - raw = mock.Mock(spec=["stream"]) - raw.stream.return_value = iter([body]) - response = mock.Mock(raw=raw, _content=False, spec=["raw", "_content"]) - assert body == _helpers.RequestsMixin._get_body(response) - raw.stream.assert_called_once_with( - _helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False - ) - - def test__get_body_w_content_consumed(self): - body = b"This is the payload." - response = mock.Mock(_content=body, spec=["_content"]) -======= headers = {u'fruit': u'apple'} response = mock.Mock(headers=headers, spec=[u'headers']) assert headers == _helpers.RequestsMixin._get_headers(response) @@ -53,7 +34,6 @@ def test__get_body_w_content_consumed(self): def test__get_body(self): body = b'This is the payload.' response = mock.Mock(content=body, spec=[u'content']) ->>>>>>> parent of 2b9ffc8... Always use raw response data. (#87) assert body == _helpers.RequestsMixin._get_body(response) From 9c2a5ad9c4190d474789a3ba34393bec61fbdd25 Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 16 Sep 2019 16:23:48 -0700 Subject: [PATCH 3/4] keep test fix --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index b419f445..22d86cd3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,7 +24,7 @@ GOOGLE_AUTH = 'google-auth >= 0.10.0' -@nox.session(python=['2,7', '3.4', '3.5', '3.6', '3.7']) +@nox.session(python=['2.7', '3.4', '3.5', '3.6', '3.7']) def unit_tests(session): """Run the unit test suite.""" From 1ed6653e29851e03d84c77a34e565d4fa589d2cf Mon Sep 17 00:00:00 2001 From: Chris Wilcox Date: Mon, 16 Sep 2019 17:50:40 -0700 Subject: [PATCH 4/4] blacken and test fixes --- docs/latest/.buildinfo | 2 +- google/resumable_media/_download.py | 23 +++-- google/resumable_media/requests/download.py | 35 ++++--- tests/system/requests/test_download.py | 35 ++++--- tests/unit/requests/test__helpers.py | 8 +- tests/unit/requests/test_download.py | 100 +++++++++----------- tests/unit/test__download.py | 40 ++++---- 7 files changed, 126 insertions(+), 117 deletions(-) diff --git a/docs/latest/.buildinfo b/docs/latest/.buildinfo index 44e4d9da..38761a16 100644 --- a/docs/latest/.buildinfo +++ b/docs/latest/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 68666af324b279e5e7d7ada9d27ddd71 +config: b911398855f7668d2aa125e82024376d tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/google/resumable_media/_download.py b/google/resumable_media/_download.py index 9d194c47..6260d742 100644 --- a/google/resumable_media/_download.py +++ b/google/resumable_media/_download.py @@ -349,20 +349,29 @@ def _process_response(self, response): return _helpers.require_status_code( - response, _ACCEPTABLE_STATUS_CODES, - self._get_status_code, callback=self._make_invalid) + response, + _ACCEPTABLE_STATUS_CODES, + self._get_status_code, + callback=self._make_invalid, + ) content_length = _helpers.header_required( - response, u'content-length', self._get_headers, - callback=self._make_invalid) + response, u"content-length", self._get_headers, callback=self._make_invalid + ) num_bytes = int(content_length) _, end_byte, total_bytes = get_range_info( - response, self._get_headers, callback=self._make_invalid) + response, self._get_headers, callback=self._make_invalid + ) response_body = self._get_body(response) if len(response_body) != num_bytes: self._make_invalid() raise common.InvalidResponse( - response, u'Response is different size than content-length', - u'Expected', num_bytes, u'Received', len(response_body)) + response, + u"Response is different size than content-length", + u"Expected", + num_bytes, + u"Received", + len(response_body), + ) # First update ``bytes_downloaded``. self._bytes_downloaded += num_bytes diff --git a/google/resumable_media/requests/download.py b/google/resumable_media/requests/download.py index d63c58ba..1cb1313b 100644 --- a/google/resumable_media/requests/download.py +++ b/google/resumable_media/requests/download.py @@ -27,7 +27,7 @@ _LOGGER = logging.getLogger(__name__) _SINGLE_GET_CHUNK_SIZE = 8192 -_HASH_HEADER = u'x-goog-hash' +_HASH_HEADER = u"x-goog-hash" _MISSING_MD5 = u"""\ No MD5 checksum was returned from the service while downloading {} (which happens for composite objects), so client-side content integrity @@ -118,7 +118,8 @@ def _write_to_stream(self, response): # it with a ``_DoNothingHash``. local_hash = _add_decoder(response.raw, md5_hash) body_iter = response.iter_content( - chunk_size=_SINGLE_GET_CHUNK_SIZE, decode_unicode=False) + chunk_size=_SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) for chunk in body_iter: self._stream.write(chunk) local_hash.update(chunk) @@ -158,22 +159,21 @@ def consume(self, transport): method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. request_kwargs = { - u'data': payload, - u'headers': headers, - u'retry_strategy': self._retry_strategy, + u"data": payload, + u"headers": headers, + u"retry_strategy": self._retry_strategy, } if self._stream is not None: - request_kwargs[u'stream'] = True + request_kwargs[u"stream"] = True - result = _helpers.http_request( - transport, method, url, **request_kwargs) + result = _helpers.http_request(transport, method, url, **request_kwargs) - self._process_response(response) + self._process_response(result) if self._stream is not None: - self._write_to_stream(response) + self._write_to_stream(result) - return response + return result class ChunkedDownload(_helpers.RequestsMixin, _download.ChunkedDownload): @@ -220,8 +220,13 @@ def consume_next_chunk(self, transport): method, url, payload, headers = self._prepare_request() # NOTE: We assume "payload is None" but pass it along anyway. result = _helpers.http_request( - transport, method, url, data=payload, headers=headers, - retry_strategy=self._retry_strategy) + transport, + method, + url, + data=payload, + headers=headers, + retry_strategy=self._retry_strategy, + ) self._process_response(result) return result @@ -312,8 +317,8 @@ def _add_decoder(response_raw, md5_hash): if ``_decoder`` is not patched. Otherwise, returns a ``_DoNothingHash`` since the caller will no longer need to hash to decoded bytes. """ - encoding = response_raw.headers.get(u'content-encoding', u'').lower() - if encoding != u'gzip': + encoding = response_raw.headers.get(u"content-encoding", u"").lower() + if encoding != u"gzip": return md5_hash response_raw._decoder = _GzipDecoder(md5_hash) diff --git a/tests/system/requests/test_download.py b/tests/system/requests/test_download.py index 9a7cb3b2..df5d40f1 100644 --- a/tests/system/requests/test_download.py +++ b/tests/system/requests/test_download.py @@ -56,21 +56,20 @@ slice(-256, None, None), # obj[-256:] slice(262144, None, None), # obj[262144:] ), - }, { - u'path': os.path.realpath(os.path.join(DATA_DIR, u'file.txt')), - u'content_type': PLAIN_TEXT, - u'checksum': u'KHRs/+ZSrc/FuuR4qz/PZQ==', - u'slices': (), - }, { - u'path': os.path.realpath(os.path.join(DATA_DIR, u'gzipped.txt.gz')), - u'uncompressed': - os.path.realpath(os.path.join(DATA_DIR, u'gzipped.txt')), - u'content_type': PLAIN_TEXT, - u'checksum': u'KHRs/+ZSrc/FuuR4qz/PZQ==', - u'slices': (), - u'metadata': { - u'contentEncoding': u'gzip', - }, + }, + { + u"path": os.path.realpath(os.path.join(DATA_DIR, u"file.txt")), + u"content_type": PLAIN_TEXT, + u"checksum": u"KHRs/+ZSrc/FuuR4qz/PZQ==", + u"slices": (), + }, + { + u"path": os.path.realpath(os.path.join(DATA_DIR, u"gzipped.txt.gz")), + u"uncompressed": os.path.realpath(os.path.join(DATA_DIR, u"gzipped.txt")), + u"content_type": PLAIN_TEXT, + u"checksum": u"KHRs/+ZSrc/FuuR4qz/PZQ==", + u"slices": (), + u"metadata": {u"contentEncoding": u"gzip"}, }, ) ENCRYPTED_ERR = b"The target object is encrypted by a customer-supplied encryption key." @@ -127,13 +126,13 @@ def _get_contents_for_upload(info): def _get_contents(info): - full_path = info.get(u'uncompressed', info[u'path']) - with open(full_path, u'rb') as file_obj: + full_path = info.get(u"uncompressed", info[u"path"]) + with open(full_path, u"rb") as file_obj: return file_obj.read() def _get_blob_name(info): - full_path = info.get(u'uncompressed', info[u'path']) + full_path = info.get(u"uncompressed", info[u"path"]) return os.path.basename(full_path) diff --git a/tests/unit/requests/test__helpers.py b/tests/unit/requests/test__helpers.py index 0e7b37c2..e5ec9746 100644 --- a/tests/unit/requests/test__helpers.py +++ b/tests/unit/requests/test__helpers.py @@ -27,13 +27,13 @@ def test__get_status_code(self): assert status_code == _helpers.RequestsMixin._get_status_code(response) def test__get_headers(self): - headers = {u'fruit': u'apple'} - response = mock.Mock(headers=headers, spec=[u'headers']) + headers = {u"fruit": u"apple"} + response = mock.Mock(headers=headers, spec=[u"headers"]) assert headers == _helpers.RequestsMixin._get_headers(response) def test__get_body(self): - body = b'This is the payload.' - response = mock.Mock(content=body, spec=[u'content']) + body = b"This is the payload." + response = mock.Mock(content=body, spec=[u"content"]) assert body == _helpers.RequestsMixin._get_body(response) diff --git a/tests/unit/requests/test_download.py b/tests/unit/requests/test_download.py index 3e724a73..c86f60ad 100644 --- a/tests/unit/requests/test_download.py +++ b/tests/unit/requests/test_download.py @@ -30,8 +30,7 @@ class TestDownload(object): - - @mock.patch(u'google.resumable_media.requests.download._LOGGER') + @mock.patch(u"google.resumable_media.requests.download._LOGGER") def test__get_expected_md5_present(self, _LOGGER): download = download_mod.Download(EXAMPLE_URL) @@ -44,7 +43,7 @@ def test__get_expected_md5_present(self, _LOGGER): assert expected_md5_hash == checksum _LOGGER.info.assert_not_called() - @mock.patch(u'google.resumable_media.requests.download._LOGGER') + @mock.patch(u"google.resumable_media.requests.download._LOGGER") def test__get_expected_md5_missing(self, _LOGGER): download = download_mod.Download(EXAMPLE_URL) @@ -73,8 +72,8 @@ def test__write_to_stream_no_hash_check(self): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.iter_content.assert_called_once_with( - chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, - decode_unicode=False) + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) def test__write_to_stream_with_hash_check_success(self): stream = io.BytesIO() @@ -96,8 +95,8 @@ def test__write_to_stream_with_hash_check_success(self): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.iter_content.assert_called_once_with( - chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, - decode_unicode=False) + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) def test__write_to_stream_with_hash_check_fail(self): stream = io.BytesIO() @@ -129,15 +128,16 @@ def test__write_to_stream_with_hash_check_fail(self): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.iter_content.assert_called_once_with( - chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, - decode_unicode=False) + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) def _consume_helper( self, stream=None, end=65536, headers=None, chunks=(), response_headers=None ): download = download_mod.Download( - EXAMPLE_URL, stream=stream, end=end, headers=headers) - transport = mock.Mock(spec=[u'request']) + EXAMPLE_URL, stream=stream, end=end, headers=headers + ) + transport = mock.Mock(spec=[u"request"]) transport.request.return_value = _mock_response( chunks=chunks, headers=response_headers ) @@ -146,12 +146,13 @@ def _consume_helper( ret_val = download.consume(transport) assert ret_val is transport.request.return_value - called_kwargs = {u'data': None, u'headers': download._headers} + called_kwargs = {u"data": None, u"headers": download._headers} if chunks: assert stream is not None - called_kwargs[u'stream'] = True + called_kwargs[u"stream"] = True transport.request.assert_called_once_with( - u'GET', EXAMPLE_URL, timeout=EXPECTED_TIMEOUT, **called_kwargs) + u"GET", EXAMPLE_URL, timeout=EXPECTED_TIMEOUT, **called_kwargs + ) range_bytes = u"bytes={:d}-{:d}".format(0, end) assert download._headers[u"range"] == range_bytes @@ -174,8 +175,8 @@ def test_consume_with_stream(self): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.iter_content.assert_called_once_with( - chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, - decode_unicode=False) + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) def test_consume_with_stream_hash_check_success(self): stream = io.BytesIO() @@ -193,8 +194,8 @@ def test_consume_with_stream_hash_check_success(self): response.__enter__.assert_called_once_with() response.__exit__.assert_called_once_with(None, None, None) response.iter_content.assert_called_once_with( - chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, - decode_unicode=False) + chunk_size=download_mod._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) def test_consume_with_stream_hash_check_fail(self): stream = io.BytesIO() @@ -204,9 +205,8 @@ def test_consume_with_stream_hash_check_fail(self): bad_checksum = u"anVzdCBub3QgdGhpcyAxLA==" header_value = u"crc32c=V0FUPw==,md5={}".format(bad_checksum) headers = {download_mod._HASH_HEADER: header_value} - transport = mock.Mock(spec=[u'request']) - transport.request.return_value = _mock_response( - chunks=chunks, headers=headers) + transport = mock.Mock(spec=[u"request"]) + transport.request.return_value = _mock_response(chunks=chunks, headers=headers) assert not download.finished with pytest.raises(common.DataCorruption) as exc_info: @@ -265,11 +265,7 @@ def _mock_response( content=content, headers=response_headers, status_code=status_code, - spec=[ - u'content', - u'headers', - u'status_code', - ], + spec=[u"content", u"headers", u"status_code"], ) def test_consume_next_chunk_already_finished(self): @@ -278,8 +274,8 @@ def test_consume_next_chunk_already_finished(self): with pytest.raises(ValueError): download.consume_next_chunk(None) - def _mock_transport(self, start, chunk_size, total_bytes, content=b''): - transport = mock.Mock(spec=[u'request']) + def _mock_transport(self, start, chunk_size, total_bytes, content=b""): + transport = mock.Mock(spec=[u"request"]) assert len(content) == chunk_size transport.request.return_value = self._mock_response( start, @@ -312,8 +308,12 @@ def test_consume_next_chunk(self): range_bytes = u"bytes={:d}-{:d}".format(start, start + chunk_size - 1) download_headers = {u"range": range_bytes} transport.request.assert_called_once_with( - u'GET', EXAMPLE_URL, data=None, headers=download_headers, - timeout=EXPECTED_TIMEOUT) + u"GET", + EXAMPLE_URL, + data=None, + headers=download_headers, + timeout=EXPECTED_TIMEOUT, + ) assert stream.getvalue() == data # Go back and check the internal state after consuming the chunk. assert not download.finished @@ -374,20 +374,16 @@ def test__DoNothingHash(): class Test__add_decoder(object): - def test_non_gzipped(self): - response_raw = mock.Mock(headers={}, spec=[u'headers']) - md5_hash = download_mod._add_decoder( - response_raw, mock.sentinel.md5_hash) + response_raw = mock.Mock(headers={}, spec=[u"headers"]) + md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash) assert md5_hash is mock.sentinel.md5_hash def test_gzipped(self): - headers = {u'content-encoding': u'gzip'} - response_raw = mock.Mock( - headers=headers, spec=[u'headers', u'_decoder']) - md5_hash = download_mod._add_decoder( - response_raw, mock.sentinel.md5_hash) + headers = {u"content-encoding": u"gzip"} + response_raw = mock.Mock(headers=headers, spec=[u"headers", u"_decoder"]) + md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash) assert md5_hash is not mock.sentinel.md5_hash assert isinstance(md5_hash, download_mod._DoNothingHash) @@ -396,19 +392,18 @@ def test_gzipped(self): class Test_GzipDecoder(object): - def test_constructor(self): decoder = download_mod._GzipDecoder(mock.sentinel.md5_hash) assert decoder._md5_hash is mock.sentinel.md5_hash def test_decompress(self): - md5_hash = mock.Mock(spec=['update']) + md5_hash = mock.Mock(spec=["update"]) decoder = download_mod._GzipDecoder(md5_hash) - data = b'\x1f\x8b\x08\x08' + data = b"\x1f\x8b\x08\x08" result = decoder.decompress(data) - assert result == b'' + assert result == b"" md5_hash.update.assert_called_once_with(data) @@ -417,18 +412,18 @@ def _mock_response(status_code=http_client.OK, chunks=(), headers=None): headers = {} if chunks: - mock_raw = mock.Mock(headers=headers, spec=[u'headers']) + mock_raw = mock.Mock(headers=headers, spec=[u"headers"]) response = mock.MagicMock( headers=headers, status_code=int(status_code), raw=mock_raw, spec=[ - u'__enter__', - u'__exit__', - u'iter_content', - u'status_code', - u'headers', - u'raw', + u"__enter__", + u"__exit__", + u"iter_content", + u"status_code", + u"headers", + u"raw", ], ) # i.e. context manager returns ``self``. @@ -440,8 +435,5 @@ def _mock_response(status_code=http_client.OK, chunks=(), headers=None): return mock.Mock( headers=headers, status_code=int(status_code), - spec=[ - u'status_code', - u'headers', - ], + spec=[u"status_code", u"headers"], ) diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index edda8fe1..a984b876 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -128,8 +128,7 @@ def test__process_response(self): # Make sure **not finished** before. assert not download.finished - response = mock.Mock( - status_code=int(http_client.OK), spec=[u'status_code']) + response = mock.Mock(status_code=int(http_client.OK), spec=[u"status_code"]) ret_val = download._process_response(response) assert ret_val is None # Make sure **finished** after. @@ -142,7 +141,8 @@ def test__process_response_bad_status(self): # Make sure **not finished** before. assert not download.finished response = mock.Mock( - status_code=int(http_client.NOT_FOUND), spec=[u'status_code']) + status_code=int(http_client.NOT_FOUND), spec=[u"status_code"] + ) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -260,8 +260,11 @@ def _mock_response( ): response_headers = self._response_headers(start_byte, end_byte, total_bytes) return mock.Mock( - content=content, headers=response_headers, status_code=status_code, - spec=[u'content', u'headers', u'status_code']) + content=content, + headers=response_headers, + status_code=status_code, + spec=[u"content", u"headers", u"status_code"], + ) def test__prepare_request_already_finished(self): download = _download.ChunkedDownload(EXAMPLE_URL, 64, None) @@ -333,7 +336,7 @@ def test__process_response(self): assert download.bytes_downloaded == already assert download.total_bytes is None # Actually call the method to update. - data = b'1234xyztL' * 37 # 9 * 37 == 33 + data = b"1234xyztL" * 37 # 9 * 37 == 33 response = self._mock_response( already, already + chunk_size - 1, @@ -350,9 +353,8 @@ def test__process_response(self): def test__process_response_bad_status(self): chunk_size = 384 - stream = mock.Mock(spec=[u'write']) - download = _download.ChunkedDownload( - EXAMPLE_URL, chunk_size, stream) + stream = mock.Mock(spec=[u"write"]) + download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) _fix_up_virtual(download) total_bytes = 300 @@ -392,8 +394,10 @@ def test__process_response_missing_content_length(self): assert not download.invalid # Actually call the method to update. response = mock.Mock( - headers={}, status_code=int(http_client.PARTIAL_CONTENT), - spec=[u'headers', u'status_code']) + headers={}, + status_code=int(http_client.PARTIAL_CONTENT), + spec=[u"headers", u"status_code"], + ) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -426,7 +430,8 @@ def test__process_response_bad_content_range(self): content=data, headers=headers, status_code=int(http_client.PARTIAL_CONTENT), - spec=[u'content', u'headers', u'status_code']) + spec=[u"content", u"headers", u"status_code"], + ) with pytest.raises(common.InvalidResponse) as exc_info: download._process_response(response) @@ -442,9 +447,8 @@ def test__process_response_bad_content_range(self): def test__process_response_body_wrong_length(self): chunk_size = 10 - stream = mock.Mock(spec=[u'write']) - download = _download.ChunkedDownload( - EXAMPLE_URL, chunk_size, stream) + stream = mock.Mock(spec=[u"write"]) + download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream) _fix_up_virtual(download) total_bytes = 100 @@ -599,8 +603,8 @@ def test_start_as_offset(self): class Test_get_range_info(object): @staticmethod def _make_response(content_range): - headers = {u'content-range': content_range} - return mock.Mock(headers=headers, spec=[u'headers']) + headers = {u"content-range": content_range} + return mock.Mock(headers=headers, spec=[u"headers"]) def _success_helper(self, **kwargs): content_range = u"Bytes 7-11/42" @@ -640,7 +644,7 @@ def test_failure_with_callback(self): callback.assert_called_once_with() def _missing_header_helper(self, **kwargs): - response = mock.Mock(headers={}, spec=[u'headers']) + response = mock.Mock(headers={}, spec=[u"headers"]) with pytest.raises(common.InvalidResponse) as exc_info: _download.get_range_info(response, _get_headers, **kwargs)