From 9824024b829d8e18425b5f46ccb6c259702f9e53 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Mon, 24 Jan 2022 17:10:23 -0700 Subject: [PATCH 1/7] (WIP) fix: add user-agent on requests --- google/resumable_media/_download.py | 3 +++ google/resumable_media/_upload.py | 3 +++ tests/unit/test__download.py | 22 +++++++++++++--------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/google/resumable_media/_download.py b/google/resumable_media/_download.py index b2bac98c..579388b6 100644 --- a/google/resumable_media/_download.py +++ b/google/resumable_media/_download.py @@ -16,6 +16,7 @@ import http.client +import pkg_resources import re from google.resumable_media import _helpers @@ -58,6 +59,8 @@ def __init__(self, media_url, stream=None, start=None, end=None, headers=None): self.end = end if headers is None: headers = {} + version = pkg_resources.get_distribution('google-resumable-media').version + headers['User-Agent'] = "google-resumable-media-python/{}".format(version) self._headers = headers self._finished = False self._retry_strategy = common.RetryStrategy() diff --git a/google/resumable_media/_upload.py b/google/resumable_media/_upload.py index dc22cd1c..44033da4 100644 --- a/google/resumable_media/_upload.py +++ b/google/resumable_media/_upload.py @@ -24,6 +24,7 @@ import http.client import json import os +import pkg_resources import random import re import sys @@ -83,6 +84,8 @@ def __init__(self, upload_url, headers=None): self.upload_url = upload_url if headers is None: headers = {} + version = pkg_resources.get_distribution('google-resumable-media').version + headers['User-Agent'] = "google-resumable-media-python/{}".format(version) self._headers = headers self._finished = False self._retry_strategy = common.RetryStrategy() diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index 46026b16..7ea379d1 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -16,6 +16,7 @@ import io import mock +import pkg_resources import pytest # type: ignore from google.resumable_media import _download @@ -26,6 +27,9 @@ "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/{OBJECT}?alt=media" ) +def _headers_with_agent(headers): + version = pkg_resources.get_distribution('google-resumable-media').version + return { **headers, 'User-Agent': "google-resumable-media-python/{}".format(version)} class TestDownloadBase(object): def test_constructor_defaults(self): @@ -34,7 +38,7 @@ def test_constructor_defaults(self): assert download._stream is None assert download.start is None assert download.end is None - assert download._headers == {} + assert download._headers == _headers_with_agent({}) assert not download._finished _check_retry_strategy(download) @@ -53,7 +57,7 @@ def test_constructor_explicit(self): assert download._stream is mock.sentinel.stream assert download.start == start assert download.end == end - assert download._headers is headers + assert download._headers is _headers_with_agent(headers) assert not download._finished _check_retry_strategy(download) @@ -102,14 +106,14 @@ def test__prepare_request(self): assert method1 == "GET" assert url1 == EXAMPLE_URL assert payload1 is None - assert headers1 == {} + assert headers1 == _headers_with_agent({}) download2 = _download.Download(EXAMPLE_URL, start=53) method2, url2, payload2, headers2 = download2._prepare_request() assert method2 == "GET" assert url2 == EXAMPLE_URL assert payload2 is None - assert headers2 == {"range": "bytes=53-"} + assert headers2 == _headers_with_agent({"range": "bytes=53-"}) def test__prepare_request_with_headers(self): headers = {"spoonge": "borb"} @@ -118,7 +122,7 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers is headers + assert new_headers is _headers_with_agent(headers) assert headers == {"range": "bytes=11-111", "spoonge": "borb"} def test__process_response(self): @@ -171,7 +175,7 @@ def test_constructor_defaults(self): assert download.chunk_size == chunk_size assert download.start == 0 assert download.end is None - assert download._headers == {} + assert download._headers == _headers_with_agent({}) assert not download._finished _check_retry_strategy(download) assert download._stream is stream @@ -288,7 +292,7 @@ def test__prepare_request(self): assert method1 == "GET" assert url1 == EXAMPLE_URL assert payload1 is None - assert headers1 == {"range": "bytes=0-2047"} + assert headers1 == _headers_with_agent({"range": "bytes=0-2047"}) download2 = _download.ChunkedDownload( EXAMPLE_URL, chunk_size, None, start=19991 @@ -298,7 +302,7 @@ def test__prepare_request(self): assert method2 == "GET" assert url2 == EXAMPLE_URL assert payload2 is None - assert headers2 == {"range": "bytes=19991-20100"} + assert headers2 == _headers_with_agent({"range": "bytes=19991-20100"}) def test__prepare_request_with_headers(self): chunk_size = 2048 @@ -310,7 +314,7 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers is headers + assert new_headers is _headers_with_agent(headers) expected = {"patrizio": "Starf-ish", "range": "bytes=0-2047"} assert headers == expected From 2450ddc7a3ffb03a484c2b73c589847a4bc78354 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 25 Jan 2022 11:10:29 -0700 Subject: [PATCH 2/7] adjusted tests for download and upload --- tests/unit/test__download.py | 10 +++---- tests/unit/test__upload.py | 52 ++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index 7ea379d1..2111338d 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -57,7 +57,7 @@ def test_constructor_explicit(self): assert download._stream is mock.sentinel.stream assert download.start == start assert download.end == end - assert download._headers is _headers_with_agent(headers) + assert download._headers == _headers_with_agent(headers) assert not download._finished _check_retry_strategy(download) @@ -122,8 +122,8 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers is _headers_with_agent(headers) - assert headers == {"range": "bytes=11-111", "spoonge": "borb"} + assert new_headers == _headers_with_agent(headers) + assert headers == _headers_with_agent({"range": "bytes=11-111", "spoonge": "borb"}) def test__process_response(self): download = _download.Download(EXAMPLE_URL) @@ -314,9 +314,9 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers is _headers_with_agent(headers) + assert new_headers == _headers_with_agent(headers) expected = {"patrizio": "Starf-ish", "range": "bytes=0-2047"} - assert headers == expected + assert headers == _headers_with_agent(expected) def test__make_invalid(self): download = _download.ChunkedDownload(EXAMPLE_URL, 512, None) diff --git a/tests/unit/test__upload.py b/tests/unit/test__upload.py index 4e948f0d..8aaf5317 100644 --- a/tests/unit/test__upload.py +++ b/tests/unit/test__upload.py @@ -17,6 +17,7 @@ import sys import mock +import pkg_resources import pytest # type: ignore from google.resumable_media import _helpers @@ -34,11 +35,16 @@ JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n" +def _headers_with_agent(headers): + version = pkg_resources.get_distribution('google-resumable-media').version + return { **headers, 'User-Agent': "google-resumable-media-python/{}".format(version)} + + class TestUploadBase(object): def test_constructor_defaults(self): upload = _upload.UploadBase(SIMPLE_URL) assert upload.upload_url == SIMPLE_URL - assert upload._headers == {} + assert upload._headers == _headers_with_agent({}) assert not upload._finished _check_retry_strategy(upload) @@ -46,7 +52,7 @@ def test_constructor_explicit(self): headers = {"spin": "doctors"} upload = _upload.UploadBase(SIMPLE_URL, headers=headers) assert upload.upload_url == SIMPLE_URL - assert upload._headers is headers + assert upload._headers == _headers_with_agent(headers) assert not upload._finished _check_retry_strategy(upload) @@ -139,7 +145,7 @@ def test__prepare_request(self): assert method == "POST" assert url == SIMPLE_URL assert payload == data - assert headers == {"content-type": content_type} + assert headers == _headers_with_agent({"content-type": content_type}) def test__prepare_request_with_headers(self): headers = {"x-goog-cheetos": "spicy"} @@ -151,9 +157,9 @@ def test__prepare_request_with_headers(self): assert method == "POST" assert url == SIMPLE_URL assert payload == data - assert new_headers is headers + assert new_headers == _headers_with_agent(headers) expected = {"content-type": content_type, "x-goog-cheetos": "spicy"} - assert headers == expected + assert headers == _headers_with_agent(expected) def test_transmit(self): upload = _upload.SimpleUpload(SIMPLE_URL) @@ -167,7 +173,7 @@ class TestMultipartUpload(object): def test_constructor_defaults(self): upload = _upload.MultipartUpload(MULTIPART_URL) assert upload.upload_url == MULTIPART_URL - assert upload._headers == {} + assert upload._headers == _headers_with_agent({}) assert upload._checksum_type is None assert not upload._finished _check_retry_strategy(upload) @@ -176,7 +182,7 @@ def test_constructor_explicit(self): headers = {"spin": "doctors"} upload = _upload.MultipartUpload(MULTIPART_URL, headers=headers, checksum="md5") assert upload.upload_url == MULTIPART_URL - assert upload._headers is headers + assert upload._headers == _headers_with_agent(headers) assert upload._checksum_type == "md5" assert not upload._finished _check_retry_strategy(upload) @@ -255,17 +261,17 @@ def _prepare_request_helper( def test__prepare_request(self): headers, multipart_type = self._prepare_request_helper() - assert headers == {"content-type": multipart_type} + assert headers == _headers_with_agent({"content-type": multipart_type}) def test__prepare_request_with_headers(self): headers = {"best": "shirt", "worst": "hat"} new_headers, multipart_type = self._prepare_request_helper(headers=headers) - assert new_headers is headers - expected_headers = { + assert new_headers == _headers_with_agent(headers) + expected_headers = _headers_with_agent({ "best": "shirt", "content-type": multipart_type, "worst": "hat", - } + }) assert expected_headers == headers @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) @@ -277,9 +283,9 @@ def test__prepare_request_with_checksum(self, checksum): headers, multipart_type = self._prepare_request_helper( checksum=checksum, expected_checksum=checksums[checksum] ) - assert headers == { + assert headers == _headers_with_agent({ "content-type": multipart_type, - } + }) @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) def test__prepare_request_with_checksum_overwrite(self, checksum): @@ -292,9 +298,9 @@ def test__prepare_request_with_checksum_overwrite(self, checksum): expected_checksum=checksums[checksum], test_overwrite=True, ) - assert headers == { + assert headers == _headers_with_agent({ "content-type": multipart_type, - } + }) def test_transmit(self): upload = _upload.MultipartUpload(MULTIPART_URL) @@ -309,7 +315,7 @@ def test_constructor(self): chunk_size = ONE_MB upload = _upload.ResumableUpload(RESUMABLE_URL, chunk_size) assert upload.upload_url == RESUMABLE_URL - assert upload._headers == {} + assert upload._headers == _headers_with_agent({}) assert not upload._finished _check_retry_strategy(upload) assert upload._chunk_size == chunk_size @@ -435,7 +441,7 @@ def test__prepare_initiate_request(self): "x-upload-content-length": "{:d}".format(len(data)), "x-upload-content-type": BASIC_CONTENT, } - assert headers == expected_headers + assert headers == _headers_with_agent(expected_headers) def test_prepare_initiate_request_with_signed_url(self): signed_urls = [ @@ -450,7 +456,7 @@ def test_prepare_initiate_request_with_signed_url(self): "content-type": BASIC_CONTENT, "x-upload-content-length": "{:d}".format(len(data)), } - assert headers == expected_headers + assert headers == _headers_with_agent(expected_headers) def test__prepare_initiate_request_with_headers(self): headers = {"caviar": "beluga", "top": "quark"} @@ -464,7 +470,7 @@ def test__prepare_initiate_request_with_headers(self): "x-upload-content-length": "{:d}".format(len(data)), "x-upload-content-type": BASIC_CONTENT, } - assert new_headers == expected_headers + assert new_headers == _headers_with_agent(expected_headers) def test__prepare_initiate_request_known_size(self): total_bytes = 25 @@ -475,7 +481,7 @@ def test__prepare_initiate_request_known_size(self): "x-upload-content-length": "{:d}".format(total_bytes), "x-upload-content-type": BASIC_CONTENT, } - assert headers == expected_headers + assert headers == _headers_with_agent(expected_headers) def test__prepare_initiate_request_unknown_size(self): _, headers = self._prepare_initiate_request_helper(stream_final=False) @@ -483,7 +489,7 @@ def test__prepare_initiate_request_unknown_size(self): "content-type": "application/json; charset=UTF-8", "x-upload-content-type": BASIC_CONTENT, } - assert headers == expected_headers + assert headers == _headers_with_agent(expected_headers) def test__prepare_initiate_request_already_initiated(self): upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB) @@ -960,7 +966,7 @@ def test__prepare_recover_request(self): assert payload is None assert headers == {"content-range": "bytes */*"} # Make sure headers are untouched. - assert upload._headers == {} + assert upload._headers == _headers_with_agent({}) def test__prepare_recover_request_with_headers(self): headers = {"lake": "ocean"} @@ -975,7 +981,7 @@ def test__prepare_recover_request_with_headers(self): # Make sure the ``_headers`` are not incorporated. assert "lake" not in new_headers # Make sure headers are untouched. - assert upload._headers == {"lake": "ocean"} + assert upload._headers == _headers_with_agent({"lake": "ocean"}) def test__process_recover_response_bad_status(self): upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB) From 141faa3b96e67b5eb2c6201500b478e89652234c Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 25 Jan 2022 12:27:25 -0700 Subject: [PATCH 3/7] clean it up, test it right --- google/resumable_media/_download.py | 4 +-- google/resumable_media/_helpers.py | 7 +++++ google/resumable_media/_upload.py | 4 +-- tests/unit/requests/test_download.py | 22 ++++++------- tests/unit/requests/test_upload.py | 13 ++++---- tests/unit/test__download.py | 25 ++++++--------- tests/unit/test__upload.py | 47 ++++++++++++---------------- 7 files changed, 57 insertions(+), 65 deletions(-) diff --git a/google/resumable_media/_download.py b/google/resumable_media/_download.py index 579388b6..3c2625a3 100644 --- a/google/resumable_media/_download.py +++ b/google/resumable_media/_download.py @@ -59,9 +59,7 @@ def __init__(self, media_url, stream=None, start=None, end=None, headers=None): self.end = end if headers is None: headers = {} - version = pkg_resources.get_distribution('google-resumable-media').version - headers['User-Agent'] = "google-resumable-media-python/{}".format(version) - self._headers = headers + self._headers = _helpers._base_headers(headers) self._finished = False self._retry_strategy = common.RetryStrategy() diff --git a/google/resumable_media/_helpers.py b/google/resumable_media/_helpers.py index dc6fdaf1..fafc1eff 100644 --- a/google/resumable_media/_helpers.py +++ b/google/resumable_media/_helpers.py @@ -20,6 +20,7 @@ import hashlib import logging import random +import pkg_resources import warnings from google.resumable_media import common @@ -45,6 +46,12 @@ def do_nothing(): """Simple default callback.""" +def _base_headers(headers): + version = pkg_resources.get_distribution('google-resumable-media').version + headers['User-Agent'] = "google-resumable-media-python/{}".format(version) + return headers + + def header_required(response, name, get_headers, callback=do_nothing): """Checks that a specific header is in a headers dictionary. diff --git a/google/resumable_media/_upload.py b/google/resumable_media/_upload.py index 44033da4..3e42828e 100644 --- a/google/resumable_media/_upload.py +++ b/google/resumable_media/_upload.py @@ -84,9 +84,7 @@ def __init__(self, upload_url, headers=None): self.upload_url = upload_url if headers is None: headers = {} - version = pkg_resources.get_distribution('google-resumable-media').version - headers['User-Agent'] = "google-resumable-media-python/{}".format(version) - self._headers = headers + self._headers = _helpers._base_headers(headers) self._finished = False self._retry_strategy = common.RetryStrategy() diff --git a/tests/unit/requests/test_download.py b/tests/unit/requests/test_download.py index df12ca44..2c82effc 100644 --- a/tests/unit/requests/test_download.py +++ b/tests/unit/requests/test_download.py @@ -22,7 +22,7 @@ from google.resumable_media import _helpers from google.resumable_media.requests import download as download_mod from google.resumable_media.requests import _request_helpers - +from google.resumable_media._helpers import _base_headers URL_PREFIX = "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/" EXAMPLE_URL = URL_PREFIX + "{OBJECT}?alt=media" @@ -241,7 +241,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): assert stream.getvalue() == b"".join(chunks) assert download.finished - assert download._headers == {} + assert download._headers == _base_headers({}) error = exc_info.value assert error.response is transport.request.return_value @@ -260,7 +260,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): "GET", EXAMPLE_URL, data=None, - headers={}, + headers=_base_headers({}), stream=True, timeout=EXPECTED_TIMEOUT, ) @@ -271,7 +271,7 @@ def test_consume_with_headers(self): self._consume_helper(end=end, headers=headers) range_bytes = "bytes={:d}-{:d}".format(0, end) # Make sure the headers have been modified. - assert headers == {"range": range_bytes} + assert headers == _base_headers({"range": range_bytes}) class TestRawDownload(object): @@ -494,7 +494,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): assert stream.getvalue() == b"".join(chunks) assert download.finished - assert download._headers == {} + assert download._headers == _base_headers({}) error = exc_info.value assert error.response is transport.request.return_value @@ -513,7 +513,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): "GET", EXAMPLE_URL, data=None, - headers={}, + headers=_base_headers({}), stream=True, timeout=EXPECTED_TIMEOUT, ) @@ -524,7 +524,7 @@ def test_consume_with_headers(self): self._consume_helper(end=end, headers=headers) range_bytes = "bytes={:d}-{:d}".format(0, end) # Make sure the headers have been modified. - assert headers == {"range": range_bytes} + assert headers == _base_headers({"range": range_bytes}) class TestChunkedDownload(object): @@ -589,7 +589,7 @@ def test_consume_next_chunk(self): ret_val = download.consume_next_chunk(transport) assert ret_val is transport.request.return_value range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1) - download_headers = {"range": range_bytes} + download_headers = _base_headers({"range": range_bytes}) transport.request.assert_called_once_with( "GET", EXAMPLE_URL, @@ -618,7 +618,7 @@ def test_consume_next_chunk_with_custom_timeout(self): download.consume_next_chunk(transport, timeout=14.7) range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1) - download_headers = {"range": range_bytes} + download_headers = _base_headers({"range": range_bytes}) transport.request.assert_called_once_with( "GET", EXAMPLE_URL, @@ -695,7 +695,7 @@ def test_consume_next_chunk(self): "GET", EXAMPLE_URL, data=None, - headers=download_headers, + headers=_base_headers(download_headers), stream=True, timeout=EXPECTED_TIMEOUT, ) @@ -725,7 +725,7 @@ def test_consume_next_chunk_with_custom_timeout(self): "GET", EXAMPLE_URL, data=None, - headers=download_headers, + headers=_base_headers(download_headers), stream=True, timeout=14.7, ) diff --git a/tests/unit/requests/test_upload.py b/tests/unit/requests/test_upload.py index 3694c750..72caaf12 100644 --- a/tests/unit/requests/test_upload.py +++ b/tests/unit/requests/test_upload.py @@ -19,6 +19,7 @@ import mock import google.resumable_media.requests.upload as upload_mod +from google.resumable_media._helpers import _base_headers URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o" @@ -48,7 +49,7 @@ def test_transmit(self): "POST", SIMPLE_URL, data=data, - headers=upload_headers, + headers=_base_headers(upload_headers), timeout=EXPECTED_TIMEOUT, ) assert upload.finished @@ -67,7 +68,7 @@ def test_transmit_w_custom_timeout(self): "POST", SIMPLE_URL, data=data, - headers=expected_headers, + headers=_base_headers(expected_headers), timeout=12.6, ) @@ -103,7 +104,7 @@ def test_transmit(self, mock_get_boundary): "POST", MULTIPART_URL, data=expected_payload, - headers=upload_headers, + headers=_base_headers(upload_headers), timeout=EXPECTED_TIMEOUT, ) assert upload.finished @@ -141,7 +142,7 @@ def test_transmit_w_custom_timeout(self, mock_get_boundary): "POST", MULTIPART_URL, data=expected_payload, - headers=upload_headers, + headers=_base_headers(upload_headers), timeout=12.6, ) assert upload.finished @@ -187,7 +188,7 @@ def test_initiate(self): "POST", RESUMABLE_URL, data=json_bytes, - headers=expected_headers, + headers=_base_headers(expected_headers), timeout=EXPECTED_TIMEOUT, ) @@ -223,7 +224,7 @@ def test_initiate_w_custom_timeout(self): "POST", RESUMABLE_URL, data=json_bytes, - headers=expected_headers, + headers=_base_headers(expected_headers), timeout=12.6, ) diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index 2111338d..7d40ba9e 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -21,15 +21,12 @@ from google.resumable_media import _download from google.resumable_media import common - +from google.resumable_media._helpers import _base_headers EXAMPLE_URL = ( "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/{OBJECT}?alt=media" ) -def _headers_with_agent(headers): - version = pkg_resources.get_distribution('google-resumable-media').version - return { **headers, 'User-Agent': "google-resumable-media-python/{}".format(version)} class TestDownloadBase(object): def test_constructor_defaults(self): @@ -38,7 +35,7 @@ def test_constructor_defaults(self): assert download._stream is None assert download.start is None assert download.end is None - assert download._headers == _headers_with_agent({}) + assert download._headers == _base_headers({}) assert not download._finished _check_retry_strategy(download) @@ -57,7 +54,7 @@ def test_constructor_explicit(self): assert download._stream is mock.sentinel.stream assert download.start == start assert download.end == end - assert download._headers == _headers_with_agent(headers) + assert download._headers == _base_headers(headers) assert not download._finished _check_retry_strategy(download) @@ -106,14 +103,14 @@ def test__prepare_request(self): assert method1 == "GET" assert url1 == EXAMPLE_URL assert payload1 is None - assert headers1 == _headers_with_agent({}) + assert headers1 == _base_headers({}) download2 = _download.Download(EXAMPLE_URL, start=53) method2, url2, payload2, headers2 = download2._prepare_request() assert method2 == "GET" assert url2 == EXAMPLE_URL assert payload2 is None - assert headers2 == _headers_with_agent({"range": "bytes=53-"}) + assert headers2 == _base_headers({"range": "bytes=53-"}) def test__prepare_request_with_headers(self): headers = {"spoonge": "borb"} @@ -122,8 +119,7 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers == _headers_with_agent(headers) - assert headers == _headers_with_agent({"range": "bytes=11-111", "spoonge": "borb"}) + assert new_headers == _base_headers({"range": "bytes=11-111", "spoonge": "borb"}) def test__process_response(self): download = _download.Download(EXAMPLE_URL) @@ -175,7 +171,7 @@ def test_constructor_defaults(self): assert download.chunk_size == chunk_size assert download.start == 0 assert download.end is None - assert download._headers == _headers_with_agent({}) + assert download._headers == _base_headers({}) assert not download._finished _check_retry_strategy(download) assert download._stream is stream @@ -292,7 +288,7 @@ def test__prepare_request(self): assert method1 == "GET" assert url1 == EXAMPLE_URL assert payload1 is None - assert headers1 == _headers_with_agent({"range": "bytes=0-2047"}) + assert headers1 == _base_headers({"range": "bytes=0-2047"}) download2 = _download.ChunkedDownload( EXAMPLE_URL, chunk_size, None, start=19991 @@ -302,7 +298,7 @@ def test__prepare_request(self): assert method2 == "GET" assert url2 == EXAMPLE_URL assert payload2 is None - assert headers2 == _headers_with_agent({"range": "bytes=19991-20100"}) + assert headers2 == _base_headers({"range": "bytes=19991-20100"}) def test__prepare_request_with_headers(self): chunk_size = 2048 @@ -314,9 +310,8 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers == _headers_with_agent(headers) expected = {"patrizio": "Starf-ish", "range": "bytes=0-2047"} - assert headers == _headers_with_agent(expected) + assert new_headers == _base_headers(expected) def test__make_invalid(self): download = _download.ChunkedDownload(EXAMPLE_URL, 512, None) diff --git a/tests/unit/test__upload.py b/tests/unit/test__upload.py index 8aaf5317..fdab863e 100644 --- a/tests/unit/test__upload.py +++ b/tests/unit/test__upload.py @@ -17,12 +17,12 @@ import sys import mock -import pkg_resources import pytest # type: ignore from google.resumable_media import _helpers from google.resumable_media import _upload from google.resumable_media import common +from google.resumable_media._helpers import _base_headers URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o" @@ -35,16 +35,11 @@ JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n" -def _headers_with_agent(headers): - version = pkg_resources.get_distribution('google-resumable-media').version - return { **headers, 'User-Agent': "google-resumable-media-python/{}".format(version)} - - class TestUploadBase(object): def test_constructor_defaults(self): upload = _upload.UploadBase(SIMPLE_URL) assert upload.upload_url == SIMPLE_URL - assert upload._headers == _headers_with_agent({}) + assert upload._headers == _base_headers({}) assert not upload._finished _check_retry_strategy(upload) @@ -52,7 +47,7 @@ def test_constructor_explicit(self): headers = {"spin": "doctors"} upload = _upload.UploadBase(SIMPLE_URL, headers=headers) assert upload.upload_url == SIMPLE_URL - assert upload._headers == _headers_with_agent(headers) + assert upload._headers == _base_headers(headers) assert not upload._finished _check_retry_strategy(upload) @@ -145,7 +140,7 @@ def test__prepare_request(self): assert method == "POST" assert url == SIMPLE_URL assert payload == data - assert headers == _headers_with_agent({"content-type": content_type}) + assert headers == _base_headers({"content-type": content_type}) def test__prepare_request_with_headers(self): headers = {"x-goog-cheetos": "spicy"} @@ -157,9 +152,8 @@ def test__prepare_request_with_headers(self): assert method == "POST" assert url == SIMPLE_URL assert payload == data - assert new_headers == _headers_with_agent(headers) expected = {"content-type": content_type, "x-goog-cheetos": "spicy"} - assert headers == _headers_with_agent(expected) + assert new_headers == _base_headers(expected) def test_transmit(self): upload = _upload.SimpleUpload(SIMPLE_URL) @@ -173,7 +167,7 @@ class TestMultipartUpload(object): def test_constructor_defaults(self): upload = _upload.MultipartUpload(MULTIPART_URL) assert upload.upload_url == MULTIPART_URL - assert upload._headers == _headers_with_agent({}) + assert upload._headers == _base_headers({}) assert upload._checksum_type is None assert not upload._finished _check_retry_strategy(upload) @@ -182,7 +176,7 @@ def test_constructor_explicit(self): headers = {"spin": "doctors"} upload = _upload.MultipartUpload(MULTIPART_URL, headers=headers, checksum="md5") assert upload.upload_url == MULTIPART_URL - assert upload._headers == _headers_with_agent(headers) + assert upload._headers == _base_headers(headers) assert upload._checksum_type == "md5" assert not upload._finished _check_retry_strategy(upload) @@ -261,18 +255,17 @@ def _prepare_request_helper( def test__prepare_request(self): headers, multipart_type = self._prepare_request_helper() - assert headers == _headers_with_agent({"content-type": multipart_type}) + assert headers == _base_headers({"content-type": multipart_type}) def test__prepare_request_with_headers(self): headers = {"best": "shirt", "worst": "hat"} new_headers, multipart_type = self._prepare_request_helper(headers=headers) - assert new_headers == _headers_with_agent(headers) - expected_headers = _headers_with_agent({ + expected_headers = _base_headers({ "best": "shirt", "content-type": multipart_type, "worst": "hat", }) - assert expected_headers == headers + assert new_headers == expected_headers @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) def test__prepare_request_with_checksum(self, checksum): @@ -283,7 +276,7 @@ def test__prepare_request_with_checksum(self, checksum): headers, multipart_type = self._prepare_request_helper( checksum=checksum, expected_checksum=checksums[checksum] ) - assert headers == _headers_with_agent({ + assert headers == _base_headers({ "content-type": multipart_type, }) @@ -298,7 +291,7 @@ def test__prepare_request_with_checksum_overwrite(self, checksum): expected_checksum=checksums[checksum], test_overwrite=True, ) - assert headers == _headers_with_agent({ + assert headers == _base_headers({ "content-type": multipart_type, }) @@ -315,7 +308,7 @@ def test_constructor(self): chunk_size = ONE_MB upload = _upload.ResumableUpload(RESUMABLE_URL, chunk_size) assert upload.upload_url == RESUMABLE_URL - assert upload._headers == _headers_with_agent({}) + assert upload._headers == _base_headers({}) assert not upload._finished _check_retry_strategy(upload) assert upload._chunk_size == chunk_size @@ -441,7 +434,7 @@ def test__prepare_initiate_request(self): "x-upload-content-length": "{:d}".format(len(data)), "x-upload-content-type": BASIC_CONTENT, } - assert headers == _headers_with_agent(expected_headers) + assert headers == _base_headers(expected_headers) def test_prepare_initiate_request_with_signed_url(self): signed_urls = [ @@ -456,7 +449,7 @@ def test_prepare_initiate_request_with_signed_url(self): "content-type": BASIC_CONTENT, "x-upload-content-length": "{:d}".format(len(data)), } - assert headers == _headers_with_agent(expected_headers) + assert headers == _base_headers(expected_headers) def test__prepare_initiate_request_with_headers(self): headers = {"caviar": "beluga", "top": "quark"} @@ -470,7 +463,7 @@ def test__prepare_initiate_request_with_headers(self): "x-upload-content-length": "{:d}".format(len(data)), "x-upload-content-type": BASIC_CONTENT, } - assert new_headers == _headers_with_agent(expected_headers) + assert new_headers == _base_headers(expected_headers) def test__prepare_initiate_request_known_size(self): total_bytes = 25 @@ -481,7 +474,7 @@ def test__prepare_initiate_request_known_size(self): "x-upload-content-length": "{:d}".format(total_bytes), "x-upload-content-type": BASIC_CONTENT, } - assert headers == _headers_with_agent(expected_headers) + assert headers == _base_headers(expected_headers) def test__prepare_initiate_request_unknown_size(self): _, headers = self._prepare_initiate_request_helper(stream_final=False) @@ -489,7 +482,7 @@ def test__prepare_initiate_request_unknown_size(self): "content-type": "application/json; charset=UTF-8", "x-upload-content-type": BASIC_CONTENT, } - assert headers == _headers_with_agent(expected_headers) + assert headers == _base_headers(expected_headers) def test__prepare_initiate_request_already_initiated(self): upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB) @@ -966,7 +959,7 @@ def test__prepare_recover_request(self): assert payload is None assert headers == {"content-range": "bytes */*"} # Make sure headers are untouched. - assert upload._headers == _headers_with_agent({}) + assert upload._headers == _base_headers({}) def test__prepare_recover_request_with_headers(self): headers = {"lake": "ocean"} @@ -981,7 +974,7 @@ def test__prepare_recover_request_with_headers(self): # Make sure the ``_headers`` are not incorporated. assert "lake" not in new_headers # Make sure headers are untouched. - assert upload._headers == _headers_with_agent({"lake": "ocean"}) + assert upload._headers == _base_headers({"lake": "ocean"}) def test__process_recover_response_bad_status(self): upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB) From 29b4ee5a58b1a1e89bba17859775a836681d2fba Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 25 Jan 2022 12:27:59 -0700 Subject: [PATCH 4/7] blacken it --- google/resumable_media/_helpers.py | 4 ++-- tests/unit/test__download.py | 4 +++- tests/unit/test__upload.py | 28 +++++++++++++++++----------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/google/resumable_media/_helpers.py b/google/resumable_media/_helpers.py index fafc1eff..d4b89a70 100644 --- a/google/resumable_media/_helpers.py +++ b/google/resumable_media/_helpers.py @@ -47,8 +47,8 @@ def do_nothing(): def _base_headers(headers): - version = pkg_resources.get_distribution('google-resumable-media').version - headers['User-Agent'] = "google-resumable-media-python/{}".format(version) + version = pkg_resources.get_distribution("google-resumable-media").version + headers["User-Agent"] = "google-resumable-media-python/{}".format(version) return headers diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index 7d40ba9e..f547496c 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -119,7 +119,9 @@ def test__prepare_request_with_headers(self): assert method == "GET" assert url == EXAMPLE_URL assert payload is None - assert new_headers == _base_headers({"range": "bytes=11-111", "spoonge": "borb"}) + assert new_headers == _base_headers( + {"range": "bytes=11-111", "spoonge": "borb"} + ) def test__process_response(self): download = _download.Download(EXAMPLE_URL) diff --git a/tests/unit/test__upload.py b/tests/unit/test__upload.py index fdab863e..8dc44445 100644 --- a/tests/unit/test__upload.py +++ b/tests/unit/test__upload.py @@ -260,11 +260,13 @@ def test__prepare_request(self): def test__prepare_request_with_headers(self): headers = {"best": "shirt", "worst": "hat"} new_headers, multipart_type = self._prepare_request_helper(headers=headers) - expected_headers = _base_headers({ - "best": "shirt", - "content-type": multipart_type, - "worst": "hat", - }) + expected_headers = _base_headers( + { + "best": "shirt", + "content-type": multipart_type, + "worst": "hat", + } + ) assert new_headers == expected_headers @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) @@ -276,9 +278,11 @@ def test__prepare_request_with_checksum(self, checksum): headers, multipart_type = self._prepare_request_helper( checksum=checksum, expected_checksum=checksums[checksum] ) - assert headers == _base_headers({ - "content-type": multipart_type, - }) + assert headers == _base_headers( + { + "content-type": multipart_type, + } + ) @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) def test__prepare_request_with_checksum_overwrite(self, checksum): @@ -291,9 +295,11 @@ def test__prepare_request_with_checksum_overwrite(self, checksum): expected_checksum=checksums[checksum], test_overwrite=True, ) - assert headers == _base_headers({ - "content-type": multipart_type, - }) + assert headers == _base_headers( + { + "content-type": multipart_type, + } + ) def test_transmit(self): upload = _upload.MultipartUpload(MULTIPART_URL) From 6d75db4206ed412fbb3b30be685a92727df45c41 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 25 Jan 2022 14:18:07 -0700 Subject: [PATCH 5/7] adjust headers --- google/resumable_media/_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/resumable_media/_helpers.py b/google/resumable_media/_helpers.py index d4b89a70..a014f52e 100644 --- a/google/resumable_media/_helpers.py +++ b/google/resumable_media/_helpers.py @@ -21,6 +21,7 @@ import logging import random import pkg_resources +import platform import warnings from google.resumable_media import common @@ -48,7 +49,8 @@ def do_nothing(): def _base_headers(headers): version = pkg_resources.get_distribution("google-resumable-media").version - headers["User-Agent"] = "google-resumable-media-python/{}".format(version) + headers["User-Agent"] = "gcloud-python/{}-resumable-media".format(version) + headers["X-Goog-Api-Client"] = "gl-python/" + platform.python_version() + " gccl/{}" return headers From 279802dc599ac71c3b366aaa48cd33f8325aee99 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 25 Jan 2022 15:16:02 -0700 Subject: [PATCH 6/7] lint --- google/resumable_media/_download.py | 1 - google/resumable_media/_upload.py | 1 - tests/unit/test__download.py | 1 - 3 files changed, 3 deletions(-) diff --git a/google/resumable_media/_download.py b/google/resumable_media/_download.py index 3c2625a3..8a664b58 100644 --- a/google/resumable_media/_download.py +++ b/google/resumable_media/_download.py @@ -16,7 +16,6 @@ import http.client -import pkg_resources import re from google.resumable_media import _helpers diff --git a/google/resumable_media/_upload.py b/google/resumable_media/_upload.py index 3e42828e..355a5781 100644 --- a/google/resumable_media/_upload.py +++ b/google/resumable_media/_upload.py @@ -24,7 +24,6 @@ import http.client import json import os -import pkg_resources import random import re import sys diff --git a/tests/unit/test__download.py b/tests/unit/test__download.py index f547496c..55b026f2 100644 --- a/tests/unit/test__download.py +++ b/tests/unit/test__download.py @@ -16,7 +16,6 @@ import io import mock -import pkg_resources import pytest # type: ignore from google.resumable_media import _download From 6b9ddfc35826d7569ef21ca5095586425d9365c2 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Wed, 26 Jan 2022 10:31:53 -0700 Subject: [PATCH 7/7] match x-goog and user-agent --- google/resumable_media/_helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/resumable_media/_helpers.py b/google/resumable_media/_helpers.py index a014f52e..56aeae57 100644 --- a/google/resumable_media/_helpers.py +++ b/google/resumable_media/_helpers.py @@ -21,7 +21,6 @@ import logging import random import pkg_resources -import platform import warnings from google.resumable_media import common @@ -50,7 +49,7 @@ def do_nothing(): def _base_headers(headers): version = pkg_resources.get_distribution("google-resumable-media").version headers["User-Agent"] = "gcloud-python/{}-resumable-media".format(version) - headers["X-Goog-Api-Client"] = "gl-python/" + platform.python_version() + " gccl/{}" + headers["X-Goog-Api-Client"] = "gcloud-python/{}-resumable-media".format(version) return headers