From 40162ee5eab04dcdfc930886a49c0af0124ebed8 Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Fri, 11 Sep 2020 15:29:38 -0700 Subject: [PATCH 1/5] added sorting fix --- .../storage/blob/_shared/authentication.py | 14 +- ...t_header_metadata_sort_in_upload_blob.yaml | 146 ++++++++++++++++++ 2 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py index d04c1e4fb539..7eaca80c84b3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py @@ -6,6 +6,7 @@ import logging import sys +import locale try: from urllib.parse import urlparse, unquote @@ -30,7 +31,7 @@ logger = logging.getLogger(__name__) - +locale.setlocale(locale.LC_ALL, "") # wraps a given exception with the desired exception type @@ -92,10 +93,19 @@ def _get_canonicalized_resource(self, request): def _get_canonicalized_headers(request): string_to_sign = '' x_ms_headers = [] + x_ms_headers_dict = {} + x_ms_headers_strings = [] for name, value in request.http_request.headers.items(): if name.startswith('x-ms-'): x_ms_headers.append((name.lower(), value)) - x_ms_headers.sort() + for tup in x_ms_headers: + tup_string = ''.join(tup) + x_ms_headers_dict[tup_string] = tup + x_ms_headers_strings.append(tup_string) + + x_ms_headers_strings.sort(key=locale.strxfrm) + x_ms_headers = [x_ms_headers_dict[sorted_string] for sorted_string in x_ms_headers_strings] + for name, value in x_ms_headers: if value is not None: string_to_sign += ''.join([name, ':', value, '\n']) diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml new file mode 100644 index 000000000000..08cee4f5a16a --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml @@ -0,0 +1,146 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) + x-ms-date: + - Fri, 11 Sep 2020 22:29:04 GMT + x-ms-version: + - '2019-12-12' + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainerbd791bbe?restype=container + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 11 Sep 2020 22:29:04 GMT + etag: + - '"0x8D856A217ED337A"' + last-modified: + - Fri, 11 Sep 2020 22:29:05 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2019-12-12' + status: + code: 201 + message: Created +- request: + body: hello world + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '11' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 11 Sep 2020 22:29:05 GMT + x-ms-version: + - '2019-12-12' + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainerbd791bbe/blob1 + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - XrY7u+Ae7tCTyyK7j1rNww== + date: + - Fri, 11 Sep 2020 22:29:05 GMT + etag: + - '"0x8D856A217FD884F"' + last-modified: + - Fri, 11 Sep 2020 22:29:06 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - vo7q9sPVKY0= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + status: + code: 201 + message: Created +- request: + body: hello world + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '11' + Content-Type: + - application/octet-stream + If-Modified-Since: + - Fri, 11 Sep 2020 22:14:05 GMT + User-Agent: + - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 11 Sep 2020 22:29:05 GMT + x-ms-meta-_a_: + - d + x-ms-meta-i0: + - a + x-ms-meta-i_: + - a + x-ms-version: + - '2019-12-12' + method: PUT + uri: https://storagename.blob.core.windows.net/utcontainerbd791bbe/blob1 + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - XrY7u+Ae7tCTyyK7j1rNww== + date: + - Fri, 11 Sep 2020 22:29:05 GMT + etag: + - '"0x8D856A2180E543D"' + last-modified: + - Fri, 11 Sep 2020 22:29:06 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - vo7q9sPVKY0= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2019-12-12' + status: + code: 201 + message: Created +version: 1 From d122f4cd9183364c034159df0e4f3e84ebed190f Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Fri, 11 Sep 2020 15:31:03 -0700 Subject: [PATCH 2/5] added test --- .../tests/test_blob_access_conditions.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py index d711422decb5..7036ad7cc061 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py @@ -285,6 +285,22 @@ def test_delete_container_with_if_unmodified_fail(self, resource_group, location self.assertEqual(StorageErrorCode.condition_not_met, e.exception.error_code) + @GlobalStorageAccountPreparer() + def test_header_metadata_sort_in_upload_blob(self, resource_group, location, storage_account, storage_account_key): + bsc = BlobServiceClient(self.account_url(storage_account, "blob"), storage_account_key, connection_data_block_size=4 * 1024) + self._setup() + data = b'hello world' + container, blob = self._create_container_and_block_blob( + self.container_name, 'blob1', data, bsc) + test_datetime = (datetime.utcnow() - + timedelta(minutes=15)) + metadata = {'i0': 'a', + 'i_': 'a', + '_a_': 'd' + } + # Act + resp = blob.upload_blob(data, length=len(data), if_modified_since=test_datetime, metadata=metadata) + @GlobalStorageAccountPreparer() def test_put_blob_with_if_modified(self, resource_group, location, storage_account, storage_account_key): bsc = BlobServiceClient(self.account_url(storage_account, "blob"), storage_account_key, connection_data_block_size=4 * 1024) From 2cc2afe955bbbb5f867a1bcdbb58d51ec72a651e Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Sun, 13 Sep 2020 10:36:38 -0700 Subject: [PATCH 3/5] manually set the env var --- .../storage/blob/_shared/authentication.py | 2 +- ...t_header_metadata_sort_in_upload_blob.yaml | 26 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py index 7eaca80c84b3..8a2dbbd69645 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -locale.setlocale(locale.LC_ALL, "") +locale.setlocale(locale.LC_COLLATE, "sv_SE.UTF-8") # wraps a given exception with the desired exception type diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml index 08cee4f5a16a..a84c9a5047dc 100644 --- a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml @@ -13,7 +13,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) x-ms-date: - - Fri, 11 Sep 2020 22:29:04 GMT + - Sun, 13 Sep 2020 17:36:17 GMT x-ms-version: - '2019-12-12' method: PUT @@ -25,11 +25,11 @@ interactions: content-length: - '0' date: - - Fri, 11 Sep 2020 22:29:04 GMT + - Sun, 13 Sep 2020 17:36:17 GMT etag: - - '"0x8D856A217ED337A"' + - '"0x8D8580B85FC6B38"' last-modified: - - Fri, 11 Sep 2020 22:29:05 GMT + - Sun, 13 Sep 2020 17:36:18 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -57,7 +57,7 @@ interactions: x-ms-blob-type: - BlockBlob x-ms-date: - - Fri, 11 Sep 2020 22:29:05 GMT + - Sun, 13 Sep 2020 17:36:19 GMT x-ms-version: - '2019-12-12' method: PUT @@ -71,11 +71,11 @@ interactions: content-md5: - XrY7u+Ae7tCTyyK7j1rNww== date: - - Fri, 11 Sep 2020 22:29:05 GMT + - Sun, 13 Sep 2020 17:36:18 GMT etag: - - '"0x8D856A217FD884F"' + - '"0x8D8580B861573D6"' last-modified: - - Fri, 11 Sep 2020 22:29:06 GMT + - Sun, 13 Sep 2020 17:36:19 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-content-crc64: @@ -101,13 +101,13 @@ interactions: Content-Type: - application/octet-stream If-Modified-Since: - - Fri, 11 Sep 2020 22:14:05 GMT + - Sun, 13 Sep 2020 17:21:19 GMT User-Agent: - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) x-ms-blob-type: - BlockBlob x-ms-date: - - Fri, 11 Sep 2020 22:29:05 GMT + - Sun, 13 Sep 2020 17:36:19 GMT x-ms-meta-_a_: - d x-ms-meta-i0: @@ -127,11 +127,11 @@ interactions: content-md5: - XrY7u+Ae7tCTyyK7j1rNww== date: - - Fri, 11 Sep 2020 22:29:05 GMT + - Sun, 13 Sep 2020 17:36:18 GMT etag: - - '"0x8D856A2180E543D"' + - '"0x8D8580B8632EC47"' last-modified: - - Fri, 11 Sep 2020 22:29:06 GMT + - Sun, 13 Sep 2020 17:36:19 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-content-crc64: From 8db56e0537558e74306a3769de4e37b08d05688b Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Sun, 13 Sep 2020 11:17:31 -0700 Subject: [PATCH 4/5] changed setting --- .../storage/blob/_shared/authentication.py | 2 +- ...t_header_metadata_sort_in_upload_blob.yaml | 26 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py index 8a2dbbd69645..fffccd0ecaba 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -locale.setlocale(locale.LC_COLLATE, "sv_SE.UTF-8") +locale.setlocale(locale.LC_COLLATE, "en_US.UTF-8") # wraps a given exception with the desired exception type diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml index a84c9a5047dc..68cdfebbe0f4 100644 --- a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_header_metadata_sort_in_upload_blob.yaml @@ -13,7 +13,7 @@ interactions: User-Agent: - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) x-ms-date: - - Sun, 13 Sep 2020 17:36:17 GMT + - Sun, 13 Sep 2020 18:17:12 GMT x-ms-version: - '2019-12-12' method: PUT @@ -25,11 +25,11 @@ interactions: content-length: - '0' date: - - Sun, 13 Sep 2020 17:36:17 GMT + - Sun, 13 Sep 2020 18:17:12 GMT etag: - - '"0x8D8580B85FC6B38"' + - '"0x8D858113CD5E782"' last-modified: - - Sun, 13 Sep 2020 17:36:18 GMT + - Sun, 13 Sep 2020 18:17:13 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-version: @@ -57,7 +57,7 @@ interactions: x-ms-blob-type: - BlockBlob x-ms-date: - - Sun, 13 Sep 2020 17:36:19 GMT + - Sun, 13 Sep 2020 18:17:13 GMT x-ms-version: - '2019-12-12' method: PUT @@ -71,11 +71,11 @@ interactions: content-md5: - XrY7u+Ae7tCTyyK7j1rNww== date: - - Sun, 13 Sep 2020 17:36:18 GMT + - Sun, 13 Sep 2020 18:17:12 GMT etag: - - '"0x8D8580B861573D6"' + - '"0x8D858113CE74816"' last-modified: - - Sun, 13 Sep 2020 17:36:19 GMT + - Sun, 13 Sep 2020 18:17:13 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-content-crc64: @@ -101,13 +101,13 @@ interactions: Content-Type: - application/octet-stream If-Modified-Since: - - Sun, 13 Sep 2020 17:21:19 GMT + - Sun, 13 Sep 2020 18:02:13 GMT User-Agent: - azsdk-python-storage-blob/12.5.0 Python/3.8.5 (Windows-10-10.0.18362-SP0) x-ms-blob-type: - BlockBlob x-ms-date: - - Sun, 13 Sep 2020 17:36:19 GMT + - Sun, 13 Sep 2020 18:17:13 GMT x-ms-meta-_a_: - d x-ms-meta-i0: @@ -127,11 +127,11 @@ interactions: content-md5: - XrY7u+Ae7tCTyyK7j1rNww== date: - - Sun, 13 Sep 2020 17:36:18 GMT + - Sun, 13 Sep 2020 18:17:12 GMT etag: - - '"0x8D8580B8632EC47"' + - '"0x8D858113CFBE550"' last-modified: - - Sun, 13 Sep 2020 17:36:19 GMT + - Sun, 13 Sep 2020 18:17:13 GMT server: - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 x-ms-content-crc64: From e3e643d460fcd47be70b2b4a0d68f99ba66e2790 Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Mon, 14 Sep 2020 10:36:59 -0700 Subject: [PATCH 5/5] removed hardcoded env setting --- .../azure/storage/blob/_shared/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py index fffccd0ecaba..90ade822dab1 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/authentication.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -locale.setlocale(locale.LC_COLLATE, "en_US.UTF-8") +locale.setlocale(locale.LC_COLLATE, "") # wraps a given exception with the desired exception type