From 70dfb72318dc4aba59c250920db6f085b895e9d1 Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Fri, 16 Apr 2021 11:22:12 -0700 Subject: [PATCH 1/3] Enabled exists() for CPK encrypted blobs --- .../azure/storage/blob/_blob_client.py | 5 +- .../storage/blob/aio/_blob_client_async.py | 5 +- ...nditions.test_if_blob_with_cpk_exists.yaml | 137 ++++++++++++++++++ ...ns_async.test_if_blob_with_cpk_exists.yaml | 104 +++++++++++++ .../tests/test_blob_access_conditions.py | 17 ++- .../test_blob_access_conditions_async.py | 18 ++- .../tests/test_block_blob_async.py | 4 +- 7 files changed, 284 insertions(+), 6 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_if_blob_with_cpk_exists.yaml create mode 100644 sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions_async.test_if_blob_with_cpk_exists.yaml diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 1b3203c5170a..02ef9774bfce 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -20,7 +20,7 @@ import six from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace -from azure.core.exceptions import ResourceNotFoundError, HttpResponseError +from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError from ._shared import encode_base64 from ._shared.base_client import StorageAccountHostsMixin, parse_connection_str, parse_query, TransportWrapper @@ -1111,6 +1111,9 @@ def exists(self, **kwargs): snapshot=self.snapshot, **kwargs) return True + # Encrypted with CPK + except ResourceExistsError: + return True except HttpResponseError as error: try: process_storage_error(error) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index ba715606ed74..5b8c31b695a3 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -13,7 +13,7 @@ from azure.core.pipeline import AsyncPipeline from azure.core.tracing.decorator_async import distributed_trace_async -from azure.core.exceptions import ResourceNotFoundError, HttpResponseError +from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper from .._shared.policies_async import ExponentialRetry @@ -583,6 +583,9 @@ async def exists(self, **kwargs): snapshot=self.snapshot, **kwargs) return True + # Encrypted with CPK + except ResourceExistsError: + return True except HttpResponseError as error: try: process_storage_error(error) diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_if_blob_with_cpk_exists.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_if_blob_with_cpk_exists.yaml new file mode 100644 index 000000000000..3e4039e892d9 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions.test_if_blob_with_cpk_exists.yaml @@ -0,0 +1,137 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-date: + - Fri, 16 Apr 2021 18:19:25 GMT + x-ms-version: + - '2020-06-12' + method: PUT + uri: https://storagename.blob.core.windows.net/testcontainer18bd516fa?restype=container + response: + body: + string: '' + headers: + content-length: + - '0' + date: + - Fri, 16 Apr 2021 18:19:26 GMT + etag: + - '"0x8D901042B8B036A"' + last-modified: + - Fri, 16 Apr 2021 18:19:27 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: + - '2020-06-12' + status: + code: 201 + message: Created +- request: + body: hello world + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '11' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 16 Apr 2021 18:19:26 GMT + x-ms-encryption-algorithm: + - AES256 + x-ms-encryption-key: + - MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc= + x-ms-encryption-key-sha256: + - 3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE= + x-ms-version: + - '2020-06-12' + method: PUT + uri: https://storagename.blob.core.windows.net/testcontainer18bd516fa/test_blob + response: + body: + string: '' + headers: + content-length: + - '0' + content-md5: + - XrY7u+Ae7tCTyyK7j1rNww== + date: + - Fri, 16 Apr 2021 18:19:26 GMT + etag: + - '"0x8D901042BA21D7B"' + last-modified: + - Fri, 16 Apr 2021 18:19:27 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: + - vo7q9sPVKY0= + x-ms-encryption-key-sha256: + - 3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE= + x-ms-request-server-encrypted: + - 'true' + x-ms-version: + - '2020-06-12' + x-ms-version-id: + - '2021-04-16T18:19:27.3177467Z' + status: + code: 201 + message: Created +- request: + body: null + headers: + Accept: + - application/xml + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-date: + - Fri, 16 Apr 2021 18:19:26 GMT + x-ms-encryption-algorithm: + - AES256 + x-ms-version: + - '2020-06-12' + method: HEAD + uri: https://storagename.blob.core.windows.net/testcontainer18bd516fa/test_blob + response: + body: + string: '' + headers: + date: + - Fri, 16 Apr 2021 18:19:27 GMT + server: + - Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: + - chunked + x-ms-error-code: + - BlobUsesCustomerSpecifiedEncryption + x-ms-version: + - '2020-06-12' + status: + code: 409 + message: The blob is encrypted with customer specified encryption, but it was + not provided in the request. +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions_async.test_if_blob_with_cpk_exists.yaml b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions_async.test_if_blob_with_cpk_exists.yaml new file mode 100644 index 000000000000..955489f8944a --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/recordings/test_blob_access_conditions_async.test_if_blob_with_cpk_exists.yaml @@ -0,0 +1,104 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-date: + - Fri, 16 Apr 2021 18:21:19 GMT + x-ms-version: + - '2020-06-12' + method: PUT + uri: https://storagename.blob.core.windows.net/testcontainer11f981977?restype=container + response: + body: + string: '' + headers: + content-length: '0' + date: Fri, 16 Apr 2021 18:21:20 GMT + etag: '"0x8D901046F61435E"' + last-modified: Fri, 16 Apr 2021 18:21:20 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-version: '2020-06-12' + status: + code: 201 + message: Created + url: https://tamerdevtest.blob.core.windows.net/testcontainer11f981977?restype=container +- request: + body: hello world + headers: + Accept: + - application/xml + Content-Length: + - '11' + Content-Type: + - application/octet-stream + If-None-Match: + - '*' + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-blob-type: + - BlockBlob + x-ms-date: + - Fri, 16 Apr 2021 18:21:19 GMT + x-ms-encryption-algorithm: + - AES256 + x-ms-encryption-key: + - MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc= + x-ms-encryption-key-sha256: + - 3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE= + x-ms-version: + - '2020-06-12' + method: PUT + uri: https://storagename.blob.core.windows.net/testcontainer11f981977/test_blob + response: + body: + string: '' + headers: + content-length: '0' + content-md5: XrY7u+Ae7tCTyyK7j1rNww== + date: Fri, 16 Apr 2021 18:21:20 GMT + etag: '"0x8D901046F72B9B3"' + last-modified: Fri, 16 Apr 2021 18:21:21 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + x-ms-content-crc64: vo7q9sPVKY0= + x-ms-encryption-key-sha256: 3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE= + x-ms-request-server-encrypted: 'true' + x-ms-version: '2020-06-12' + x-ms-version-id: '2021-04-16T18:21:21.0922419Z' + status: + code: 201 + message: Created + url: https://tamerdevtest.blob.core.windows.net/testcontainer11f981977/test_blob +- request: + body: null + headers: + Accept: + - application/xml + User-Agent: + - azsdk-python-storage-blob/12.8.1b1 Python/3.8.5 (Windows-10-10.0.19041-SP0) + x-ms-date: + - Fri, 16 Apr 2021 18:21:20 GMT + x-ms-encryption-algorithm: + - AES256 + x-ms-version: + - '2020-06-12' + method: HEAD + uri: https://storagename.blob.core.windows.net/testcontainer11f981977/test_blob + response: + body: + string: '' + headers: + date: Fri, 16 Apr 2021 18:21:20 GMT + server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + transfer-encoding: chunked + x-ms-error-code: BlobUsesCustomerSpecifiedEncryption + x-ms-version: '2020-06-12' + status: + code: 409 + message: The blob is encrypted with customer specified encryption, but it was + not provided in the request. + url: https://tamerdevtest.blob.core.windows.net/testcontainer11f981977/test_blob +version: 1 diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py index dce6d0988f3c..9c00f59f2b5c 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py @@ -27,7 +27,7 @@ BlobSasPermissions, generate_account_sas, ResourceTypes, - AccountSasPermissions, generate_container_sas, ContainerClient, + AccountSasPermissions, generate_container_sas, ContainerClient, CustomerProvidedEncryptionKey, ) from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer @@ -853,6 +853,21 @@ def test_if_blob_exists(self, resource_group, location, storage_account, storage self.assertEqual(blob_snapshot.exists(), True) self.assertEqual(blob.exists(), True) + @GlobalStorageAccountPreparer() + def test_if_blob_with_cpk_exists(self, resource_group, location, storage_account, storage_account_key): + container_name = self.get_resource_name("testcontainer1") + cc = ContainerClient( + self.account_url(storage_account, "blob"), credential=storage_account_key, container_name=container_name, + connection_data_block_size=4 * 1024) + cc.create_container() + self._setup() + test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", + key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") + blob_client = cc.get_blob_client("test_blob") + blob_client.upload_blob(b"hello world", cpk=test_cpk) + # Act + self.assertTrue(blob_client.exists()) + @GlobalStorageAccountPreparer() def test_get_blob_properties_with_if_modified_fail(self, resource_group, location, storage_account, storage_account_key): bsc = BlobServiceClient(self.account_url(storage_account, "blob"), storage_account_key, connection_data_block_size=4 * 1024) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py index 26cda0f97880..80eb2abc3682 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py @@ -26,7 +26,7 @@ BlobProperties, ContainerSasPermissions, AccessPolicy, generate_account_sas, ResourceTypes, AccountSasPermissions, generate_blob_sas, BlobSasPermissions, - generate_container_sas, + generate_container_sas, CustomerProvidedEncryptionKey, ) from _shared.testcase import GlobalStorageAccountPreparer from _shared.asynctestcase import AsyncStorageTestCase @@ -875,6 +875,22 @@ async def test_if_blob_exists(self, resource_group, location, storage_account, s self.assertEqual(await blob_snapshot.exists(), True) self.assertEqual(await blob.exists(), True) + @GlobalStorageAccountPreparer() + @AsyncStorageTestCase.await_prepared_test + async def test_if_blob_with_cpk_exists(self, resource_group, location, storage_account, storage_account_key): + container_name = self.get_resource_name("testcontainer1") + cc = ContainerClient( + self.account_url(storage_account, "blob"), credential=storage_account_key, container_name=container_name, + connection_data_block_size=4 * 1024) + await cc.create_container() + self._setup() + test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", + key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") + blob_client = cc.get_blob_client("test_blob") + await blob_client.upload_blob(b"hello world", cpk=test_cpk) + # Act + self.assertTrue(await blob_client.exists()) + @GlobalStorageAccountPreparer() @AsyncStorageTestCase.await_prepared_test async def test_get_blob_properties_with_if_modified(self, resource_group, location, storage_account, storage_account_key): diff --git a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py index a2c9155b762f..32eb5b19a994 100644 --- a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py @@ -266,8 +266,8 @@ async def test_upload_blob_from_url_with_cpk(self, resource_group, location, sto # Act await self._setup(storage_account, storage_account_key) source_blob = await self._create_blob(data=b"This is test data to be copied over.") - test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", - key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") +` test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", + key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=")` sas = generate_blob_sas(account_name=storage_account.name, account_key=storage_account_key, container_name=self.container_name, blob_name=source_blob.blob_name, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1)) From 6caa2125bb8a57f182717ac5a88c4d2ee766a867 Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Fri, 16 Apr 2021 11:24:15 -0700 Subject: [PATCH 2/3] fixed mistake I made --- sdk/storage/azure-storage-blob/tests/test_block_blob_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py index 32eb5b19a994..1557dfbc17fa 100644 --- a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py @@ -266,7 +266,7 @@ async def test_upload_blob_from_url_with_cpk(self, resource_group, location, sto # Act await self._setup(storage_account, storage_account_key) source_blob = await self._create_blob(data=b"This is test data to be copied over.") -` test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", + test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=")` sas = generate_blob_sas(account_name=storage_account.name, account_key=storage_account_key, container_name=self.container_name, blob_name=source_blob.blob_name, From 2a330a96d6f7f0151c4b501b20f704660bffecbc Mon Sep 17 00:00:00 2001 From: Tamer Sherif Date: Fri, 16 Apr 2021 11:24:37 -0700 Subject: [PATCH 3/3] fixed mistake I made --- sdk/storage/azure-storage-blob/tests/test_block_blob_async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py index 1557dfbc17fa..a2c9155b762f 100644 --- a/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_block_blob_async.py @@ -267,7 +267,7 @@ async def test_upload_blob_from_url_with_cpk(self, resource_group, location, sto await self._setup(storage_account, storage_account_key) source_blob = await self._create_blob(data=b"This is test data to be copied over.") test_cpk = CustomerProvidedEncryptionKey(key_value="MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=", - key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=")` + key_hash="3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=") sas = generate_blob_sas(account_name=storage_account.name, account_key=storage_account_key, container_name=self.container_name, blob_name=source_blob.blob_name, permission=BlobSasPermissions(read=True), expiry=datetime.utcnow() + timedelta(hours=1))