From 88055b3d4c9a75c25f9a6d78016fe87bbb51d48e Mon Sep 17 00:00:00 2001 From: Jacob Lauzon <96087589+jalauzon-msft@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:01:39 -0700 Subject: [PATCH] [Storage] Remove client-side encryption code from shared (#24931) --- .../azure/storage/blob/_blob_client.py | 52 +- .../storage/blob/_blob_service_client.py | 30 +- .../azure/storage/blob/_container_client.py | 41 +- .../azure/storage/blob/_download.py | 11 +- .../{_shared/encryption.py => _encryption.py} | 18 +- .../azure/storage/blob/_shared/base_client.py | 11 - .../azure/storage/blob/_shared/uploads.py | 25 +- .../storage/blob/_shared/uploads_async.py | 25 +- .../azure/storage/blob/_upload_helpers.py | 55 +- .../storage/blob/aio/_blob_client_async.py | 32 +- .../blob/aio/_blob_service_client_async.py | 31 +- .../blob/aio/_container_client_async.py | 33 +- .../azure/storage/blob/aio/_download_async.py | 16 +- .../azure/storage/blob/aio/_upload_helpers.py | 55 +- .../tests/test_blob_encryption.py | 28 +- .../tests/test_blob_encryption_async.py | 30 +- .../tests/test_blob_encryption_v2.py | 4 +- .../tests/test_blob_encryption_v2_async.py | 8 +- .../_data_lake_directory_client.py | 15 +- .../filedatalake/_data_lake_file_client.py | 4 +- .../filedatalake/_data_lake_service_client.py | 15 +- .../filedatalake/_file_system_client.py | 15 +- .../storage/filedatalake/_path_client.py | 15 +- .../filedatalake/_shared/base_client.py | 3 - .../storage/filedatalake/_shared/uploads.py | 16 +- .../filedatalake/_shared/uploads_async.py | 20 +- .../aio/_data_lake_directory_client_async.py | 15 +- .../aio/_data_lake_file_client_async.py | 4 +- .../aio/_data_lake_service_client_async.py | 15 +- .../aio/_file_system_client_async.py | 15 +- .../filedatalake/aio/_path_client_async.py | 7 +- .../storage/fileshare/_directory_client.py | 3 +- .../azure/storage/fileshare/_download.py | 69 +- .../azure/storage/fileshare/_file_client.py | 17 +- .../storage/fileshare/_shared/base_client.py | 3 - .../storage/fileshare/_shared/encryption.py | 965 ------------------ .../storage/fileshare/_shared/uploads.py | 16 +- .../fileshare/_shared/uploads_async.py | 20 +- .../fileshare/aio/_directory_client_async.py | 3 +- .../storage/fileshare/aio/_download_async.py | 52 +- .../fileshare/aio/_file_client_async.py | 15 +- .../azure/storage/queue/_encryption.py} | 18 +- .../azure/storage/queue/_message_encoding.py | 4 +- .../azure/storage/queue/_queue_client.py | 23 +- .../storage/queue/_queue_service_client.py | 25 +- .../storage/queue/_shared/base_client.py | 11 - .../azure/storage/queue/_shared/encryption.py | 965 ------------------ .../azure/storage/queue/_shared/uploads.py | 16 +- .../storage/queue/_shared/uploads_async.py | 20 +- .../storage/queue/aio/_queue_client_async.py | 26 +- .../queue/aio/_queue_service_client_async.py | 30 +- .../tests/test_queue_encryption.py | 34 +- .../tests/test_queue_encryption_async.py | 41 +- 53 files changed, 400 insertions(+), 2640 deletions(-) rename sdk/storage/azure-storage-blob/azure/storage/blob/{_shared/encryption.py => _encryption.py} (97%) delete mode 100644 sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/encryption.py rename sdk/storage/{azure-storage-file-datalake/azure/storage/filedatalake/_shared/encryption.py => azure-storage-queue/azure/storage/queue/_encryption.py} (97%) delete mode 100644 sdk/storage/azure-storage-queue/azure/storage/queue/_shared/encryption.py diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py index 7522aebd9f91..8ef774fe8418 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py @@ -4,35 +4,31 @@ # license information. # -------------------------------------------------------------------------- # pylint: disable=too-many-lines,no-self-use + from functools import partial from io import BytesIO -from typing import ( # pylint: disable=unused-import - Union, Optional, Any, IO, Iterable, AnyStr, Dict, List, Tuple, - TYPE_CHECKING, - TypeVar, Type) +from typing import ( + Any, AnyStr, Dict, IO, Iterable, List, Optional, Tuple, Type, TypeVar, Union, + TYPE_CHECKING +) +from urllib.parse import urlparse, quote, unquote import warnings -try: - from urllib.parse import urlparse, quote, unquote -except ImportError: - from urlparse import urlparse # type: ignore - from urllib2 import quote, unquote # type: ignore import six +from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError from azure.core.paging import ItemPaged from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace -from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError from ._shared import encode_base64 from ._shared.base_client import StorageAccountHostsMixin, parse_connection_str, parse_query, TransportWrapper -from ._shared.encryption import generate_blob_encryption_data from ._shared.uploads import IterStreamer from ._shared.request_handlers import ( add_metadata_headers, get_length, read_length, validate_and_format_range_headers) from ._shared.response_handlers import return_response_headers, process_storage_error, return_headers_and_deserialized from ._generated import AzureBlobStorage -from ._generated.models import ( # pylint: disable=unused-import +from ._generated.models import ( DeleteSnapshotsOptionType, BlobHTTPHeaders, BlockLookupList, @@ -49,22 +45,30 @@ serialize_blob_tags, serialize_query_format, get_access_conditions ) -from ._deserialize import get_page_ranges_result, deserialize_blob_properties, deserialize_blob_stream, parse_tags, \ +from ._deserialize import ( + get_page_ranges_result, + deserialize_blob_properties, + deserialize_blob_stream, + parse_tags, deserialize_pipeline_response_into_cls +) +from ._download import StorageStreamDownloader +from ._encryption import StorageEncryptionMixin +from ._lease import BlobLeaseClient +from ._models import BlobType, BlobBlock, BlobProperties, BlobQueryError, QuickQueryDialect, \ + DelimitedJsonDialect, DelimitedTextDialect, PageRangePaged, PageRange from ._quick_query_helper import BlobQueryReader from ._upload_helpers import ( upload_block_blob, upload_append_blob, - upload_page_blob, _any_conditions) -from ._models import BlobType, BlobBlock, BlobProperties, BlobQueryError, QuickQueryDialect, \ - DelimitedJsonDialect, DelimitedTextDialect, PageRangePaged, PageRange -from ._download import StorageStreamDownloader -from ._lease import BlobLeaseClient + upload_page_blob, + _any_conditions +) if TYPE_CHECKING: from datetime import datetime from ._generated.models import BlockList - from ._models import ( # pylint: disable=unused-import + from ._models import ( ContentSettings, ImmutabilityPolicy, PremiumPageBlobTier, @@ -79,7 +83,7 @@ ClassType = TypeVar("ClassType") -class BlobClient(StorageAccountHostsMixin): # pylint: disable=too-many-public-methods +class BlobClient(StorageAccountHostsMixin, StorageEncryptionMixin): # pylint: disable=too-many-public-methods """A client to interact with a specific blob, although that blob may not yet exist. For more optional configuration, please click @@ -181,6 +185,7 @@ def __init__( super(BlobClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) def _format_url(self, hostname): container_name = self.container_name @@ -359,13 +364,6 @@ def _upload_blob_options( # pylint:disable=too-many-statements 'key': self.key_encryption_key, 'resolver': self.key_resolver_function, } - if self.key_encryption_key is not None: - cek, iv, encryption_data = generate_blob_encryption_data( - self.key_encryption_key, - self.encryption_version) - encryption_options['cek'] = cek - encryption_options['vector'] = iv - encryption_options['data'] = encryption_data encoding = kwargs.pop('encoding', 'UTF-8') if isinstance(data, six.text_type): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py index 9c2651638204..b3b0e834f9e9 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py @@ -7,34 +7,33 @@ import functools import warnings from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, - TYPE_CHECKING, - TypeVar) + Any, Dict, List, Optional, TypeVar, Union, + TYPE_CHECKING +) +from urllib.parse import urlparse - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse # type: ignore - -from azure.core.paging import ItemPaged from azure.core.exceptions import HttpResponseError +from azure.core.paging import ItemPaged from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace -from ._shared.models import LocationMode from ._shared.base_client import StorageAccountHostsMixin, TransportWrapper, parse_connection_str, parse_query +from ._shared.models import LocationMode from ._shared.parser import _to_utc_datetime -from ._shared.response_handlers import return_response_headers, process_storage_error, \ +from ._shared.response_handlers import ( + return_response_headers, + process_storage_error, parse_to_internal_user_delegation_key +) from ._generated import AzureBlobStorage from ._generated.models import StorageServiceProperties, KeyInfo from ._container_client import ContainerClient from ._blob_client import BlobClient -from ._models import ContainerPropertiesPaged +from ._deserialize import service_stats_deserialize, service_properties_deserialize +from ._encryption import StorageEncryptionMixin from ._list_blobs_helper import FilteredBlobPaged +from ._models import ContainerPropertiesPaged from ._serialize import get_api_version -from ._deserialize import service_stats_deserialize, service_properties_deserialize if TYPE_CHECKING: from datetime import datetime @@ -55,7 +54,7 @@ ClassType = TypeVar("ClassType") -class BlobServiceClient(StorageAccountHostsMixin): +class BlobServiceClient(StorageAccountHostsMixin, StorageEncryptionMixin): """A client to interact with the Blob Service at the account level. This client provides operations to retrieve and configure the account properties @@ -137,6 +136,7 @@ def __init__( super(BlobServiceClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) def _format_url(self, hostname): """Format the endpoint URL according to the current location diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py index 5bc1cb6feede..35746aeb7e96 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_container_client.py @@ -7,53 +7,47 @@ import functools from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, AnyStr, Dict, List, Tuple, IO, Iterator, - TYPE_CHECKING, - TypeVar) - - -try: - from urllib.parse import urlparse, quote, unquote -except ImportError: - from urlparse import urlparse # type: ignore - from urllib2 import quote, unquote # type: ignore + Any, AnyStr, Dict, List, IO, Iterable, Iterator, Optional, TypeVar, Union, + TYPE_CHECKING +) +from urllib.parse import urlparse, quote, unquote import six - from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.core.paging import ItemPaged -from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline import Pipeline from azure.core.pipeline.transport import HttpRequest +from azure.core.tracing.decorator import distributed_trace from ._shared.base_client import StorageAccountHostsMixin, TransportWrapper, parse_connection_str, parse_query from ._shared.request_handlers import add_metadata_headers, serialize_iso from ._shared.response_handlers import ( process_storage_error, return_response_headers, - return_headers_and_deserialized) + return_headers_and_deserialized +) from ._generated import AzureBlobStorage from ._generated.models import SignedIdentifier +from ._blob_client import BlobClient from ._deserialize import deserialize_container_properties -from ._serialize import get_modify_conditions, get_container_cpk_scope_info, get_api_version, get_access_conditions -from ._models import ( # pylint: disable=unused-import +from ._encryption import StorageEncryptionMixin +from ._lease import BlobLeaseClient +from ._list_blobs_helper import BlobPrefix, BlobPropertiesPaged, FilteredBlobPaged +from ._models import ( ContainerProperties, BlobProperties, BlobType, - FilteredBlob) -from ._list_blobs_helper import BlobPrefix, BlobPropertiesPaged, FilteredBlobPaged -from ._lease import BlobLeaseClient -from ._blob_client import BlobClient + FilteredBlob +) +from ._serialize import get_modify_conditions, get_container_cpk_scope_info, get_api_version, get_access_conditions if TYPE_CHECKING: - from azure.core.pipeline.transport import HttpTransport, HttpResponse # pylint: disable=ungrouped-imports - from azure.core.pipeline.policies import HTTPPolicy # pylint: disable=ungrouped-imports + from azure.core.pipeline.transport import HttpResponse # pylint: disable=ungrouped-imports from datetime import datetime from ._models import ( # pylint: disable=unused-import PublicAccess, AccessPolicy, - ContentSettings, StandardBlobTier, PremiumPageBlobTier) @@ -73,7 +67,7 @@ def _get_blob_name(blob): ClassType = TypeVar("ClassType") -class ContainerClient(StorageAccountHostsMixin): # pylint: disable=too-many-public-methods +class ContainerClient(StorageAccountHostsMixin, StorageEncryptionMixin): # pylint: disable=too-many-public-methods """A client to interact with a specific container, although that container may not yet exist. @@ -161,6 +155,7 @@ def __init__( super(ContainerClient, self).__init__(parsed_url, service='blob', credential=credential, **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) def _format_url(self, hostname): container_name = self.container_name diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py index 4e47ed9095cf..854340b0b8de 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py @@ -7,25 +7,24 @@ import sys import threading import time - import warnings from io import BytesIO from typing import Iterator, Union import requests from azure.core.exceptions import HttpResponseError, ServiceResponseError - from azure.core.tracing.common import with_current_context -from ._shared.encryption import ( + +from ._shared.request_handlers import validate_and_format_range_headers +from ._shared.response_handlers import process_storage_error, parse_length_from_content_range +from ._deserialize import deserialize_blob_properties, get_page_ranges_result +from ._encryption import ( adjust_blob_size_for_encryption, decrypt_blob, get_adjusted_download_range_and_offset, is_encryption_v2, parse_encryption_data ) -from ._shared.request_handlers import validate_and_format_range_headers -from ._shared.response_handlers import process_storage_error, parse_length_from_content_range -from ._deserialize import deserialize_blob_properties, get_page_ranges_result def process_range_and_offset(start_range, end_range, length, encryption_options, encryption_data): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py similarity index 97% rename from sdk/storage/azure-storage-blob/azure/storage/blob/_shared/encryption.py rename to sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 0e46796b2ff3..af063083877d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -7,6 +7,7 @@ import os import math import sys +import warnings from collections import OrderedDict from io import BytesIO from json import ( @@ -24,8 +25,8 @@ from azure.core.exceptions import HttpResponseError -from .._version import VERSION -from . import encode_base64, decode_base64_to_bytes +from ._version import VERSION +from ._shared import encode_base64, decode_base64_to_bytes _ENCRYPTION_PROTOCOL_V1 = '1.0' @@ -53,6 +54,19 @@ def _validate_key_encryption_key_wrap(kek): raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) +class StorageEncryptionMixin(object): + def configure_encryption(self, kwargs): + self.require_encryption = kwargs.get("require_encryption", False) + self.encryption_version = kwargs.get("encryption_version", "1.0") + self.key_encryption_key = kwargs.get("key_encryption_key") + self.key_resolver_function = kwargs.get("key_resolver_function") + if self.key_encryption_key and self.encryption_version == '1.0': + warnings.warn("This client has been configured to use encryption with version 1.0. " + + "Version 1.0 is deprecated and no longer considered secure. It is highly " + + "recommended that you switch to using version 2.0. The version can be " + + "specified using the 'encryption_version' keyword.") + + class _EncryptionAlgorithm(object): ''' Specifies which client encryption algorithm is used. diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py index 60d1d775f9db..6365e1688e49 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/base_client.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- import logging import uuid -import warnings from typing import ( # pylint: disable=unused-import Optional, Any, @@ -105,16 +104,6 @@ def __init__( primary_hostname = (parsed_url.netloc + parsed_url.path).rstrip('/') self._hosts = {LocationMode.PRIMARY: primary_hostname, LocationMode.SECONDARY: secondary_hostname} - self.require_encryption = kwargs.get("require_encryption", False) - self.encryption_version = kwargs.get("encryption_version", "1.0") - self.key_encryption_key = kwargs.get("key_encryption_key") - self.key_resolver_function = kwargs.get("key_resolver_function") - if self.key_encryption_key and self.encryption_version == '1.0': - warnings.warn("This client has been configured to use encryption with version 1.0. \ - Version 1.0 is deprecated and no longer considered secure. It is highly \ - recommended that you switch to using version 2.0. The version can be \ - specified using the 'encryption_version' keyword.") - self._config, self._pipeline = self._create_pipeline(self.credential, storage_sdk=service, **kwargs) def __enter__(self): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads.py index 4a3ec5051f27..ba2b49218dcd 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads.py @@ -6,23 +6,17 @@ # pylint: disable=no-self-use from concurrent import futures -from io import (BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation) -from threading import Lock +from io import BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation from itertools import islice from math import ceil +from threading import Lock import six - from azure.core.tracing.common import with_current_context from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import ( - GCMBlobEncryptionStream, - get_blob_encryptor_and_padder, - _ENCRYPTION_PROTOCOL_V1, - _ENCRYPTION_PROTOCOL_V2) _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 @@ -56,24 +50,9 @@ def upload_data_chunks( max_concurrency=None, stream=None, validate_content=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - # V1 uses an encryptor/padder to encrypt each chunk - if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - - # V2 wraps the data stream with an encryption stream - elif encryption_options['version'] == _ENCRYPTION_PROTOCOL_V2: - stream = GCMBlobEncryptionStream(encryption_options.get('cek'), stream) - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads_async.py index 99859a08fe72..97be2caf89f8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/uploads_async.py @@ -6,10 +6,9 @@ # pylint: disable=no-self-use import asyncio +import threading from asyncio import Lock from itertools import islice -import threading - from math import ceil import six @@ -17,12 +16,7 @@ from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import ( - GCMBlobEncryptionStream, - get_blob_encryptor_and_padder, - _ENCRYPTION_PROTOCOL_V1, - _ENCRYPTION_PROTOCOL_V2) -from .uploads import SubStream +from .uploads import SubStream, IterStreamer # pylint: disable=unused-import async def _parallel_uploads(uploader, pending, running): @@ -52,24 +46,9 @@ async def upload_data_chunks( chunk_size=None, max_concurrency=None, stream=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - # V1 uses an encryptor/padder to encrypt each chunk - if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - - # V2 wraps the data stream with an encryption stream - elif encryption_options['version'] == _ENCRYPTION_PROTOCOL_V2: - stream = GCMBlobEncryptionStream(encryption_options.get('cek'), stream) - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py index 5757126bf8c7..c0a2fe6e42ba 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_upload_helpers.py @@ -6,34 +6,36 @@ # pylint: disable=no-self-use from io import SEEK_SET, UnsupportedOperation -from typing import Optional, Union, Any, TypeVar, TYPE_CHECKING # pylint: disable=unused-import +from typing import TypeVar, TYPE_CHECKING import six from azure.core.exceptions import ResourceExistsError, ResourceModifiedError, HttpResponseError -from ._shared.response_handlers import ( - process_storage_error, - return_response_headers) +from ._shared.response_handlers import process_storage_error, return_response_headers from ._shared.models import StorageErrorCode from ._shared.uploads import ( upload_data_chunks, upload_substream_blocks, BlockBlobChunkUploader, PageBlobChunkUploader, - AppendBlobChunkUploader) -from ._shared.encryption import ( - encrypt_blob, - get_adjusted_upload_size, - generate_blob_encryption_data, - _ENCRYPTION_PROTOCOL_V2) + AppendBlobChunkUploader +) from ._generated.models import ( BlockLookupList, AppendPositionAccessConditions, ModifiedAccessConditions, ) +from ._encryption import ( + GCMBlobEncryptionStream, + encrypt_blob, + get_adjusted_upload_size, + get_blob_encryptor_and_padder, + generate_blob_encryption_data, + _ENCRYPTION_PROTOCOL_V1, + _ENCRYPTION_PROTOCOL_V2 +) if TYPE_CHECKING: - from datetime import datetime # pylint: disable=unused-import BlobLeaseClient = TypeVar("BlobLeaseClient") _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 @@ -62,7 +64,7 @@ def _any_conditions(modified_access_conditions=None, **kwargs): # pylint: disab ]) -def upload_block_blob( # pylint: disable=too-many-locals +def upload_block_blob( # pylint: disable=too-many-locals, too-many-statements client=None, data=None, stream=None, @@ -131,17 +133,22 @@ def upload_block_blob( # pylint: disable=too-many-locals if use_original_upload_path: total_size = length - if encryption_options.get('key'): + encryptor, padder = None, None + if encryption_options and encryption_options.get('key'): cek, iv, encryption_data = generate_blob_encryption_data( encryption_options['key'], encryption_options['version']) headers['x-ms-meta-encryptiondata'] = encryption_data - encryption_options['cek'] = cek - encryption_options['vector'] = iv + + if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: + encryptor, padder = get_blob_encryptor_and_padder(cek, iv, True) # Adjust total_size for encryption V2 if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V2: + # Adjust total_size for encryption V2 total_size = adjusted_count + # V2 wraps the data stream with an encryption stream + stream = GCMBlobEncryptionStream(cek, stream) block_ids = upload_data_chunks( service=client, @@ -151,8 +158,9 @@ def upload_block_blob( # pylint: disable=too-many-locals max_concurrency=max_concurrency, stream=stream, validate_content=validate_content, - encryption_options=encryption_options, progress_hook=progress_hook, + encryptor=encryptor, + padder=padder, headers=headers, **kwargs ) @@ -220,8 +228,12 @@ def upload_page_blob( except AttributeError: tier = premium_page_blob_tier - if encryption_options and encryption_options.get('data'): - headers['x-ms-meta-encryptiondata'] = encryption_options['data'] + if encryption_options and encryption_options.get('key'): + cek, iv, encryption_data = generate_blob_encryption_data( + encryption_options['key'], + encryption_options['version']) + headers['x-ms-meta-encryptiondata'] = encryption_data + blob_tags_string = kwargs.pop('blob_tags_string', None) progress_hook = kwargs.pop('progress_hook', None) @@ -238,6 +250,12 @@ def upload_page_blob( if length == 0: return response + if encryption_options and encryption_options.get('key'): + if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: + encryptor, padder = get_blob_encryptor_and_padder(cek, iv, False) + kwargs['encryptor'] = encryptor + kwargs['padder'] = padder + kwargs['modified_access_conditions'] = ModifiedAccessConditions(if_match=response['etag']) return upload_data_chunks( service=client, @@ -247,7 +265,6 @@ def upload_page_blob( stream=stream, max_concurrency=max_concurrency, validate_content=validate_content, - encryption_options=encryption_options, progress_hook=progress_hook, headers=headers, **kwargs) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py index 0bc10f73afec..d5e21419e20d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_client_async.py @@ -4,12 +4,13 @@ # license information. # -------------------------------------------------------------------------- # pylint: disable=too-many-lines, invalid-overridden-method + +import warnings from functools import partial from typing import ( # pylint: disable=unused-import - Union, Optional, Any, IO, Iterable, AnyStr, Dict, List, Tuple, + Any, AnyStr, Dict, IO, Iterable, List, Tuple, Optional, Union, TYPE_CHECKING ) -import warnings from azure.core.async_paging import AsyncItemPaged from azure.core.exceptions import ResourceNotFoundError, HttpResponseError, ResourceExistsError @@ -17,25 +18,29 @@ from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async - from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper from .._shared.policies_async import ExponentialRetry from .._shared.response_handlers import return_response_headers, process_storage_error -from .._deserialize import get_page_ranges_result, parse_tags, deserialize_pipeline_response_into_cls -from .._serialize import get_modify_conditions, get_api_version, get_access_conditions from .._generated.aio import AzureBlobStorage from .._generated.models import CpkInfo -from .._deserialize import deserialize_blob_properties from .._blob_client import BlobClient as BlobClientBase +from .._deserialize import ( + deserialize_blob_properties, + deserialize_pipeline_response_into_cls, + get_page_ranges_result, + parse_tags +) +from .._encryption import StorageEncryptionMixin +from .._models import BlobType, BlobBlock, BlobProperties, PageRange +from .._serialize import get_modify_conditions, get_api_version, get_access_conditions +from ._download_async import StorageStreamDownloader +from ._lease_async import BlobLeaseClient +from ._models import PageRangePaged from ._upload_helpers import ( upload_block_blob, upload_append_blob, - upload_page_blob) -from .._models import BlobType, BlobBlock, BlobProperties, PageRange -from ._models import PageRangePaged -from ._lease_async import BlobLeaseClient -from ._download_async import StorageStreamDownloader - + upload_page_blob +) if TYPE_CHECKING: from datetime import datetime @@ -48,7 +53,7 @@ ) -class BlobClient(AsyncStorageAccountHostsMixin, BlobClientBase): # pylint: disable=too-many-public-methods +class BlobClient(AsyncStorageAccountHostsMixin, BlobClientBase, StorageEncryptionMixin): # pylint: disable=too-many-public-methods """A client to interact with a specific blob, although that blob may not yet exist. :param str account_url: @@ -126,6 +131,7 @@ def __init__( **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) @distributed_trace_async async def get_account_information(self, **kwargs): # type: ignore diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py index a952cf9fa39b..7d98e6fd9e4d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_blob_service_client_async.py @@ -4,38 +4,44 @@ # license information. # -------------------------------------------------------------------------- # pylint: disable=invalid-overridden-method + import functools import warnings from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, + Any, Dict, List, Optional, Union, TYPE_CHECKING ) +from azure.core.async_paging import AsyncItemPaged from azure.core.exceptions import HttpResponseError -from azure.core.tracing.decorator import distributed_trace from azure.core.pipeline import AsyncPipeline +from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async -from azure.core.async_paging import AsyncItemPaged -from .._shared.models import LocationMode -from .._shared.policies_async import ExponentialRetry + from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper -from .._shared.response_handlers import return_response_headers, process_storage_error +from .._shared.response_handlers import ( + parse_to_internal_user_delegation_key, + process_storage_error, + return_response_headers, +) +from .._shared.models import LocationMode from .._shared.parser import _to_utc_datetime -from .._shared.response_handlers import parse_to_internal_user_delegation_key +from .._shared.policies_async import ExponentialRetry from .._generated.aio import AzureBlobStorage from .._generated.models import StorageServiceProperties, KeyInfo from .._blob_service_client import BlobServiceClient as BlobServiceClientBase -from ._container_client_async import ContainerClient -from ._blob_client_async import BlobClient -from .._models import ContainerProperties from .._deserialize import service_stats_deserialize, service_properties_deserialize +from .._encryption import StorageEncryptionMixin +from .._models import ContainerProperties from .._serialize import get_api_version +from ._blob_client_async import BlobClient +from ._container_client_async import ContainerClient from ._models import ContainerPropertiesPaged, FilteredBlobPaged if TYPE_CHECKING: from datetime import datetime - from .._shared.models import AccountSasPermissions, ResourceTypes, UserDelegationKey + from .._shared.models import UserDelegationKey from ._lease_async import BlobLeaseClient from .._models import ( BlobProperties, @@ -48,7 +54,7 @@ ) -class BlobServiceClient(AsyncStorageAccountHostsMixin, BlobServiceClientBase): +class BlobServiceClient(AsyncStorageAccountHostsMixin, BlobServiceClientBase, StorageEncryptionMixin): """A client to interact with the Blob Service at the account level. This client provides operations to retrieve and configure the account properties @@ -119,6 +125,7 @@ def __init__( **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) @distributed_trace_async async def get_user_delegation_key(self, key_start_time, # type: datetime diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py index a72555d36212..64057d5dd030 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_container_client_async.py @@ -1,22 +1,22 @@ -# pylint: disable=too-many-lines # ------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# pylint: disable=invalid-overridden-method +# pylint: disable=too-many-lines, invalid-overridden-method + import functools from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, AnyStr, Dict, List, IO, AsyncIterator, + Any, AnyStr, AsyncIterator, Dict, List, IO, Iterable, Optional, Union, TYPE_CHECKING ) -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError -from azure.core.tracing.decorator import distributed_trace -from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.async_paging import AsyncItemPaged +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.core.pipeline import AsyncPipeline from azure.core.pipeline.transport import AsyncHttpResponse +from azure.core.tracing.decorator import distributed_trace +from azure.core.tracing.decorator_async import distributed_trace_async from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper from .._shared.policies_async import ExponentialRetry @@ -24,29 +24,31 @@ from .._shared.response_handlers import ( process_storage_error, return_response_headers, - return_headers_and_deserialized) + return_headers_and_deserialized +) from .._generated.aio import AzureBlobStorage from .._generated.models import SignedIdentifier +from .._container_client import ContainerClient as ContainerClientBase, _get_blob_name from .._deserialize import deserialize_container_properties +from .._encryption import StorageEncryptionMixin +from .._models import ContainerProperties, BlobType, BlobProperties, FilteredBlob from .._serialize import get_modify_conditions, get_container_cpk_scope_info, get_api_version, get_access_conditions -from .._container_client import ContainerClient as ContainerClientBase, _get_blob_name -from .._models import ContainerProperties, BlobType, BlobProperties, FilteredBlob # pylint: disable=unused-import -from ._list_blobs_helper import BlobPropertiesPaged, BlobPrefix -from ._lease_async import BlobLeaseClient from ._blob_client_async import BlobClient +from ._lease_async import BlobLeaseClient +from ._list_blobs_helper import BlobPropertiesPaged, BlobPrefix from ._models import FilteredBlobPaged if TYPE_CHECKING: - from .._models import PublicAccess - from ._download_async import StorageStreamDownloader from datetime import datetime + from ._download_async import StorageStreamDownloader from .._models import ( # pylint: disable=unused-import AccessPolicy, StandardBlobTier, - PremiumPageBlobTier) + PremiumPageBlobTier, + PublicAccess) -class ContainerClient(AsyncStorageAccountHostsMixin, ContainerClientBase): +class ContainerClient(AsyncStorageAccountHostsMixin, ContainerClientBase, StorageEncryptionMixin): """A client to interact with a specific container, although that container may not yet exist. @@ -119,6 +121,7 @@ def __init__( **kwargs) self._client = AzureBlobStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) @distributed_trace_async async def create_container(self, metadata=None, public_access=None, **kwargs): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index c95c3c6dfc22..8131c0076489 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -5,25 +5,27 @@ # -------------------------------------------------------------------------- # pylint: disable=invalid-overridden-method -import asyncio import sys +import warnings from io import BytesIO from itertools import islice -import warnings from typing import AsyncIterator +import asyncio from aiohttp import ClientPayloadError from azure.core.exceptions import HttpResponseError, ServiceResponseError -from .._shared.encryption import ( + +from .._shared.request_handlers import validate_and_format_range_headers +from .._shared.response_handlers import process_storage_error, parse_length_from_content_range +from .._deserialize import deserialize_blob_properties, get_page_ranges_result +from .._download import process_range_and_offset, _ChunkDownloader +from .._encryption import ( adjust_blob_size_for_encryption, decrypt_blob, is_encryption_v2, parse_encryption_data ) -from .._shared.request_handlers import validate_and_format_range_headers -from .._shared.response_handlers import process_storage_error, parse_length_from_content_range -from .._deserialize import deserialize_blob_properties, get_page_ranges_result -from .._download import process_range_and_offset, _ChunkDownloader + async def process_content(data, start_offset, end_offset, encryption): if data is None: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py index 81a5e07990f8..2716d7be8139 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_upload_helpers.py @@ -6,38 +6,40 @@ # pylint: disable=no-self-use from io import SEEK_SET, UnsupportedOperation -from typing import Optional, Union, Any, TypeVar, TYPE_CHECKING # pylint: disable=unused-import +from typing import TypeVar, TYPE_CHECKING import six from azure.core.exceptions import ResourceModifiedError, HttpResponseError -from .._shared.response_handlers import ( - process_storage_error, - return_response_headers) +from .._shared.response_handlers import process_storage_error, return_response_headers from .._shared.uploads_async import ( upload_data_chunks, upload_substream_blocks, BlockBlobChunkUploader, PageBlobChunkUploader, - AppendBlobChunkUploader) -from .._shared.encryption import ( - encrypt_blob, - get_adjusted_upload_size, - generate_blob_encryption_data, - _ENCRYPTION_PROTOCOL_V2) + AppendBlobChunkUploader +) from .._generated.models import ( BlockLookupList, AppendPositionAccessConditions, ModifiedAccessConditions, ) +from .._encryption import ( + GCMBlobEncryptionStream, + encrypt_blob, + get_adjusted_upload_size, + get_blob_encryptor_and_padder, + generate_blob_encryption_data, + _ENCRYPTION_PROTOCOL_V1, + _ENCRYPTION_PROTOCOL_V2 +) from .._upload_helpers import _convert_mod_error, _any_conditions if TYPE_CHECKING: - from datetime import datetime # pylint: disable=unused-import BlobLeaseClient = TypeVar("BlobLeaseClient") -async def upload_block_blob( # pylint: disable=too-many-locals +async def upload_block_blob( # pylint: disable=too-many-locals, too-many-statements client=None, data=None, stream=None, @@ -105,17 +107,22 @@ async def upload_block_blob( # pylint: disable=too-many-locals if use_original_upload_path: total_size = length - if encryption_options.get('key'): + encryptor, padder = None, None + if encryption_options and encryption_options.get('key'): cek, iv, encryption_data = generate_blob_encryption_data( encryption_options['key'], encryption_options['version']) headers['x-ms-meta-encryptiondata'] = encryption_data - encryption_options['cek'] = cek - encryption_options['vector'] = iv + + if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: + encryptor, padder = get_blob_encryptor_and_padder(cek, iv, True) # Adjust total_size for encryption V2 if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V2: + # Adjust total_size for encryption V2 total_size = adjusted_count + # V2 wraps the data stream with an encryption stream + stream = GCMBlobEncryptionStream(cek, stream) block_ids = await upload_data_chunks( service=client, @@ -125,8 +132,9 @@ async def upload_block_blob( # pylint: disable=too-many-locals max_concurrency=max_concurrency, stream=stream, validate_content=validate_content, - encryption_options=encryption_options, progress_hook=progress_hook, + encryptor=encryptor, + padder=padder, headers=headers, **kwargs ) @@ -194,8 +202,12 @@ async def upload_page_blob( except AttributeError: tier = premium_page_blob_tier - if encryption_options and encryption_options.get('data'): - headers['x-ms-meta-encryptiondata'] = encryption_options['data'] + if encryption_options and encryption_options.get('key'): + cek, iv, encryption_data = generate_blob_encryption_data( + encryption_options['key'], + encryption_options['version']) + headers['x-ms-meta-encryptiondata'] = encryption_data + blob_tags_string = kwargs.pop('blob_tags_string', None) progress_hook = kwargs.pop('progress_hook', None) @@ -212,6 +224,12 @@ async def upload_page_blob( if length == 0: return response + if encryption_options and encryption_options.get('key'): + if encryption_options['version'] == _ENCRYPTION_PROTOCOL_V1: + encryptor, padder = get_blob_encryptor_and_padder(cek, iv, False) + kwargs['encryptor'] = encryptor + kwargs['padder'] = padder + kwargs['modified_access_conditions'] = ModifiedAccessConditions(if_match=response['etag']) return await upload_data_chunks( service=client, @@ -221,7 +239,6 @@ async def upload_page_blob( stream=stream, max_concurrency=max_concurrency, validate_content=validate_content, - encryption_options=encryption_options, progress_hook=progress_hook, headers=headers, **kwargs) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py index 9d881e31534a..5a915305ff96 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py @@ -3,15 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import tempfile - -import pytest -import unittest -from io import ( - StringIO, - BytesIO, -) +import tempfile +from io import StringIO, BytesIO from json import loads from os import ( urandom, @@ -20,29 +14,26 @@ unlink ) +import pytest from azure.core.exceptions import HttpResponseError -from azure.storage.blob._shared.encryption import ( +from azure.storage.blob import BlobServiceClient, BlobType +from azure.storage.blob._blob_client import _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION +from azure.storage.blob._encryption import ( _dict_to_encryption_data, _validate_and_unwrap_cek, _generate_AES_CBC_cipher, _ERROR_OBJECT_INVALID, ) -from azure.storage.blob._blob_client import _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from cryptography.hazmat.primitives.padding import PKCS7 -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer -from azure.storage.blob import ( - BlobServiceClient, - ContainerClient, - BlobClient, - BlobType -) + +from devtools_testutils.storage import StorageTestCase from encryption_test_helper import ( KeyWrapper, KeyResolver, RSAKeyWrapper, ) from settings.testcase import BlobPreparer -from devtools_testutils.storage import StorageTestCase + # ------------------------------------------------------------------------------ TEST_CONTAINER_PREFIX = 'encryption_container' @@ -51,6 +42,7 @@ 'AppendBlob': 'foo'} _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = 'The require_encryption flag is set, but encryption is not supported' + \ ' for this method.' +# ------------------------------------------------------------------------------ class StorageBlobEncryptionTest(StorageTestCase): diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py index 1d7a718355ae..8b5982f27983 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py @@ -4,14 +4,7 @@ # license information. # -------------------------------------------------------------------------- -import pytest -import asyncio - -import unittest -from io import ( - StringIO, - BytesIO, -) +from io import StringIO, BytesIO from json import loads from os import ( urandom, @@ -19,32 +12,29 @@ remove, ) +import pytest from azure.core.exceptions import HttpResponseError from azure.core.pipeline.transport import AioHttpTransport -from multidict import CIMultiDict, CIMultiDictProxy -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer -from azure.storage.blob._shared.encryption import ( +from azure.storage.blob import BlobType +from azure.storage.blob.aio import BlobServiceClient +from azure.storage.blob._blob_client import _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION +from azure.storage.blob._encryption import ( _dict_to_encryption_data, _validate_and_unwrap_cek, _generate_AES_CBC_cipher, _ERROR_OBJECT_INVALID, ) -from azure.storage.blob._blob_client import _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION from cryptography.hazmat.primitives.padding import PKCS7 +from multidict import CIMultiDict, CIMultiDictProxy -from azure.storage.blob import BlobType -from azure.storage.blob.aio import ( - BlobServiceClient, - ContainerClient, - BlobClient, -) +from devtools_testutils.storage.aio import AsyncStorageTestCase from encryption_test_helper import ( KeyWrapper, KeyResolver, RSAKeyWrapper, ) from settings.testcase import BlobPreparer -from devtools_testutils.storage.aio import AsyncStorageTestCase + # ------------------------------------------------------------------------------ TEST_CONTAINER_PREFIX = 'encryption_container' @@ -53,8 +43,6 @@ 'AppendBlob': 'foo'} _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = 'The require_encryption flag is set, but encryption is not supported' + \ ' for this method.' - - # ------------------------------------------------------------------------------ diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 12819d2e569c..4f8b95e98a03 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -6,16 +6,16 @@ import base64 import os -import pytest from json import dumps, loads +import pytest from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError from azure.storage.blob import ( BlobServiceClient, BlobType ) -from azure.storage.blob._shared.encryption import ( +from azure.storage.blob._encryption import ( _dict_to_encryption_data, _validate_and_unwrap_cek, _GCM_NONCE_LENGTH, diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py index 04b673fb7989..eaeb4968852d 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py @@ -6,16 +6,16 @@ import base64 import os -import pytest from json import dumps, loads +import pytest from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError from azure.storage.blob import BlobType from azure.storage.blob.aio import ( BlobServiceClient ) -from azure.storage.blob._shared.encryption import ( +from azure.storage.blob._encryption import ( _dict_to_encryption_data, _validate_and_unwrap_cek, _GCM_NONCE_LENGTH, @@ -23,7 +23,7 @@ ) from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from devtools_testutils.storage import StorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase from encryption_test_helper import ( KeyWrapper, KeyResolver, @@ -36,7 +36,7 @@ MiB = 1024 * 1024 -class StorageBlobEncryptionV2TestAsync(StorageTestCase): +class StorageBlobEncryptionV2TestAsync(AsyncStorageTestCase): # --Helpers----------------------------------------------------------------- async def _setup(self, storage_account_name, key): self.bsc = BlobServiceClient( diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py index fb7584680b04..eeaa5252f596 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_directory_client.py @@ -370,10 +370,7 @@ def rename_directory(self, new_name, **kwargs): new_directory_client = DataLakeDirectoryClient( "{}://{}".format(self.scheme, self.primary_hostname), new_file_system, directory_name=new_path, credential=self._raw_credential or new_dir_sas, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) new_directory_client._rename_path( # pylint: disable=protected-access '/{}/{}{}'.format(quote(unquote(self.file_system_name)), quote(unquote(self.path_name)), @@ -613,10 +610,7 @@ def get_file_client(self, file # type: Union[FileProperties, str] return DataLakeFileClient( self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str] ): @@ -644,7 +638,4 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert return DataLakeDirectoryClient( self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py index 40f8402798ce..60ac24751cb7 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_file_client.py @@ -776,9 +776,7 @@ def rename_file(self, new_name, **kwargs): "{}://{}".format(self.scheme, self.primary_hostname), new_file_system, file_path=new_path, credential=self._raw_credential or new_file_sas, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function + _location_mode=self._location_mode ) new_file_client._rename_path( # pylint: disable=protected-access '/{}/{}{}'.format(quote(unquote(self.file_system_name)), diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py index 4ac2a4447711..a5bfff3c531c 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_data_lake_service_client.py @@ -404,9 +404,7 @@ def get_file_system_client(self, file_system # type: Union[FileSystemProperties return FileSystemClient(self.url, file_system_name, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=_pipeline, _hosts=self._hosts, - require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _pipeline=_pipeline, _hosts=self._hosts) def get_directory_client(self, file_system, # type: Union[FileSystemProperties, str] directory # type: Union[DirectoryProperties, str] @@ -453,11 +451,7 @@ def get_directory_client(self, file_system, # type: Union[FileSystemProperties, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function - ) + _hosts=self._hosts) def get_file_client(self, file_system, # type: Union[FileSystemProperties, str] file_path # type: Union[FileProperties, str] @@ -503,10 +497,7 @@ def get_file_client(self, file_system, # type: Union[FileSystemProperties, str] return DataLakeFileClient( self.url, file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) def set_service_properties(self, **kwargs): # type: (**Any) -> None diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py index 9b33a6166928..79ac40de8c28 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_file_system_client.py @@ -304,9 +304,7 @@ def _rename_file_system(self, new_name, **kwargs): renamed_file_system = FileSystemClient( "{}://{}".format(self.scheme, self.primary_hostname), file_system_name=new_name, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts, - require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts) return renamed_file_system def delete_file_system(self, **kwargs): @@ -900,11 +898,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, _pipeline=_pipeline, - _hosts=self._hosts, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function - ) + _hosts=self._hosts) def get_file_client(self, file_path # type: Union[FileProperties, str] ): @@ -940,10 +934,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str] return DataLakeFileClient( self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline) def list_deleted_paths(self, **kwargs): # type: (Any) -> ItemPaged[DeletedPathProperties] diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py index 16f7cf8a3f44..3119fb81d2da 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_path_client.py @@ -30,12 +30,7 @@ from ._shared.response_handlers import return_response_headers, return_headers_and_deserialized if TYPE_CHECKING: - from ._models import ContentSettings - from ._models import FileProperties - -_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = ( - 'The require_encryption flag is set, but encryption is not supported' - ' for this method.') + from ._models import ContentSettings, FileProperties class PathClient(StorageAccountHostsMixin): @@ -151,9 +146,6 @@ def _create_path_options(self, resource_type, metadata=None, # type: Optional[Dict[str, str]] **kwargs): # type: (...) -> Dict[str, Any] - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - access_conditions = get_access_conditions(kwargs.pop('lease', None)) mod_conditions = get_mod_conditions(kwargs) @@ -722,13 +714,12 @@ def _set_access_control_internal(self, options, progress_hook, max_batches=None) error.continuation_token = last_continuation_token raise error - def _rename_path_options(self, rename_source, + def _rename_path_options(self, # pylint: disable=no-self-use + rename_source, # type: str content_settings=None, # type: Optional[ContentSettings] metadata=None, # type: Optional[Dict[str, str]] **kwargs): # type: (...) -> Dict[str, Any] - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) if metadata or kwargs.pop('permissions', None) or kwargs.pop('umask', None): raise ValueError("metadata, permissions, umask is not supported for this operation") diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py index b80d04a57363..3af10b496f22 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/base_client.py @@ -103,9 +103,6 @@ def __init__( primary_hostname = (parsed_url.netloc + parsed_url.path).rstrip('/') self._hosts = {LocationMode.PRIMARY: primary_hostname, LocationMode.SECONDARY: secondary_hostname} - self.require_encryption = kwargs.get("require_encryption", False) - self.key_encryption_key = kwargs.get("key_encryption_key") - self.key_resolver_function = kwargs.get("key_resolver_function") self._config, self._pipeline = self._create_pipeline(self.credential, storage_sdk=service, **kwargs) def __enter__(self): diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py index 167217734bea..279f084ff970 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads.py @@ -6,19 +6,17 @@ # pylint: disable=no-self-use from concurrent import futures -from io import (BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation) -from threading import Lock +from io import BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation from itertools import islice from math import ceil +from threading import Lock import six - from azure.core.tracing.common import with_current_context from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 @@ -52,18 +50,9 @@ def upload_data_chunks( max_concurrency=None, stream=None, validate_content=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -149,7 +138,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = Lock() if parallel else None # Progress feedback diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads_async.py index 2d8376aff237..97be2caf89f8 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/uploads_async.py @@ -6,10 +6,9 @@ # pylint: disable=no-self-use import asyncio +import threading from asyncio import Lock from itertools import islice -import threading - from math import ceil import six @@ -17,14 +16,9 @@ from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder from .uploads import SubStream, IterStreamer # pylint: disable=unused-import -_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 -_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.' - - async def _parallel_uploads(uploader, pending, running): range_ids = [] while True: @@ -52,18 +46,9 @@ async def upload_data_chunks( chunk_size=None, max_concurrency=None, stream=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -152,7 +137,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = threading.Lock() if parallel else None # Progress feedback @@ -217,7 +201,7 @@ async def _update_progress(self, length): self.progress_total += length if self.progress_hook: - self.progress_hook(self.progress_total, self.total_size) + await self.progress_hook(self.progress_total, self.total_size) async def _upload_chunk(self, chunk_offset, chunk_data): raise NotImplementedError("Must be implemented by child class.") diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py index a31bbaca52df..155846845a89 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_directory_client_async.py @@ -337,10 +337,7 @@ async def rename_directory(self, new_name, # type: str new_directory_client = DataLakeDirectoryClient( "{}://{}".format(self.scheme, self.primary_hostname), new_file_system, directory_name=new_path, credential=self._raw_credential or new_dir_sas, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) await new_directory_client._rename_path( # pylint: disable=protected-access '/{}/{}{}'.format(quote(unquote(self.file_system_name)), quote(unquote(self.path_name)), @@ -590,10 +587,7 @@ def get_file_client(self, file # type: Union[FileProperties, str] return DataLakeFileClient( self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) def get_sub_directory_client(self, sub_directory # type: Union[DirectoryProperties, str] ): @@ -630,7 +624,4 @@ def get_sub_directory_client(self, sub_directory # type: Union[DirectoryPropert return DataLakeDirectoryClient( self.url, self.file_system_name, directory_name=subdir_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py index 3f17c560d62e..271fbd481af3 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_file_client_async.py @@ -621,9 +621,7 @@ async def rename_file(self, new_name, **kwargs): "{}://{}".format(self.scheme, self.primary_hostname), new_file_system, file_path=new_path, credential=self._raw_credential or new_file_sas, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _location_mode=self._location_mode) await new_file_client._rename_path( # pylint: disable=protected-access '/{}/{}{}'.format(quote(unquote(self.file_system_name)), quote(unquote(self.path_name)), diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py index 7e68d3015882..c4c48dddad9f 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_data_lake_service_client_async.py @@ -347,9 +347,7 @@ def get_file_system_client(self, file_system # type: Union[FileSystemProperties return FileSystemClient(self.url, file_system_name, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=self._pipeline, _hosts=self._hosts, - require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _pipeline=self._pipeline, _hosts=self._hosts) def get_directory_client(self, file_system, # type: Union[FileSystemProperties, str] directory # type: Union[DirectoryProperties, str] @@ -396,11 +394,7 @@ def get_directory_client(self, file_system, # type: Union[FileSystemProperties, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, _pipeline=self._pipeline, - _hosts=self._hosts, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function - ) + _hosts=self._hosts) def get_file_client(self, file_system, # type: Union[FileSystemProperties, str] file_path # type: Union[FileProperties, str] @@ -446,10 +440,7 @@ def get_file_client(self, file_system, # type: Union[FileSystemProperties, str] return DataLakeFileClient( self.url, file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline) async def set_service_properties(self, **kwargs): # type: (**Any) -> None diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py index 27548cc9d7b5..d96d559da029 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_file_system_client_async.py @@ -246,9 +246,7 @@ async def _rename_file_system(self, new_name, **kwargs): renamed_file_system = FileSystemClient( "{}://{}".format(self.scheme, self.primary_hostname), file_system_name=new_name, credential=self._raw_credential, api_version=self.api_version, _configuration=self._config, - _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts, - require_encryption=self.require_encryption, key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function) + _pipeline=self._pipeline, _location_mode=self._location_mode, _hosts=self._hosts) return renamed_file_system @distributed_trace_async @@ -845,11 +843,7 @@ def get_directory_client(self, directory # type: Union[DirectoryProperties, str api_version=self.api_version, _configuration=self._config, _pipeline=_pipeline, _hosts=self._hosts, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function, - loop=self._loop - ) + loop=self._loop) def get_file_client(self, file_path # type: Union[FileProperties, str] ): @@ -885,10 +879,7 @@ def get_file_client(self, file_path # type: Union[FileProperties, str] return DataLakeFileClient( self.url, self.file_system_name, file_path=file_path, credential=self._raw_credential, api_version=self.api_version, - _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline, - require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, - key_resolver_function=self.key_resolver_function, loop=self._loop) + _hosts=self._hosts, _configuration=self._config, _pipeline=_pipeline, loop=self._loop) @distributed_trace def list_deleted_paths(self, **kwargs): diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py index eafb88b490d3..7d0b11a92c69 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/aio/_path_client_async.py @@ -22,12 +22,7 @@ from .._shared.policies_async import ExponentialRetry if TYPE_CHECKING: - from .._models import ContentSettings - from .._models import FileProperties - -_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = ( - 'The require_encryption flag is set, but encryption is not supported' - ' for this method.') + from .._models import ContentSettings, FileProperties class PathClient(AsyncStorageAccountHostsMixin, PathClientBase): diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_directory_client.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_directory_client.py index 9c5e756a52b2..29e7eb76cab2 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_directory_client.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_directory_client.py @@ -441,8 +441,7 @@ def rename_directory( '{}://{}'.format(self.scheme, self.primary_hostname), self.share_name, new_dir_path, credential=new_dir_sas or self.credential, api_version=self.api_version, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, key_resolver_function=self.key_resolver_function + _location_mode=self._location_mode ) kwargs.update(get_rename_smb_properties(kwargs)) diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_download.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_download.py index a2db5aab44e0..fd10c67c270b 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_download.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_download.py @@ -12,55 +12,18 @@ from azure.core.exceptions import HttpResponseError, ResourceModifiedError from azure.core.tracing.common import with_current_context -from ._shared.encryption import decrypt_blob from ._shared.request_handlers import validate_and_format_range_headers from ._shared.response_handlers import process_storage_error, parse_length_from_content_range -def process_range_and_offset(start_range, end_range, length, encryption): - start_offset, end_offset = 0, 0 - if encryption.get("key") is not None or encryption.get("resolver") is not None: - if start_range is not None: - # Align the start of the range along a 16 byte block - start_offset = start_range % 16 - start_range -= start_offset - - # Include an extra 16 bytes for the IV if necessary - # Because of the previous offsetting, start_range will always - # be a multiple of 16. - if start_range > 0: - start_offset += 16 - start_range -= 16 - - if length is not None: - # Align the end of the range along a 16 byte block - end_offset = 15 - (end_range % 16) - end_range += end_offset - - return (start_range, end_range), (start_offset, end_offset) - - -def process_content(data, start_offset, end_offset, encryption): +def process_content(data): if data is None: raise ValueError("Response cannot be None.") + try: - content = b"".join(list(data)) + return b"".join(list(data)) except Exception as error: raise HttpResponseError(message="Download stream interrupted.", response=data.response, error=error) - if content and encryption.get("key") is not None or encryption.get("resolver") is not None: - try: - return decrypt_blob( - encryption.get("required"), - encryption.get("key"), - encryption.get("resolver"), - content, - start_offset, - end_offset, - data.response.headers, - ) - except Exception as error: - raise HttpResponseError(message="Decryption failed.", response=data.response, error=error) - return content class _ChunkDownloader(object): # pylint: disable=too-many-instance-attributes @@ -75,7 +38,6 @@ def __init__( stream=None, parallel=None, validate_content=None, - encryption_options=None, etag=None, **kwargs ): @@ -99,9 +61,6 @@ def __init__( # Download progress so far self.progress_total = current_progress - # Encryption - self.encryption_options = encryption_options - # Parameters for each get operation self.validate_content = validate_content self.request_options = kwargs @@ -147,11 +106,8 @@ def _write_to_stream(self, chunk_data, chunk_start): self.stream.write(chunk_data) def _download_chunk(self, chunk_start, chunk_end): - download_range, offset = process_range_and_offset( - chunk_start, chunk_end, chunk_end, self.encryption_options - ) range_header, range_validation = validate_and_format_range_headers( - download_range[0], download_range[1], check_content_md5=self.validate_content + chunk_start, chunk_end, check_content_md5=self.validate_content ) try: @@ -169,7 +125,7 @@ def _download_chunk(self, chunk_start, chunk_end): except HttpResponseError as error: process_storage_error(error) - chunk_data = process_content(response, offset[0], offset[1], self.encryption_options) + chunk_data = process_content(response) return chunk_data @@ -251,7 +207,6 @@ def __init__( start_range=None, end_range=None, validate_content=None, - encryption_options=None, max_concurrency=1, name=None, path=None, @@ -272,7 +227,6 @@ def __init__( self._max_concurrency = max_concurrency self._encoding = encoding self._validate_content = validate_content - self._encryption_options = encryption_options or {} self._request_options = kwargs self._location_mode = None self._download_complete = False @@ -293,9 +247,7 @@ def __init__( else: initial_request_end = initial_request_start + self._first_get_size - 1 - self._initial_range, self._initial_offset = process_range_and_offset( - initial_request_start, initial_request_end, self._end_range, self._encryption_options - ) + self._initial_range = (initial_request_start, initial_request_end) self._response = self._initial_request() self.properties = self._response.properties @@ -322,12 +274,7 @@ def __init__( if self.size == 0: self._current_content = b"" else: - self._current_content = process_content( - self._response, - self._initial_offset[0], - self._initial_offset[1], - self._encryption_options - ) + self._current_content = process_content(self._response) def __len__(self): return self.size @@ -417,7 +364,6 @@ def chunks(self): stream=None, parallel=False, validate_content=self._validate_content, - encryption_options=self._encryption_options, use_location=self._location_mode, etag=self._etag, **self._request_options @@ -518,7 +464,6 @@ def readinto(self, stream): stream=stream, parallel=parallel, validate_content=self._validate_content, - encryption_options=self._encryption_options, use_location=self._location_mode, etag=self._etag, **self._request_options diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py index 7fa33632a785..44e216d90f54 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_file_client.py @@ -387,9 +387,6 @@ def create_file( # type: ignore content_settings = kwargs.pop('content_settings', None) metadata = kwargs.pop('metadata', None) timeout = kwargs.pop('timeout', None) - if self.require_encryption and not self.key_encryption_key: - raise ValueError("Encryption required but no key was provided.") - headers = kwargs.pop('headers', {}) headers.update(add_metadata_headers(metadata)) file_http_headers = None @@ -514,8 +511,6 @@ def upload_file( validate_content = kwargs.pop('validate_content', False) timeout = kwargs.pop('timeout', None) encoding = kwargs.pop('encoding', 'UTF-8') - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if isinstance(data, six.text_type): data = data.encode(encoding) @@ -759,8 +754,6 @@ def download_file( :dedent: 12 :caption: Download a file. """ - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if length is not None and offset is None: raise ValueError("Offset value must not be None if length is set.") @@ -775,7 +768,6 @@ def download_file( config=self._config, start_range=offset, end_range=range_end, - encryption_options=None, name=self.file_name, path='/'.join(self.file_path), share=self.share_name, @@ -898,8 +890,7 @@ def rename_file( '{}://{}'.format(self.scheme, self.primary_hostname), self.share_name, new_file_path, credential=new_file_sas or self.credential, api_version=self.api_version, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, key_resolver_function=self.key_resolver_function + _location_mode=self._location_mode ) kwargs.update(get_rename_smb_properties(kwargs)) @@ -1148,8 +1139,6 @@ def upload_range( # type: ignore timeout = kwargs.pop('timeout', None) encoding = kwargs.pop('encoding', 'UTF-8') file_last_write_mode = kwargs.pop('file_last_write_mode', None) - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if isinstance(data, six.text_type): data = data.encode(encoding) @@ -1296,8 +1285,6 @@ def _get_ranges_options( # type: ignore **kwargs ): # type: (...) -> Dict[str, Any] - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Unsupported method for encryption.") access_conditions = get_access_conditions(kwargs.pop('lease', None)) content_range = None @@ -1433,8 +1420,6 @@ def clear_range( # type: ignore """ access_conditions = get_access_conditions(kwargs.pop('lease', None)) timeout = kwargs.pop('timeout', None) - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Unsupported method for encryption.") if offset is None or offset % 512 != 0: raise ValueError("offset must be an integer that aligns with 512 bytes file size") diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client.py index b80d04a57363..3af10b496f22 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/base_client.py @@ -103,9 +103,6 @@ def __init__( primary_hostname = (parsed_url.netloc + parsed_url.path).rstrip('/') self._hosts = {LocationMode.PRIMARY: primary_hostname, LocationMode.SECONDARY: secondary_hostname} - self.require_encryption = kwargs.get("require_encryption", False) - self.key_encryption_key = kwargs.get("key_encryption_key") - self.key_resolver_function = kwargs.get("key_resolver_function") self._config, self._pipeline = self._create_pipeline(self.credential, storage_sdk=service, **kwargs) def __enter__(self): diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/encryption.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/encryption.py deleted file mode 100644 index 0e46796b2ff3..000000000000 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/encryption.py +++ /dev/null @@ -1,965 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import os -import math -import sys -from collections import OrderedDict -from io import BytesIO -from json import ( - dumps, - loads, -) -from typing import Any, BinaryIO, Dict, Optional, Tuple - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.ciphers import Cipher -from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.hazmat.primitives.ciphers.algorithms import AES -from cryptography.hazmat.primitives.ciphers.modes import CBC -from cryptography.hazmat.primitives.padding import PKCS7 - -from azure.core.exceptions import HttpResponseError - -from .._version import VERSION -from . import encode_base64, decode_base64_to_bytes - - -_ENCRYPTION_PROTOCOL_V1 = '1.0' -_ENCRYPTION_PROTOCOL_V2 = '2.0' -_GCM_REGION_DATA_LENGTH = 4 * 1024 * 1024 -_GCM_NONCE_LENGTH = 12 -_GCM_TAG_LENGTH = 16 - -_ERROR_OBJECT_INVALID = \ - '{0} does not define a complete interface. Value of {1} is either missing or invalid.' - - -def _validate_not_none(param_name, param): - if param is None: - raise ValueError('{0} should not be None.'.format(param_name)) - - -def _validate_key_encryption_key_wrap(kek): - # Note that None is not callable and so will fail the second clause of each check. - if not hasattr(kek, 'wrap_key') or not callable(kek.wrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'wrap_key')) - if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(kek, 'get_key_wrap_algorithm') or not callable(kek.get_key_wrap_algorithm): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) - - -class _EncryptionAlgorithm(object): - ''' - Specifies which client encryption algorithm is used. - ''' - AES_CBC_256 = 'AES_CBC_256' - AES_GCM_256 = 'AES_GCM_256' - - -class _WrappedContentKey: - ''' - Represents the envelope key details stored on the service. - ''' - - def __init__(self, algorithm, encrypted_key, key_id): - ''' - :param str algorithm: - The algorithm used for wrapping. - :param bytes encrypted_key: - The encrypted content-encryption-key. - :param str key_id: - The key-encryption-key identifier string. - ''' - - _validate_not_none('algorithm', algorithm) - _validate_not_none('encrypted_key', encrypted_key) - _validate_not_none('key_id', key_id) - - self.algorithm = algorithm - self.encrypted_key = encrypted_key - self.key_id = key_id - - -class _EncryptedRegionInfo: - ''' - Represents the length of encryption elements. - This is only used for Encryption V2. - ''' - - def __init__(self, data_length, nonce_length, tag_length): - ''' - :param int data_length: - The length of the encryption region data (not including nonce + tag). - :param str nonce_length: - The length of nonce used when encrypting. - :param int tag_length: - The length of the encryption tag. - ''' - _validate_not_none('data_length', data_length) - _validate_not_none('nonce_length', nonce_length) - _validate_not_none('tag_length', tag_length) - - self.data_length = data_length - self.nonce_length = nonce_length - self.tag_length = tag_length - - -class _EncryptionAgent: - ''' - Represents the encryption agent stored on the service. - It consists of the encryption protocol version and encryption algorithm used. - ''' - - def __init__(self, encryption_algorithm, protocol): - ''' - :param _EncryptionAlgorithm encryption_algorithm: - The algorithm used for encrypting the message contents. - :param str protocol: - The protocol version used for encryption. - ''' - - _validate_not_none('encryption_algorithm', encryption_algorithm) - _validate_not_none('protocol', protocol) - - self.encryption_algorithm = str(encryption_algorithm) - self.protocol = protocol - - -class _EncryptionData: - ''' - Represents the encryption data that is stored on the service. - ''' - - def __init__( - self, - content_encryption_IV, - encrypted_region_info, - encryption_agent, - wrapped_content_key, - key_wrapping_metadata): - ''' - :param Optional[bytes] content_encryption_IV: - The content encryption initialization vector. - Required for AES-CBC (V1). - :param Optional[_EncryptedRegionInfo] encrypted_region_info: - The info about the autenticated block sizes. - Required for AES-GCM (V2). - :param _EncryptionAgent encryption_agent: - The encryption agent. - :param _WrappedContentKey wrapped_content_key: - An object that stores the wrapping algorithm, the key identifier, - and the encrypted key bytes. - :param dict key_wrapping_metadata: - A dict containing metadata related to the key wrapping. - ''' - - _validate_not_none('encryption_agent', encryption_agent) - _validate_not_none('wrapped_content_key', wrapped_content_key) - - # Validate we have the right matching optional parameter for the specified algorithm - if encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256: - _validate_not_none('content_encryption_IV', content_encryption_IV) - elif encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_GCM_256: - _validate_not_none('encrypted_region_info', encrypted_region_info) - else: - raise ValueError("Invalid encryption algorithm.") - - self.content_encryption_IV = content_encryption_IV - self.encrypted_region_info = encrypted_region_info - self.encryption_agent = encryption_agent - self.wrapped_content_key = wrapped_content_key - self.key_wrapping_metadata = key_wrapping_metadata - - -class GCMBlobEncryptionStream: - """ - A stream that performs AES-GCM encryption on the given data as - it's streamed. Data is read and encrypted in regions. The stream - will use the same encryption key and will generate a guaranteed unique - nonce for each encryption region. - """ - def __init__( - self, - content_encryption_key: bytes, - data_stream: BinaryIO, - ): - """ - :param bytes content_encryption_key: The encryption key to use. - :param BinaryIO data_stream: The data stream to read data from. - """ - self.content_encryption_key = content_encryption_key - self.data_stream = data_stream - - self.offset = 0 - self.current = b'' - self.nonce_counter = 0 - - def read(self, size: int = -1) -> bytes: - """ - Read data from the stream. Specify -1 to read all available data. - - :param int size: The amount of data to read. Defaults to -1 for all data. - """ - result = BytesIO() - remaining = sys.maxsize if size == -1 else size - - while remaining > 0: - # Start by reading from current - if len(self.current) > 0: - read = min(remaining, len(self.current)) - result.write(self.current[:read]) - - self.current = self.current[read:] - self.offset += read - remaining -= read - - if remaining > 0: - # Read one region of data and encrypt it - data = self.data_stream.read(_GCM_REGION_DATA_LENGTH) - if len(data) == 0: - # No more data to read - break - - self.current = self._encrypt_region(data) - - return result.getvalue() - - def _encrypt_region(self, data: bytes) -> bytes: - """ - Encrypt the given region of data using AES-GCM. The result - includes the data in the form: nonce + ciphertext + tag. - - :param bytes data: The data to encrypt. - """ - # Each region MUST use a different nonce - nonce = self.nonce_counter.to_bytes(_GCM_NONCE_LENGTH, 'big') - self.nonce_counter += 1 - - aesgcm = AESGCM(self.content_encryption_key) - - # Returns ciphertext + tag - cipertext_with_tag = aesgcm.encrypt(nonce, data, None) - return nonce + cipertext_with_tag - - -def is_encryption_v2(encryption_data: Optional[_EncryptionData]) -> bool: - """ - Determine whether the given encryption data signifies version 2.0. - - :param Optional[_EncryptionData] encryption_data: The encryption data. Will return False if this is None. - """ - # If encryption_data is None, assume no encryption - return encryption_data and encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2 - - -def get_adjusted_upload_size(length: int, encryption_version: str) -> int: - """ - Get the adjusted size of the blob upload which accounts for - extra encryption data (padding OR nonce + tag). - - :param int length: The plaintext data length. - :param str encryption_version: The version of encryption being used. - """ - if encryption_version == _ENCRYPTION_PROTOCOL_V1: - return length + (16 - (length % 16)) - - if encryption_version == _ENCRYPTION_PROTOCOL_V2: - encryption_data_length = _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH - regions = math.ceil(length / _GCM_REGION_DATA_LENGTH) - return length + (regions * encryption_data_length) - - raise ValueError("Invalid encryption version specified.") - - -def get_adjusted_download_range_and_offset( - start: int, - end: int, - length: int, - encryption_data: Optional[_EncryptionData]) -> Tuple[Tuple[int, int], Tuple[int, int]]: - """ - Gets the new download range and offsets into the decrypted data for - the given user-specified range. The new download range will include all - the data needed to decrypt the user-provided range and will include only - full encryption regions. - - The offsets returned will be the offsets needed to fetch the user-requested - data out of the full decrypted data. The end offset is different based on the - encryption version. For V1, the end offset is offset from the end whereas for - V2, the end offset is the ending index into the stream. - V1: decrypted_data[start_offset : len(decrypted_data) - end_offset] - V2: decrypted_data[start_offset : end_offset] - - :param int start: The user-requested start index. - :param int end: The user-requested end index. - :param int length: The user-requested length. Only used for V1. - :param Optional[_EncryptionData] encryption_data: The encryption data to determine version and sizes. - :return: (new start, new end), (start offset, end offset) - """ - start_offset, end_offset = 0, 0 - if encryption_data is None: - return (start, end), (start_offset, end_offset) - - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - if start is not None: - # Align the start of the range along a 16 byte block - start_offset = start % 16 - start -= start_offset - - # Include an extra 16 bytes for the IV if necessary - # Because of the previous offsetting, start_range will always - # be a multiple of 16. - if start > 0: - start_offset += 16 - start -= 16 - - if length is not None: - # Align the end of the range along a 16 byte block - end_offset = 15 - (end % 16) - end += end_offset - - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - start_offset, end_offset = 0, end - - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - requested_length = end - start - - if start is not None: - # Find which data region the start is in - region_num = start // data_length - # The start of the data region is different from the start of the encryption region - data_start = region_num * data_length - region_start = region_num * region_length - # Offset is based on data region - start_offset = start - data_start - # New start is the start of the encryption region - start = region_start - - if end is not None: - # Find which data region the end is in - region_num = end // data_length - end_offset = start_offset + requested_length + 1 - # New end is the end of the encryption region - end = (region_num * region_length) + region_length - 1 - - return (start, end), (start_offset, end_offset) - - -def parse_encryption_data(metadata: Dict[str, Any]) -> Optional[_EncryptionData]: - """ - Parses the encryption data out of the given blob metadata. If metadata does - not exist or there are parsing errors, this function will just return None. - - :param Dict[str, Any] metadata: The blob metadata parsed from the response. - """ - try: - return _dict_to_encryption_data(loads(metadata['encryptiondata'])) - except: # pylint: disable=bare-except - return None - - -def adjust_blob_size_for_encryption(size: int, encryption_data: Optional[_EncryptionData]) -> int: - """ - Adjusts the given blob size for encryption by subtracting the size of - the encryption data (nonce + tag). This only has an affect for encryption V2. - - :param int size: The original blob size. - :param Optional[_EncryptionData] encryption_data: The encryption data to determine version and sizes. - """ - if is_encryption_v2(encryption_data): - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - - num_regions = math.ceil(size / region_length) - metadata_size = num_regions * (nonce_length + tag_length) - return size - metadata_size - - return size - - -def _generate_encryption_data_dict(kek, cek, iv, version): - ''' - Generates and returns the encryption metadata as a dict. - - :param object kek: The key encryption key. See calling functions for more information. - :param bytes cek: The content encryption key. - :param Optional[bytes] iv: The initialization vector. Only required for AES-CBC. - :param str version: The client encryption version used. - :return: A dict containing all the encryption metadata. - :rtype: dict - ''' - # Encrypt the cek. - if version == _ENCRYPTION_PROTOCOL_V1: - wrapped_cek = kek.wrap_key(cek) - # For V2, we include the encryption version in the wrapped key. - elif version == _ENCRYPTION_PROTOCOL_V2: - # We must pad the version to 8 bytes for AES Keywrap algorithms - to_wrap = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b'\0') + cek - wrapped_cek = kek.wrap_key(to_wrap) - - # Build the encryption_data dict. - # Use OrderedDict to comply with Java's ordering requirement. - wrapped_content_key = OrderedDict() - wrapped_content_key['KeyId'] = kek.get_kid() - wrapped_content_key['EncryptedKey'] = encode_base64(wrapped_cek) - wrapped_content_key['Algorithm'] = kek.get_key_wrap_algorithm() - - encryption_agent = OrderedDict() - encryption_agent['Protocol'] = version - - if version == _ENCRYPTION_PROTOCOL_V1: - encryption_agent['EncryptionAlgorithm'] = _EncryptionAlgorithm.AES_CBC_256 - - elif version == _ENCRYPTION_PROTOCOL_V2: - encryption_agent['EncryptionAlgorithm'] = _EncryptionAlgorithm.AES_GCM_256 - - encrypted_region_info = OrderedDict() - encrypted_region_info['DataLength'] = _GCM_REGION_DATA_LENGTH - encrypted_region_info['NonceLength'] = _GCM_NONCE_LENGTH - - encryption_data_dict = OrderedDict() - encryption_data_dict['WrappedContentKey'] = wrapped_content_key - encryption_data_dict['EncryptionAgent'] = encryption_agent - if version == _ENCRYPTION_PROTOCOL_V1: - encryption_data_dict['ContentEncryptionIV'] = encode_base64(iv) - elif version == _ENCRYPTION_PROTOCOL_V2: - encryption_data_dict['EncryptedRegionInfo'] = encrypted_region_info - encryption_data_dict['KeyWrappingMetadata'] = {'EncryptionLibrary': 'Python ' + VERSION} - - return encryption_data_dict - - -def _dict_to_encryption_data(encryption_data_dict): - ''' - Converts the specified dictionary to an EncryptionData object for - eventual use in decryption. - - :param dict encryption_data_dict: - The dictionary containing the encryption data. - :return: an _EncryptionData object built from the dictionary. - :rtype: _EncryptionData - ''' - try: - protocol = encryption_data_dict['EncryptionAgent']['Protocol'] - if protocol not in [_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2]: - raise ValueError("Unsupported encryption version.") - except KeyError: - raise ValueError("Unsupported encryption version.") - wrapped_content_key = encryption_data_dict['WrappedContentKey'] - wrapped_content_key = _WrappedContentKey(wrapped_content_key['Algorithm'], - decode_base64_to_bytes(wrapped_content_key['EncryptedKey']), - wrapped_content_key['KeyId']) - - encryption_agent = encryption_data_dict['EncryptionAgent'] - encryption_agent = _EncryptionAgent(encryption_agent['EncryptionAlgorithm'], - encryption_agent['Protocol']) - - if 'KeyWrappingMetadata' in encryption_data_dict: - key_wrapping_metadata = encryption_data_dict['KeyWrappingMetadata'] - else: - key_wrapping_metadata = None - - # AES-CBC only - encryption_iv = None - if 'ContentEncryptionIV' in encryption_data_dict: - encryption_iv = decode_base64_to_bytes(encryption_data_dict['ContentEncryptionIV']) - - # AES-GCM only - region_info = None - if 'EncryptedRegionInfo' in encryption_data_dict: - encrypted_region_info = encryption_data_dict['EncryptedRegionInfo'] - region_info = _EncryptedRegionInfo(encrypted_region_info['DataLength'], - encrypted_region_info['NonceLength'], - _GCM_TAG_LENGTH) - - encryption_data = _EncryptionData(encryption_iv, - region_info, - encryption_agent, - wrapped_content_key, - key_wrapping_metadata) - - return encryption_data - - -def _generate_AES_CBC_cipher(cek, iv): - ''' - Generates and returns an encryption cipher for AES CBC using the given cek and iv. - - :param bytes[] cek: The content encryption key for the cipher. - :param bytes[] iv: The initialization vector for the cipher. - :return: A cipher for encrypting in AES256 CBC. - :rtype: ~cryptography.hazmat.primitives.ciphers.Cipher - ''' - - backend = default_backend() - algorithm = AES(cek) - mode = CBC(iv) - return Cipher(algorithm, mode, backend) - - -def _validate_and_unwrap_cek(encryption_data, key_encryption_key=None, key_resolver=None): - ''' - Extracts and returns the content_encryption_key stored in the encryption_data object - and performs necessary validation on all parameters. - :param _EncryptionData encryption_data: - The encryption metadata of the retrieved value. - :param obj key_encryption_key: - The key_encryption_key used to unwrap the cek. Please refer to high-level service object - instance variables for more details. - :param func key_resolver: - A function used that, given a key_id, will return a key_encryption_key. Please refer - to high-level service object instance variables for more details. - :return: the content_encryption_key stored in the encryption_data object. - :rtype: bytes[] - ''' - - _validate_not_none('encrypted_key', encryption_data.wrapped_content_key.encrypted_key) - - # Validate we have the right info for the specified version - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - _validate_not_none('content_encryption_IV', encryption_data.content_encryption_IV) - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - _validate_not_none('encrypted_region_info', encryption_data.encrypted_region_info) - else: - raise ValueError('Specified encryption version is not supported.') - - content_encryption_key = None - - # If the resolver exists, give priority to the key it finds. - if key_resolver is not None: - key_encryption_key = key_resolver(encryption_data.wrapped_content_key.key_id) - - _validate_not_none('key_encryption_key', key_encryption_key) - if not hasattr(key_encryption_key, 'get_kid') or not callable(key_encryption_key.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(key_encryption_key, 'unwrap_key') or not callable(key_encryption_key.unwrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'unwrap_key')) - if encryption_data.wrapped_content_key.key_id != key_encryption_key.get_kid(): - raise ValueError('Provided or resolved key-encryption-key does not match the id of key used to encrypt.') - # Will throw an exception if the specified algorithm is not supported. - content_encryption_key = key_encryption_key.unwrap_key(encryption_data.wrapped_content_key.encrypted_key, - encryption_data.wrapped_content_key.algorithm) - - # For V2, the version is included with the cek. We need to validate it - # and remove it from the actual cek. - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - version_2_bytes = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b'\0') - cek_version_bytes = content_encryption_key[:len(version_2_bytes)] - if cek_version_bytes != version_2_bytes: - raise ValueError('The encryption metadata is not valid and may have been modified.') - - # Remove version from the start of the cek. - content_encryption_key = content_encryption_key[len(version_2_bytes):] - - _validate_not_none('content_encryption_key', content_encryption_key) - - return content_encryption_key - - -def _decrypt_message(message, encryption_data, key_encryption_key=None, resolver=None): - ''' - Decrypts the given ciphertext using AES256 in CBC mode with 128 bit padding. - Unwraps the content-encryption-key using the user-provided or resolved key-encryption-key (kek). - Returns the original plaintex. - - :param str message: - The ciphertext to be decrypted. - :param _EncryptionData encryption_data: - The metadata associated with this ciphertext. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - unwrap_key(key, algorithm) - - returns the unwrapped form of the specified symmetric key using the string-specified algorithm. - get_kid() - - returns a string key id for this key-encryption-key. - :param function resolver(kid): - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :return: The decrypted plaintext. - :rtype: str - ''' - _validate_not_none('message', message) - content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, resolver) - - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - if not encryption_data.content_encryption_IV: - raise ValueError("Missing required metadata for decryption.") - - cipher = _generate_AES_CBC_cipher(content_encryption_key, encryption_data.content_encryption_IV) - - # decrypt data - decrypted_data = message - decryptor = cipher.decryptor() - decrypted_data = (decryptor.update(decrypted_data) + decryptor.finalize()) - - # unpad data - unpadder = PKCS7(128).unpadder() - decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize()) - - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - block_info = encryption_data.encrypted_region_info - if not block_info or not block_info.nonce_length: - raise ValueError("Missing required metadata for decryption.") - - nonce_length = encryption_data.encrypted_region_info.nonce_length - - # First bytes are the nonce - nonce = message[:nonce_length] - ciphertext_with_tag = message[nonce_length:] - - aesgcm = AESGCM(content_encryption_key) - decrypted_data = aesgcm.decrypt(nonce, ciphertext_with_tag, None) - - else: - raise ValueError('Specified encryption version is not supported.') - - return decrypted_data - - -def encrypt_blob(blob, key_encryption_key, version): - ''' - Encrypts the given blob using the given encryption protocol version. - Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek). - Returns a json-formatted string containing the encryption metadata. This method should - only be used when a blob is small enough for single shot upload. Encrypting larger blobs - is done as a part of the upload_data_chunks method. - - :param bytes blob: - The blob to be encrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param str version: The client encryption version to use. - :return: A tuple of json-formatted string containing the encryption metadata and the encrypted blob data. - :rtype: (str, bytes) - ''' - - _validate_not_none('blob', blob) - _validate_not_none('key_encryption_key', key_encryption_key) - _validate_key_encryption_key_wrap(key_encryption_key) - - if version == _ENCRYPTION_PROTOCOL_V1: - # AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks - content_encryption_key = os.urandom(32) - initialization_vector = os.urandom(16) - - cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector) - - # PKCS7 with 16 byte blocks ensures compatibility with AES. - padder = PKCS7(128).padder() - padded_data = padder.update(blob) + padder.finalize() - - # Encrypt the data. - encryptor = cipher.encryptor() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - - elif version == _ENCRYPTION_PROTOCOL_V2: - # AES256 GCM uses 256 bit (32 byte) keys and a 12 byte nonce. - content_encryption_key = AESGCM.generate_key(bit_length=256) - initialization_vector = None - - data = BytesIO(blob) - encryption_stream = GCMBlobEncryptionStream(content_encryption_key, data) - - encrypted_data = encryption_stream.read() - - else: - raise ValueError("Invalid encryption version specified.") - - encryption_data = _generate_encryption_data_dict(key_encryption_key, content_encryption_key, - initialization_vector, version) - encryption_data['EncryptionMode'] = 'FullBlob' - - return dumps(encryption_data), encrypted_data - - -def generate_blob_encryption_data(key_encryption_key, version): - ''' - Generates the encryption_metadata for the blob. - - :param object key_encryption_key: - The key-encryption-key used to wrap the cek associate with this blob. - :param str version: The client encryption version to use. - :return: A tuple containing the cek and iv for this blob as well as the - serialized encryption metadata for the blob. - :rtype: (bytes, Optional[bytes], str) - ''' - encryption_data = None - content_encryption_key = None - initialization_vector = None - if key_encryption_key: - _validate_key_encryption_key_wrap(key_encryption_key) - content_encryption_key = os.urandom(32) - # Initialization vector only needed for V1 - if version == _ENCRYPTION_PROTOCOL_V1: - initialization_vector = os.urandom(16) - encryption_data = _generate_encryption_data_dict(key_encryption_key, - content_encryption_key, - initialization_vector, - version) - encryption_data['EncryptionMode'] = 'FullBlob' - encryption_data = dumps(encryption_data) - - return content_encryption_key, initialization_vector, encryption_data - - -def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements - require_encryption, - key_encryption_key, - key_resolver, - content, - start_offset, - end_offset, - response_headers): - """ - Decrypts the given blob contents and returns only the requested range. - - :param bool require_encryption: - Whether the calling blob service requires objects to be decrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param object key_resolver: - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :param bytes content: - The encrypted blob content. - :param int start_offset: - The adjusted offset from the beginning of the *decrypted* content for the caller's data. - :param int end_offset: - The adjusted offset from the end of the *decrypted* content for the caller's data. - :param Dict[str, Any] response_headers: - A dictionary of response headers from the download request. Expected to include the - 'x-ms-meta-encryptiondata' header if the blob was encrypted. - :return: The decrypted blob content. - :rtype: bytes - """ - try: - encryption_data = _dict_to_encryption_data(loads(response_headers['x-ms-meta-encryptiondata'])) - except: # pylint: disable=bare-except - if require_encryption: - raise ValueError( - 'Encryption required, but received data does not contain appropriate metatadata.' + \ - 'Data was either not encrypted or metadata has been lost.') - - return content - - algorithm = encryption_data.encryption_agent.encryption_algorithm - if algorithm not in(_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256): - raise ValueError('Specified encryption algorithm is not supported.') - - version = encryption_data.encryption_agent.protocol - if version not in (_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2): - raise ValueError('Specified encryption version is not supported.') - - content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, key_resolver) - - if version == _ENCRYPTION_PROTOCOL_V1: - blob_type = response_headers['x-ms-blob-type'] - - iv = None - unpad = False - if 'content-range' in response_headers: - content_range = response_headers['content-range'] - # Format: 'bytes x-y/size' - - # Ignore the word 'bytes' - content_range = content_range.split(' ') - - content_range = content_range[1].split('-') - content_range = content_range[1].split('/') - end_range = int(content_range[0]) - blob_size = int(content_range[1]) - - if start_offset >= 16: - iv = content[:16] - content = content[16:] - start_offset -= 16 - else: - iv = encryption_data.content_encryption_IV - - if end_range == blob_size - 1: - unpad = True - else: - unpad = True - iv = encryption_data.content_encryption_IV - - if blob_type == 'PageBlob': - unpad = False - - cipher = _generate_AES_CBC_cipher(content_encryption_key, iv) - decryptor = cipher.decryptor() - - content = decryptor.update(content) + decryptor.finalize() - if unpad: - unpadder = PKCS7(128).unpadder() - content = unpadder.update(content) + unpadder.finalize() - - return content[start_offset: len(content) - end_offset] - - if version == _ENCRYPTION_PROTOCOL_V2: - # We assume the content contains only full encryption regions - total_size = len(content) - offset = 0 - - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - - decrypted_content = bytearray() - while offset < total_size: - # Process one encryption region at a time - process_size = min(region_length, total_size) - encrypted_region = content[offset:offset + process_size] - - # First bytes are the nonce - nonce = encrypted_region[:nonce_length] - ciphertext_with_tag = encrypted_region[nonce_length:] - - aesgcm = AESGCM(content_encryption_key) - decrypted_data = aesgcm.decrypt(nonce, ciphertext_with_tag, None) - decrypted_content.extend(decrypted_data) - - offset += process_size - - # Read the caller requested data from the decrypted content - return decrypted_content[start_offset:end_offset] - - -def get_blob_encryptor_and_padder(cek, iv, should_pad): - encryptor = None - padder = None - - if cek is not None and iv is not None: - cipher = _generate_AES_CBC_cipher(cek, iv) - encryptor = cipher.encryptor() - padder = PKCS7(128).padder() if should_pad else None - - return encryptor, padder - - -def encrypt_queue_message(message, key_encryption_key, version): - ''' - Encrypts the given plain text message using the given protocol version. - Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek). - Returns a json-formatted string containing the encrypted message and the encryption metadata. - - :param object message: - The plain text messge to be encrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param str version: The client encryption version to use. - :return: A json-formatted string containing the encrypted message and the encryption metadata. - :rtype: str - ''' - - _validate_not_none('message', message) - _validate_not_none('key_encryption_key', key_encryption_key) - _validate_key_encryption_key_wrap(key_encryption_key) - - # Queue encoding functions all return unicode strings, and encryption should - # operate on binary strings. - message = message.encode('utf-8') - - if version == _ENCRYPTION_PROTOCOL_V1: - # AES256 CBC uses 256 bit (32 byte) keys and always with 16 byte blocks - content_encryption_key = os.urandom(32) - initialization_vector = os.urandom(16) - - cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector) - - # PKCS7 with 16 byte blocks ensures compatibility with AES. - padder = PKCS7(128).padder() - padded_data = padder.update(message) + padder.finalize() - - # Encrypt the data. - encryptor = cipher.encryptor() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - - elif version == _ENCRYPTION_PROTOCOL_V2: - # AES256 GCM uses 256 bit (32 byte) keys and a 12 byte nonce. - content_encryption_key = AESGCM.generate_key(bit_length=256) - initialization_vector = None - - # The nonce MUST be different for each key - nonce = os.urandom(12) - aesgcm = AESGCM(content_encryption_key) - - # Returns ciphertext + tag - cipertext_with_tag = aesgcm.encrypt(nonce, message, None) - encrypted_data = nonce + cipertext_with_tag - - else: - raise ValueError("Invalid encryption version specified.") - - # Build the dictionary structure. - queue_message = {'EncryptedMessageContents': encode_base64(encrypted_data), - 'EncryptionData': _generate_encryption_data_dict(key_encryption_key, - content_encryption_key, - initialization_vector, - version)} - - return dumps(queue_message) - - -def decrypt_queue_message(message, response, require_encryption, key_encryption_key, resolver): - ''' - Returns the decrypted message contents from an EncryptedQueueMessage. - If no encryption metadata is present, will return the unaltered message. - :param str message: - The JSON formatted QueueEncryptedMessage contents with all associated metadata. - :param bool require_encryption: - If set, will enforce that the retrieved messages are encrypted and decrypt them. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - unwrap_key(key, algorithm) - - returns the unwrapped form of the specified symmetric key usingthe string-specified algorithm. - get_kid() - - returns a string key id for this key-encryption-key. - :param function resolver(kid): - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :return: The plain text message from the queue message. - :rtype: str - ''' - response = response.http_response - - try: - message = loads(message) - - encryption_data = _dict_to_encryption_data(message['EncryptionData']) - decoded_data = decode_base64_to_bytes(message['EncryptedMessageContents']) - except (KeyError, ValueError): - # Message was not json formatted and so was not encrypted - # or the user provided a json formatted message - # or the metadata was malformed. - if require_encryption: - raise ValueError( - 'Encryption required, but received message does not contain appropriate metatadata. ' + \ - 'Message was either not encrypted or metadata was incorrect.') - - return message - try: - return _decrypt_message(decoded_data, encryption_data, key_encryption_key, resolver).decode('utf-8') - except Exception as error: - raise HttpResponseError( - message="Decryption failed.", - response=response, - error=error) diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads.py index 167217734bea..279f084ff970 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads.py @@ -6,19 +6,17 @@ # pylint: disable=no-self-use from concurrent import futures -from io import (BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation) -from threading import Lock +from io import BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation from itertools import islice from math import ceil +from threading import Lock import six - from azure.core.tracing.common import with_current_context from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 @@ -52,18 +50,9 @@ def upload_data_chunks( max_concurrency=None, stream=None, validate_content=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -149,7 +138,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = Lock() if parallel else None # Progress feedback diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads_async.py index 2d8376aff237..97be2caf89f8 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/uploads_async.py @@ -6,10 +6,9 @@ # pylint: disable=no-self-use import asyncio +import threading from asyncio import Lock from itertools import islice -import threading - from math import ceil import six @@ -17,14 +16,9 @@ from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder from .uploads import SubStream, IterStreamer # pylint: disable=unused-import -_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 -_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.' - - async def _parallel_uploads(uploader, pending, running): range_ids = [] while True: @@ -52,18 +46,9 @@ async def upload_data_chunks( chunk_size=None, max_concurrency=None, stream=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -152,7 +137,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = threading.Lock() if parallel else None # Progress feedback @@ -217,7 +201,7 @@ async def _update_progress(self, length): self.progress_total += length if self.progress_hook: - self.progress_hook(self.progress_total, self.total_size) + await self.progress_hook(self.progress_total, self.total_size) async def _upload_chunk(self, chunk_offset, chunk_data): raise NotImplementedError("Must be implemented by child class.") diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_directory_client_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_directory_client_async.py index 586571e2d7b5..6bf917208e64 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_directory_client_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_directory_client_async.py @@ -314,8 +314,7 @@ async def rename_directory( '{}://{}'.format(self.scheme, self.primary_hostname), self.share_name, new_dir_path, credential=new_dir_sas or self.credential, api_version=self.api_version, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, key_resolver_function=self.key_resolver_function + _location_mode=self._location_mode ) kwargs.update(get_rename_smb_properties(kwargs)) diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py index 971f12e8859b..eeba97261e34 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py @@ -6,41 +6,25 @@ # pylint: disable=invalid-overridden-method import asyncio import sys +import warnings from io import BytesIO from itertools import islice -import warnings - from typing import AsyncIterator + from azure.core.exceptions import HttpResponseError, ResourceModifiedError -from .._shared.encryption import decrypt_blob +from .._download import _ChunkDownloader from .._shared.request_handlers import validate_and_format_range_headers from .._shared.response_handlers import process_storage_error, parse_length_from_content_range -from .._download import process_range_and_offset, _ChunkDownloader -async def process_content(data, start_offset, end_offset, encryption): +async def process_content(data): if data is None: raise ValueError("Response cannot be None.") + try: - content = data.response.body() + return data.response.body() except Exception as error: raise HttpResponseError(message="Download stream interrupted.", response=data.response, error=error) - if encryption.get('key') is not None or encryption.get('resolver') is not None: - try: - return decrypt_blob( - encryption.get('required'), - encryption.get('key'), - encryption.get('resolver'), - content, - start_offset, - end_offset, - data.response.headers) - except Exception as error: - raise HttpResponseError( - message="Decryption failed.", - response=data.response, - error=error) - return content class _AsyncChunkDownloader(_ChunkDownloader): @@ -77,12 +61,9 @@ async def _write_to_stream(self, chunk_data, chunk_start): self.stream.write(chunk_data) async def _download_chunk(self, chunk_start, chunk_end): - download_range, offset = process_range_and_offset( - chunk_start, chunk_end, chunk_end, self.encryption_options - ) range_header, range_validation = validate_and_format_range_headers( - download_range[0], - download_range[1], + chunk_start, + chunk_end, check_content_md5=self.validate_content ) try: @@ -99,7 +80,7 @@ async def _download_chunk(self, chunk_start, chunk_end): except HttpResponseError as error: process_storage_error(error) - chunk_data = await process_content(response, offset[0], offset[1], self.encryption_options) + chunk_data = await process_content(response) return chunk_data @@ -183,7 +164,6 @@ def __init__( start_range=None, end_range=None, validate_content=None, - encryption_options=None, max_concurrency=1, name=None, path=None, @@ -204,7 +184,6 @@ def __init__( self._max_concurrency = max_concurrency self._encoding = encoding self._validate_content = validate_content - self._encryption_options = encryption_options or {} self._request_options = kwargs self._location_mode = None self._download_complete = False @@ -224,9 +203,7 @@ def __init__( else: initial_request_end = initial_request_start + self._first_get_size - 1 - self._initial_range, self._initial_offset = process_range_and_offset( - initial_request_start, initial_request_end, self._end_range, self._encryption_options - ) + self._initial_range = (initial_request_start, initial_request_end) def __len__(self): return self.size @@ -257,12 +234,7 @@ async def _setup(self): if self.size == 0: self._current_content = b"" else: - self._current_content = await process_content( - self._response, - self._initial_offset[0], - self._initial_offset[1], - self._encryption_options - ) + self._current_content = await process_content(self._response) async def _initial_request(self): range_header, range_validation = validate_and_format_range_headers( @@ -346,7 +318,6 @@ def chunks(self): stream=None, parallel=False, validate_content=self._validate_content, - encryption_options=self._encryption_options, use_location=self._location_mode, etag=self._etag, **self._request_options) @@ -447,7 +418,6 @@ async def readinto(self, stream): stream=stream, parallel=parallel, validate_content=self._validate_content, - encryption_options=self._encryption_options, use_location=self._location_mode, etag=self._etag, **self._request_options) diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py index 9bc8fc00ac1f..50d5d558a4d9 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_file_client_async.py @@ -255,9 +255,6 @@ async def create_file( # type: ignore content_settings = kwargs.pop('content_settings', None) metadata = kwargs.pop('metadata', None) timeout = kwargs.pop('timeout', None) - if self.require_encryption and not self.key_encryption_key: - raise ValueError("Encryption required but no key was provided.") - headers = kwargs.pop("headers", {}) headers.update(add_metadata_headers(metadata)) file_http_headers = None @@ -383,8 +380,6 @@ async def upload_file( validate_content = kwargs.pop('validate_content', False) timeout = kwargs.pop('timeout', None) encoding = kwargs.pop('encoding', 'UTF-8') - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if isinstance(data, six.text_type): data = data.encode(encoding) @@ -632,8 +627,6 @@ async def download_file( :dedent: 16 :caption: Download a file. """ - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if length is not None and offset is None: raise ValueError("Offset value must not be None if length is set.") @@ -648,7 +641,6 @@ async def download_file( config=self._config, start_range=offset, end_range=range_end, - encryption_options=None, name=self.file_name, path='/'.join(self.file_path), share=self.share_name, @@ -774,8 +766,7 @@ async def rename_file( '{}://{}'.format(self.scheme, self.primary_hostname), self.share_name, new_file_path, credential=new_file_sas or self.credential, api_version=self.api_version, _hosts=self._hosts, _configuration=self._config, _pipeline=self._pipeline, - _location_mode=self._location_mode, require_encryption=self.require_encryption, - key_encryption_key=self.key_encryption_key, key_resolver_function=self.key_resolver_function + _location_mode=self._location_mode ) kwargs.update(get_rename_smb_properties(kwargs)) @@ -1024,8 +1015,6 @@ async def upload_range( # type: ignore timeout = kwargs.pop('timeout', None) encoding = kwargs.pop('encoding', 'UTF-8') file_last_write_mode = kwargs.pop('file_last_write_mode', None) - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Encryption not supported.") if isinstance(data, six.text_type): data = data.encode(encoding) end_range = offset + length - 1 # Reformat to an inclusive range index @@ -1238,8 +1227,6 @@ async def clear_range( # type: ignore """ access_conditions = get_access_conditions(kwargs.pop('lease', None)) timeout = kwargs.pop('timeout', None) - if self.require_encryption or (self.key_encryption_key is not None): - raise ValueError("Unsupported method for encryption.") if offset is None or offset % 512 != 0: raise ValueError("offset must be an integer that aligns with 512 bytes file size") diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/encryption.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py similarity index 97% rename from sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/encryption.py rename to sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py index 0e46796b2ff3..af063083877d 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/encryption.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py @@ -7,6 +7,7 @@ import os import math import sys +import warnings from collections import OrderedDict from io import BytesIO from json import ( @@ -24,8 +25,8 @@ from azure.core.exceptions import HttpResponseError -from .._version import VERSION -from . import encode_base64, decode_base64_to_bytes +from ._version import VERSION +from ._shared import encode_base64, decode_base64_to_bytes _ENCRYPTION_PROTOCOL_V1 = '1.0' @@ -53,6 +54,19 @@ def _validate_key_encryption_key_wrap(kek): raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) +class StorageEncryptionMixin(object): + def configure_encryption(self, kwargs): + self.require_encryption = kwargs.get("require_encryption", False) + self.encryption_version = kwargs.get("encryption_version", "1.0") + self.key_encryption_key = kwargs.get("key_encryption_key") + self.key_resolver_function = kwargs.get("key_resolver_function") + if self.key_encryption_key and self.encryption_version == '1.0': + warnings.warn("This client has been configured to use encryption with version 1.0. " + + "Version 1.0 is deprecated and no longer considered secure. It is highly " + + "recommended that you switch to using version 2.0. The version can be " + + "specified using the 'encryption_version' keyword.") + + class _EncryptionAlgorithm(object): ''' Specifies which client encryption algorithm is used. diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_message_encoding.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_message_encoding.py index d429f7a86c6d..4e1b595e8aad 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_message_encoding.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_message_encoding.py @@ -5,13 +5,13 @@ # -------------------------------------------------------------------------- # pylint: disable=unused-argument +import sys from base64 import b64encode, b64decode -import sys import six from azure.core.exceptions import DecodeError -from ._shared.encryption import decrypt_queue_message, encrypt_queue_message, _ENCRYPTION_PROTOCOL_V1 +from ._encryption import decrypt_queue_message, encrypt_queue_message, _ENCRYPTION_PROTOCOL_V1 class MessageEncodePolicy(object): diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_client.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_client.py index 3d40078a432d..8916981a6287 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_client.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_client.py @@ -7,38 +7,34 @@ import functools import warnings from typing import ( # pylint: disable=unused-import - Optional, Any, Dict, List, + Any, Dict, List, Optional, TYPE_CHECKING) -try: - from urllib.parse import urlparse, quote, unquote -except ImportError: - from urlparse import urlparse # type: ignore - from urllib2 import quote, unquote # type: ignore +from urllib.parse import urlparse, quote, unquote import six - from azure.core.exceptions import HttpResponseError from azure.core.paging import ItemPaged from azure.core.tracing.decorator import distributed_trace -from ._serialize import get_api_version + from ._shared.base_client import StorageAccountHostsMixin, parse_connection_str, parse_query from ._shared.request_handlers import add_metadata_headers, serialize_iso from ._shared.response_handlers import ( process_storage_error, return_response_headers, return_headers_and_deserialized) -from ._message_encoding import NoEncodePolicy, NoDecodePolicy -from ._deserialize import deserialize_queue_properties, deserialize_queue_creation from ._generated import AzureQueueStorage -from ._generated.models import SignedIdentifier -from ._generated.models import QueueMessage as GenQueueMessage +from ._generated.models import SignedIdentifier, QueueMessage as GenQueueMessage +from ._deserialize import deserialize_queue_properties, deserialize_queue_creation +from ._encryption import StorageEncryptionMixin +from ._message_encoding import NoEncodePolicy, NoDecodePolicy from ._models import QueueMessage, AccessPolicy, MessagesPaged +from ._serialize import get_api_version if TYPE_CHECKING: from ._models import QueueProperties -class QueueClient(StorageAccountHostsMixin): +class QueueClient(StorageAccountHostsMixin, StorageEncryptionMixin): """A client to interact with a specific Queue. For more optional configuration, please click @@ -106,6 +102,7 @@ def __init__( self._config.message_decode_policy = kwargs.get('message_decode_policy', None) or NoDecodePolicy() self._client = AzureQueueStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) def _format_url(self, hostname): """Format the endpoint URL according to the current location diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_service_client.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_service_client.py index e58b348c5e39..dfc48abff77d 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_service_client.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_queue_service_client.py @@ -6,45 +6,39 @@ import functools from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, + Any, Dict, List, Optional, Union, TYPE_CHECKING) -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse # type: ignore +from urllib.parse import urlparse from azure.core.exceptions import HttpResponseError from azure.core.paging import ItemPaged from azure.core.pipeline import Pipeline from azure.core.tracing.decorator import distributed_trace -from ._serialize import get_api_version -from ._shared.models import LocationMode + from ._shared.base_client import StorageAccountHostsMixin, TransportWrapper, parse_connection_str, parse_query +from ._shared.models import LocationMode from ._shared.response_handlers import process_storage_error from ._generated import AzureQueueStorage from ._generated.models import StorageServiceProperties - +from ._encryption import StorageEncryptionMixin from ._models import ( QueuePropertiesPaged, service_stats_deserialize, service_properties_deserialize, ) - +from ._serialize import get_api_version from ._queue_client import QueueClient if TYPE_CHECKING: - from datetime import datetime - from azure.core.configuration import Configuration - from azure.core.pipeline.policies import HTTPPolicy from ._models import ( + CorsRule, + Metrics, QueueProperties, QueueAnalyticsLogging, - Metrics, - CorsRule, ) -class QueueServiceClient(StorageAccountHostsMixin): +class QueueServiceClient(StorageAccountHostsMixin, StorageEncryptionMixin): """A client to interact with the Queue Service at the account level. This client provides operations to retrieve and configure the account properties @@ -110,6 +104,7 @@ def __init__( super(QueueServiceClient, self).__init__(parsed_url, service='queue', credential=credential, **kwargs) self._client = AzureQueueStorage(self.url, base_url=self.url, pipeline=self._pipeline) self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access + self.configure_encryption(kwargs) def _format_url(self, hostname): """Format the endpoint URL according to the current location diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py index 7b56dba2507d..3af10b496f22 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/base_client.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- import logging import uuid -import warnings from typing import ( # pylint: disable=unused-import Optional, Any, @@ -104,16 +103,6 @@ def __init__( primary_hostname = (parsed_url.netloc + parsed_url.path).rstrip('/') self._hosts = {LocationMode.PRIMARY: primary_hostname, LocationMode.SECONDARY: secondary_hostname} - self.require_encryption = kwargs.get("require_encryption", False) - self.encryption_version = kwargs.get("encryption_version", "1.0") - self.key_encryption_key = kwargs.get("key_encryption_key") - self.key_resolver_function = kwargs.get("key_resolver_function") - if self.key_encryption_key and self.encryption_version == '1.0': - warnings.warn("This client has been configured to use encryption with version 1.0. \ - Version 1.0 is deprecated and no longer considered secure. It is highly \ - recommended that you switch to using version 2.0. The version can be \ - specified using the 'encryption_version' keyword.") - self._config, self._pipeline = self._create_pipeline(self.credential, storage_sdk=service, **kwargs) def __enter__(self): diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/encryption.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/encryption.py deleted file mode 100644 index 0e46796b2ff3..000000000000 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/encryption.py +++ /dev/null @@ -1,965 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import os -import math -import sys -from collections import OrderedDict -from io import BytesIO -from json import ( - dumps, - loads, -) -from typing import Any, BinaryIO, Dict, Optional, Tuple - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.ciphers import Cipher -from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.hazmat.primitives.ciphers.algorithms import AES -from cryptography.hazmat.primitives.ciphers.modes import CBC -from cryptography.hazmat.primitives.padding import PKCS7 - -from azure.core.exceptions import HttpResponseError - -from .._version import VERSION -from . import encode_base64, decode_base64_to_bytes - - -_ENCRYPTION_PROTOCOL_V1 = '1.0' -_ENCRYPTION_PROTOCOL_V2 = '2.0' -_GCM_REGION_DATA_LENGTH = 4 * 1024 * 1024 -_GCM_NONCE_LENGTH = 12 -_GCM_TAG_LENGTH = 16 - -_ERROR_OBJECT_INVALID = \ - '{0} does not define a complete interface. Value of {1} is either missing or invalid.' - - -def _validate_not_none(param_name, param): - if param is None: - raise ValueError('{0} should not be None.'.format(param_name)) - - -def _validate_key_encryption_key_wrap(kek): - # Note that None is not callable and so will fail the second clause of each check. - if not hasattr(kek, 'wrap_key') or not callable(kek.wrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'wrap_key')) - if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(kek, 'get_key_wrap_algorithm') or not callable(kek.get_key_wrap_algorithm): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) - - -class _EncryptionAlgorithm(object): - ''' - Specifies which client encryption algorithm is used. - ''' - AES_CBC_256 = 'AES_CBC_256' - AES_GCM_256 = 'AES_GCM_256' - - -class _WrappedContentKey: - ''' - Represents the envelope key details stored on the service. - ''' - - def __init__(self, algorithm, encrypted_key, key_id): - ''' - :param str algorithm: - The algorithm used for wrapping. - :param bytes encrypted_key: - The encrypted content-encryption-key. - :param str key_id: - The key-encryption-key identifier string. - ''' - - _validate_not_none('algorithm', algorithm) - _validate_not_none('encrypted_key', encrypted_key) - _validate_not_none('key_id', key_id) - - self.algorithm = algorithm - self.encrypted_key = encrypted_key - self.key_id = key_id - - -class _EncryptedRegionInfo: - ''' - Represents the length of encryption elements. - This is only used for Encryption V2. - ''' - - def __init__(self, data_length, nonce_length, tag_length): - ''' - :param int data_length: - The length of the encryption region data (not including nonce + tag). - :param str nonce_length: - The length of nonce used when encrypting. - :param int tag_length: - The length of the encryption tag. - ''' - _validate_not_none('data_length', data_length) - _validate_not_none('nonce_length', nonce_length) - _validate_not_none('tag_length', tag_length) - - self.data_length = data_length - self.nonce_length = nonce_length - self.tag_length = tag_length - - -class _EncryptionAgent: - ''' - Represents the encryption agent stored on the service. - It consists of the encryption protocol version and encryption algorithm used. - ''' - - def __init__(self, encryption_algorithm, protocol): - ''' - :param _EncryptionAlgorithm encryption_algorithm: - The algorithm used for encrypting the message contents. - :param str protocol: - The protocol version used for encryption. - ''' - - _validate_not_none('encryption_algorithm', encryption_algorithm) - _validate_not_none('protocol', protocol) - - self.encryption_algorithm = str(encryption_algorithm) - self.protocol = protocol - - -class _EncryptionData: - ''' - Represents the encryption data that is stored on the service. - ''' - - def __init__( - self, - content_encryption_IV, - encrypted_region_info, - encryption_agent, - wrapped_content_key, - key_wrapping_metadata): - ''' - :param Optional[bytes] content_encryption_IV: - The content encryption initialization vector. - Required for AES-CBC (V1). - :param Optional[_EncryptedRegionInfo] encrypted_region_info: - The info about the autenticated block sizes. - Required for AES-GCM (V2). - :param _EncryptionAgent encryption_agent: - The encryption agent. - :param _WrappedContentKey wrapped_content_key: - An object that stores the wrapping algorithm, the key identifier, - and the encrypted key bytes. - :param dict key_wrapping_metadata: - A dict containing metadata related to the key wrapping. - ''' - - _validate_not_none('encryption_agent', encryption_agent) - _validate_not_none('wrapped_content_key', wrapped_content_key) - - # Validate we have the right matching optional parameter for the specified algorithm - if encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256: - _validate_not_none('content_encryption_IV', content_encryption_IV) - elif encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_GCM_256: - _validate_not_none('encrypted_region_info', encrypted_region_info) - else: - raise ValueError("Invalid encryption algorithm.") - - self.content_encryption_IV = content_encryption_IV - self.encrypted_region_info = encrypted_region_info - self.encryption_agent = encryption_agent - self.wrapped_content_key = wrapped_content_key - self.key_wrapping_metadata = key_wrapping_metadata - - -class GCMBlobEncryptionStream: - """ - A stream that performs AES-GCM encryption on the given data as - it's streamed. Data is read and encrypted in regions. The stream - will use the same encryption key and will generate a guaranteed unique - nonce for each encryption region. - """ - def __init__( - self, - content_encryption_key: bytes, - data_stream: BinaryIO, - ): - """ - :param bytes content_encryption_key: The encryption key to use. - :param BinaryIO data_stream: The data stream to read data from. - """ - self.content_encryption_key = content_encryption_key - self.data_stream = data_stream - - self.offset = 0 - self.current = b'' - self.nonce_counter = 0 - - def read(self, size: int = -1) -> bytes: - """ - Read data from the stream. Specify -1 to read all available data. - - :param int size: The amount of data to read. Defaults to -1 for all data. - """ - result = BytesIO() - remaining = sys.maxsize if size == -1 else size - - while remaining > 0: - # Start by reading from current - if len(self.current) > 0: - read = min(remaining, len(self.current)) - result.write(self.current[:read]) - - self.current = self.current[read:] - self.offset += read - remaining -= read - - if remaining > 0: - # Read one region of data and encrypt it - data = self.data_stream.read(_GCM_REGION_DATA_LENGTH) - if len(data) == 0: - # No more data to read - break - - self.current = self._encrypt_region(data) - - return result.getvalue() - - def _encrypt_region(self, data: bytes) -> bytes: - """ - Encrypt the given region of data using AES-GCM. The result - includes the data in the form: nonce + ciphertext + tag. - - :param bytes data: The data to encrypt. - """ - # Each region MUST use a different nonce - nonce = self.nonce_counter.to_bytes(_GCM_NONCE_LENGTH, 'big') - self.nonce_counter += 1 - - aesgcm = AESGCM(self.content_encryption_key) - - # Returns ciphertext + tag - cipertext_with_tag = aesgcm.encrypt(nonce, data, None) - return nonce + cipertext_with_tag - - -def is_encryption_v2(encryption_data: Optional[_EncryptionData]) -> bool: - """ - Determine whether the given encryption data signifies version 2.0. - - :param Optional[_EncryptionData] encryption_data: The encryption data. Will return False if this is None. - """ - # If encryption_data is None, assume no encryption - return encryption_data and encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2 - - -def get_adjusted_upload_size(length: int, encryption_version: str) -> int: - """ - Get the adjusted size of the blob upload which accounts for - extra encryption data (padding OR nonce + tag). - - :param int length: The plaintext data length. - :param str encryption_version: The version of encryption being used. - """ - if encryption_version == _ENCRYPTION_PROTOCOL_V1: - return length + (16 - (length % 16)) - - if encryption_version == _ENCRYPTION_PROTOCOL_V2: - encryption_data_length = _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH - regions = math.ceil(length / _GCM_REGION_DATA_LENGTH) - return length + (regions * encryption_data_length) - - raise ValueError("Invalid encryption version specified.") - - -def get_adjusted_download_range_and_offset( - start: int, - end: int, - length: int, - encryption_data: Optional[_EncryptionData]) -> Tuple[Tuple[int, int], Tuple[int, int]]: - """ - Gets the new download range and offsets into the decrypted data for - the given user-specified range. The new download range will include all - the data needed to decrypt the user-provided range and will include only - full encryption regions. - - The offsets returned will be the offsets needed to fetch the user-requested - data out of the full decrypted data. The end offset is different based on the - encryption version. For V1, the end offset is offset from the end whereas for - V2, the end offset is the ending index into the stream. - V1: decrypted_data[start_offset : len(decrypted_data) - end_offset] - V2: decrypted_data[start_offset : end_offset] - - :param int start: The user-requested start index. - :param int end: The user-requested end index. - :param int length: The user-requested length. Only used for V1. - :param Optional[_EncryptionData] encryption_data: The encryption data to determine version and sizes. - :return: (new start, new end), (start offset, end offset) - """ - start_offset, end_offset = 0, 0 - if encryption_data is None: - return (start, end), (start_offset, end_offset) - - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - if start is not None: - # Align the start of the range along a 16 byte block - start_offset = start % 16 - start -= start_offset - - # Include an extra 16 bytes for the IV if necessary - # Because of the previous offsetting, start_range will always - # be a multiple of 16. - if start > 0: - start_offset += 16 - start -= 16 - - if length is not None: - # Align the end of the range along a 16 byte block - end_offset = 15 - (end % 16) - end += end_offset - - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - start_offset, end_offset = 0, end - - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - requested_length = end - start - - if start is not None: - # Find which data region the start is in - region_num = start // data_length - # The start of the data region is different from the start of the encryption region - data_start = region_num * data_length - region_start = region_num * region_length - # Offset is based on data region - start_offset = start - data_start - # New start is the start of the encryption region - start = region_start - - if end is not None: - # Find which data region the end is in - region_num = end // data_length - end_offset = start_offset + requested_length + 1 - # New end is the end of the encryption region - end = (region_num * region_length) + region_length - 1 - - return (start, end), (start_offset, end_offset) - - -def parse_encryption_data(metadata: Dict[str, Any]) -> Optional[_EncryptionData]: - """ - Parses the encryption data out of the given blob metadata. If metadata does - not exist or there are parsing errors, this function will just return None. - - :param Dict[str, Any] metadata: The blob metadata parsed from the response. - """ - try: - return _dict_to_encryption_data(loads(metadata['encryptiondata'])) - except: # pylint: disable=bare-except - return None - - -def adjust_blob_size_for_encryption(size: int, encryption_data: Optional[_EncryptionData]) -> int: - """ - Adjusts the given blob size for encryption by subtracting the size of - the encryption data (nonce + tag). This only has an affect for encryption V2. - - :param int size: The original blob size. - :param Optional[_EncryptionData] encryption_data: The encryption data to determine version and sizes. - """ - if is_encryption_v2(encryption_data): - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - - num_regions = math.ceil(size / region_length) - metadata_size = num_regions * (nonce_length + tag_length) - return size - metadata_size - - return size - - -def _generate_encryption_data_dict(kek, cek, iv, version): - ''' - Generates and returns the encryption metadata as a dict. - - :param object kek: The key encryption key. See calling functions for more information. - :param bytes cek: The content encryption key. - :param Optional[bytes] iv: The initialization vector. Only required for AES-CBC. - :param str version: The client encryption version used. - :return: A dict containing all the encryption metadata. - :rtype: dict - ''' - # Encrypt the cek. - if version == _ENCRYPTION_PROTOCOL_V1: - wrapped_cek = kek.wrap_key(cek) - # For V2, we include the encryption version in the wrapped key. - elif version == _ENCRYPTION_PROTOCOL_V2: - # We must pad the version to 8 bytes for AES Keywrap algorithms - to_wrap = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b'\0') + cek - wrapped_cek = kek.wrap_key(to_wrap) - - # Build the encryption_data dict. - # Use OrderedDict to comply with Java's ordering requirement. - wrapped_content_key = OrderedDict() - wrapped_content_key['KeyId'] = kek.get_kid() - wrapped_content_key['EncryptedKey'] = encode_base64(wrapped_cek) - wrapped_content_key['Algorithm'] = kek.get_key_wrap_algorithm() - - encryption_agent = OrderedDict() - encryption_agent['Protocol'] = version - - if version == _ENCRYPTION_PROTOCOL_V1: - encryption_agent['EncryptionAlgorithm'] = _EncryptionAlgorithm.AES_CBC_256 - - elif version == _ENCRYPTION_PROTOCOL_V2: - encryption_agent['EncryptionAlgorithm'] = _EncryptionAlgorithm.AES_GCM_256 - - encrypted_region_info = OrderedDict() - encrypted_region_info['DataLength'] = _GCM_REGION_DATA_LENGTH - encrypted_region_info['NonceLength'] = _GCM_NONCE_LENGTH - - encryption_data_dict = OrderedDict() - encryption_data_dict['WrappedContentKey'] = wrapped_content_key - encryption_data_dict['EncryptionAgent'] = encryption_agent - if version == _ENCRYPTION_PROTOCOL_V1: - encryption_data_dict['ContentEncryptionIV'] = encode_base64(iv) - elif version == _ENCRYPTION_PROTOCOL_V2: - encryption_data_dict['EncryptedRegionInfo'] = encrypted_region_info - encryption_data_dict['KeyWrappingMetadata'] = {'EncryptionLibrary': 'Python ' + VERSION} - - return encryption_data_dict - - -def _dict_to_encryption_data(encryption_data_dict): - ''' - Converts the specified dictionary to an EncryptionData object for - eventual use in decryption. - - :param dict encryption_data_dict: - The dictionary containing the encryption data. - :return: an _EncryptionData object built from the dictionary. - :rtype: _EncryptionData - ''' - try: - protocol = encryption_data_dict['EncryptionAgent']['Protocol'] - if protocol not in [_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2]: - raise ValueError("Unsupported encryption version.") - except KeyError: - raise ValueError("Unsupported encryption version.") - wrapped_content_key = encryption_data_dict['WrappedContentKey'] - wrapped_content_key = _WrappedContentKey(wrapped_content_key['Algorithm'], - decode_base64_to_bytes(wrapped_content_key['EncryptedKey']), - wrapped_content_key['KeyId']) - - encryption_agent = encryption_data_dict['EncryptionAgent'] - encryption_agent = _EncryptionAgent(encryption_agent['EncryptionAlgorithm'], - encryption_agent['Protocol']) - - if 'KeyWrappingMetadata' in encryption_data_dict: - key_wrapping_metadata = encryption_data_dict['KeyWrappingMetadata'] - else: - key_wrapping_metadata = None - - # AES-CBC only - encryption_iv = None - if 'ContentEncryptionIV' in encryption_data_dict: - encryption_iv = decode_base64_to_bytes(encryption_data_dict['ContentEncryptionIV']) - - # AES-GCM only - region_info = None - if 'EncryptedRegionInfo' in encryption_data_dict: - encrypted_region_info = encryption_data_dict['EncryptedRegionInfo'] - region_info = _EncryptedRegionInfo(encrypted_region_info['DataLength'], - encrypted_region_info['NonceLength'], - _GCM_TAG_LENGTH) - - encryption_data = _EncryptionData(encryption_iv, - region_info, - encryption_agent, - wrapped_content_key, - key_wrapping_metadata) - - return encryption_data - - -def _generate_AES_CBC_cipher(cek, iv): - ''' - Generates and returns an encryption cipher for AES CBC using the given cek and iv. - - :param bytes[] cek: The content encryption key for the cipher. - :param bytes[] iv: The initialization vector for the cipher. - :return: A cipher for encrypting in AES256 CBC. - :rtype: ~cryptography.hazmat.primitives.ciphers.Cipher - ''' - - backend = default_backend() - algorithm = AES(cek) - mode = CBC(iv) - return Cipher(algorithm, mode, backend) - - -def _validate_and_unwrap_cek(encryption_data, key_encryption_key=None, key_resolver=None): - ''' - Extracts and returns the content_encryption_key stored in the encryption_data object - and performs necessary validation on all parameters. - :param _EncryptionData encryption_data: - The encryption metadata of the retrieved value. - :param obj key_encryption_key: - The key_encryption_key used to unwrap the cek. Please refer to high-level service object - instance variables for more details. - :param func key_resolver: - A function used that, given a key_id, will return a key_encryption_key. Please refer - to high-level service object instance variables for more details. - :return: the content_encryption_key stored in the encryption_data object. - :rtype: bytes[] - ''' - - _validate_not_none('encrypted_key', encryption_data.wrapped_content_key.encrypted_key) - - # Validate we have the right info for the specified version - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - _validate_not_none('content_encryption_IV', encryption_data.content_encryption_IV) - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - _validate_not_none('encrypted_region_info', encryption_data.encrypted_region_info) - else: - raise ValueError('Specified encryption version is not supported.') - - content_encryption_key = None - - # If the resolver exists, give priority to the key it finds. - if key_resolver is not None: - key_encryption_key = key_resolver(encryption_data.wrapped_content_key.key_id) - - _validate_not_none('key_encryption_key', key_encryption_key) - if not hasattr(key_encryption_key, 'get_kid') or not callable(key_encryption_key.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(key_encryption_key, 'unwrap_key') or not callable(key_encryption_key.unwrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'unwrap_key')) - if encryption_data.wrapped_content_key.key_id != key_encryption_key.get_kid(): - raise ValueError('Provided or resolved key-encryption-key does not match the id of key used to encrypt.') - # Will throw an exception if the specified algorithm is not supported. - content_encryption_key = key_encryption_key.unwrap_key(encryption_data.wrapped_content_key.encrypted_key, - encryption_data.wrapped_content_key.algorithm) - - # For V2, the version is included with the cek. We need to validate it - # and remove it from the actual cek. - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - version_2_bytes = _ENCRYPTION_PROTOCOL_V2.encode().ljust(8, b'\0') - cek_version_bytes = content_encryption_key[:len(version_2_bytes)] - if cek_version_bytes != version_2_bytes: - raise ValueError('The encryption metadata is not valid and may have been modified.') - - # Remove version from the start of the cek. - content_encryption_key = content_encryption_key[len(version_2_bytes):] - - _validate_not_none('content_encryption_key', content_encryption_key) - - return content_encryption_key - - -def _decrypt_message(message, encryption_data, key_encryption_key=None, resolver=None): - ''' - Decrypts the given ciphertext using AES256 in CBC mode with 128 bit padding. - Unwraps the content-encryption-key using the user-provided or resolved key-encryption-key (kek). - Returns the original plaintex. - - :param str message: - The ciphertext to be decrypted. - :param _EncryptionData encryption_data: - The metadata associated with this ciphertext. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - unwrap_key(key, algorithm) - - returns the unwrapped form of the specified symmetric key using the string-specified algorithm. - get_kid() - - returns a string key id for this key-encryption-key. - :param function resolver(kid): - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :return: The decrypted plaintext. - :rtype: str - ''' - _validate_not_none('message', message) - content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, resolver) - - if encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V1: - if not encryption_data.content_encryption_IV: - raise ValueError("Missing required metadata for decryption.") - - cipher = _generate_AES_CBC_cipher(content_encryption_key, encryption_data.content_encryption_IV) - - # decrypt data - decrypted_data = message - decryptor = cipher.decryptor() - decrypted_data = (decryptor.update(decrypted_data) + decryptor.finalize()) - - # unpad data - unpadder = PKCS7(128).unpadder() - decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize()) - - elif encryption_data.encryption_agent.protocol == _ENCRYPTION_PROTOCOL_V2: - block_info = encryption_data.encrypted_region_info - if not block_info or not block_info.nonce_length: - raise ValueError("Missing required metadata for decryption.") - - nonce_length = encryption_data.encrypted_region_info.nonce_length - - # First bytes are the nonce - nonce = message[:nonce_length] - ciphertext_with_tag = message[nonce_length:] - - aesgcm = AESGCM(content_encryption_key) - decrypted_data = aesgcm.decrypt(nonce, ciphertext_with_tag, None) - - else: - raise ValueError('Specified encryption version is not supported.') - - return decrypted_data - - -def encrypt_blob(blob, key_encryption_key, version): - ''' - Encrypts the given blob using the given encryption protocol version. - Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek). - Returns a json-formatted string containing the encryption metadata. This method should - only be used when a blob is small enough for single shot upload. Encrypting larger blobs - is done as a part of the upload_data_chunks method. - - :param bytes blob: - The blob to be encrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param str version: The client encryption version to use. - :return: A tuple of json-formatted string containing the encryption metadata and the encrypted blob data. - :rtype: (str, bytes) - ''' - - _validate_not_none('blob', blob) - _validate_not_none('key_encryption_key', key_encryption_key) - _validate_key_encryption_key_wrap(key_encryption_key) - - if version == _ENCRYPTION_PROTOCOL_V1: - # AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks - content_encryption_key = os.urandom(32) - initialization_vector = os.urandom(16) - - cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector) - - # PKCS7 with 16 byte blocks ensures compatibility with AES. - padder = PKCS7(128).padder() - padded_data = padder.update(blob) + padder.finalize() - - # Encrypt the data. - encryptor = cipher.encryptor() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - - elif version == _ENCRYPTION_PROTOCOL_V2: - # AES256 GCM uses 256 bit (32 byte) keys and a 12 byte nonce. - content_encryption_key = AESGCM.generate_key(bit_length=256) - initialization_vector = None - - data = BytesIO(blob) - encryption_stream = GCMBlobEncryptionStream(content_encryption_key, data) - - encrypted_data = encryption_stream.read() - - else: - raise ValueError("Invalid encryption version specified.") - - encryption_data = _generate_encryption_data_dict(key_encryption_key, content_encryption_key, - initialization_vector, version) - encryption_data['EncryptionMode'] = 'FullBlob' - - return dumps(encryption_data), encrypted_data - - -def generate_blob_encryption_data(key_encryption_key, version): - ''' - Generates the encryption_metadata for the blob. - - :param object key_encryption_key: - The key-encryption-key used to wrap the cek associate with this blob. - :param str version: The client encryption version to use. - :return: A tuple containing the cek and iv for this blob as well as the - serialized encryption metadata for the blob. - :rtype: (bytes, Optional[bytes], str) - ''' - encryption_data = None - content_encryption_key = None - initialization_vector = None - if key_encryption_key: - _validate_key_encryption_key_wrap(key_encryption_key) - content_encryption_key = os.urandom(32) - # Initialization vector only needed for V1 - if version == _ENCRYPTION_PROTOCOL_V1: - initialization_vector = os.urandom(16) - encryption_data = _generate_encryption_data_dict(key_encryption_key, - content_encryption_key, - initialization_vector, - version) - encryption_data['EncryptionMode'] = 'FullBlob' - encryption_data = dumps(encryption_data) - - return content_encryption_key, initialization_vector, encryption_data - - -def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements - require_encryption, - key_encryption_key, - key_resolver, - content, - start_offset, - end_offset, - response_headers): - """ - Decrypts the given blob contents and returns only the requested range. - - :param bool require_encryption: - Whether the calling blob service requires objects to be decrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param object key_resolver: - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :param bytes content: - The encrypted blob content. - :param int start_offset: - The adjusted offset from the beginning of the *decrypted* content for the caller's data. - :param int end_offset: - The adjusted offset from the end of the *decrypted* content for the caller's data. - :param Dict[str, Any] response_headers: - A dictionary of response headers from the download request. Expected to include the - 'x-ms-meta-encryptiondata' header if the blob was encrypted. - :return: The decrypted blob content. - :rtype: bytes - """ - try: - encryption_data = _dict_to_encryption_data(loads(response_headers['x-ms-meta-encryptiondata'])) - except: # pylint: disable=bare-except - if require_encryption: - raise ValueError( - 'Encryption required, but received data does not contain appropriate metatadata.' + \ - 'Data was either not encrypted or metadata has been lost.') - - return content - - algorithm = encryption_data.encryption_agent.encryption_algorithm - if algorithm not in(_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256): - raise ValueError('Specified encryption algorithm is not supported.') - - version = encryption_data.encryption_agent.protocol - if version not in (_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2): - raise ValueError('Specified encryption version is not supported.') - - content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, key_resolver) - - if version == _ENCRYPTION_PROTOCOL_V1: - blob_type = response_headers['x-ms-blob-type'] - - iv = None - unpad = False - if 'content-range' in response_headers: - content_range = response_headers['content-range'] - # Format: 'bytes x-y/size' - - # Ignore the word 'bytes' - content_range = content_range.split(' ') - - content_range = content_range[1].split('-') - content_range = content_range[1].split('/') - end_range = int(content_range[0]) - blob_size = int(content_range[1]) - - if start_offset >= 16: - iv = content[:16] - content = content[16:] - start_offset -= 16 - else: - iv = encryption_data.content_encryption_IV - - if end_range == blob_size - 1: - unpad = True - else: - unpad = True - iv = encryption_data.content_encryption_IV - - if blob_type == 'PageBlob': - unpad = False - - cipher = _generate_AES_CBC_cipher(content_encryption_key, iv) - decryptor = cipher.decryptor() - - content = decryptor.update(content) + decryptor.finalize() - if unpad: - unpadder = PKCS7(128).unpadder() - content = unpadder.update(content) + unpadder.finalize() - - return content[start_offset: len(content) - end_offset] - - if version == _ENCRYPTION_PROTOCOL_V2: - # We assume the content contains only full encryption regions - total_size = len(content) - offset = 0 - - nonce_length = encryption_data.encrypted_region_info.nonce_length - data_length = encryption_data.encrypted_region_info.data_length - tag_length = encryption_data.encrypted_region_info.tag_length - region_length = nonce_length + data_length + tag_length - - decrypted_content = bytearray() - while offset < total_size: - # Process one encryption region at a time - process_size = min(region_length, total_size) - encrypted_region = content[offset:offset + process_size] - - # First bytes are the nonce - nonce = encrypted_region[:nonce_length] - ciphertext_with_tag = encrypted_region[nonce_length:] - - aesgcm = AESGCM(content_encryption_key) - decrypted_data = aesgcm.decrypt(nonce, ciphertext_with_tag, None) - decrypted_content.extend(decrypted_data) - - offset += process_size - - # Read the caller requested data from the decrypted content - return decrypted_content[start_offset:end_offset] - - -def get_blob_encryptor_and_padder(cek, iv, should_pad): - encryptor = None - padder = None - - if cek is not None and iv is not None: - cipher = _generate_AES_CBC_cipher(cek, iv) - encryptor = cipher.encryptor() - padder = PKCS7(128).padder() if should_pad else None - - return encryptor, padder - - -def encrypt_queue_message(message, key_encryption_key, version): - ''' - Encrypts the given plain text message using the given protocol version. - Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek). - Returns a json-formatted string containing the encrypted message and the encryption metadata. - - :param object message: - The plain text messge to be encrypted. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - wrap_key(key)--wraps the specified key using an algorithm of the user's choice. - get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. - get_kid()--returns a string key id for this key-encryption-key. - :param str version: The client encryption version to use. - :return: A json-formatted string containing the encrypted message and the encryption metadata. - :rtype: str - ''' - - _validate_not_none('message', message) - _validate_not_none('key_encryption_key', key_encryption_key) - _validate_key_encryption_key_wrap(key_encryption_key) - - # Queue encoding functions all return unicode strings, and encryption should - # operate on binary strings. - message = message.encode('utf-8') - - if version == _ENCRYPTION_PROTOCOL_V1: - # AES256 CBC uses 256 bit (32 byte) keys and always with 16 byte blocks - content_encryption_key = os.urandom(32) - initialization_vector = os.urandom(16) - - cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector) - - # PKCS7 with 16 byte blocks ensures compatibility with AES. - padder = PKCS7(128).padder() - padded_data = padder.update(message) + padder.finalize() - - # Encrypt the data. - encryptor = cipher.encryptor() - encrypted_data = encryptor.update(padded_data) + encryptor.finalize() - - elif version == _ENCRYPTION_PROTOCOL_V2: - # AES256 GCM uses 256 bit (32 byte) keys and a 12 byte nonce. - content_encryption_key = AESGCM.generate_key(bit_length=256) - initialization_vector = None - - # The nonce MUST be different for each key - nonce = os.urandom(12) - aesgcm = AESGCM(content_encryption_key) - - # Returns ciphertext + tag - cipertext_with_tag = aesgcm.encrypt(nonce, message, None) - encrypted_data = nonce + cipertext_with_tag - - else: - raise ValueError("Invalid encryption version specified.") - - # Build the dictionary structure. - queue_message = {'EncryptedMessageContents': encode_base64(encrypted_data), - 'EncryptionData': _generate_encryption_data_dict(key_encryption_key, - content_encryption_key, - initialization_vector, - version)} - - return dumps(queue_message) - - -def decrypt_queue_message(message, response, require_encryption, key_encryption_key, resolver): - ''' - Returns the decrypted message contents from an EncryptedQueueMessage. - If no encryption metadata is present, will return the unaltered message. - :param str message: - The JSON formatted QueueEncryptedMessage contents with all associated metadata. - :param bool require_encryption: - If set, will enforce that the retrieved messages are encrypted and decrypt them. - :param object key_encryption_key: - The user-provided key-encryption-key. Must implement the following methods: - unwrap_key(key, algorithm) - - returns the unwrapped form of the specified symmetric key usingthe string-specified algorithm. - get_kid() - - returns a string key id for this key-encryption-key. - :param function resolver(kid): - The user-provided key resolver. Uses the kid string to return a key-encryption-key - implementing the interface defined above. - :return: The plain text message from the queue message. - :rtype: str - ''' - response = response.http_response - - try: - message = loads(message) - - encryption_data = _dict_to_encryption_data(message['EncryptionData']) - decoded_data = decode_base64_to_bytes(message['EncryptedMessageContents']) - except (KeyError, ValueError): - # Message was not json formatted and so was not encrypted - # or the user provided a json formatted message - # or the metadata was malformed. - if require_encryption: - raise ValueError( - 'Encryption required, but received message does not contain appropriate metatadata. ' + \ - 'Message was either not encrypted or metadata was incorrect.') - - return message - try: - return _decrypt_message(decoded_data, encryption_data, key_encryption_key, resolver).decode('utf-8') - except Exception as error: - raise HttpResponseError( - message="Decryption failed.", - response=response, - error=error) diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads.py index 167217734bea..279f084ff970 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads.py @@ -6,19 +6,17 @@ # pylint: disable=no-self-use from concurrent import futures -from io import (BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation) -from threading import Lock +from io import BytesIO, IOBase, SEEK_CUR, SEEK_END, SEEK_SET, UnsupportedOperation from itertools import islice from math import ceil +from threading import Lock import six - from azure.core.tracing.common import with_current_context from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 @@ -52,18 +50,9 @@ def upload_data_chunks( max_concurrency=None, stream=None, validate_content=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -149,7 +138,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = Lock() if parallel else None # Progress feedback diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads_async.py index 2d8376aff237..97be2caf89f8 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/uploads_async.py @@ -6,10 +6,9 @@ # pylint: disable=no-self-use import asyncio +import threading from asyncio import Lock from itertools import islice -import threading - from math import ceil import six @@ -17,14 +16,9 @@ from . import encode_base64, url_quote from .request_handlers import get_length from .response_handlers import return_response_headers -from .encryption import get_blob_encryptor_and_padder from .uploads import SubStream, IterStreamer # pylint: disable=unused-import -_LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 -_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.' - - async def _parallel_uploads(uploader, pending, running): range_ids = [] while True: @@ -52,18 +46,9 @@ async def upload_data_chunks( chunk_size=None, max_concurrency=None, stream=None, - encryption_options=None, progress_hook=None, **kwargs): - if encryption_options: - encryptor, padder = get_blob_encryptor_and_padder( - encryption_options.get('cek'), - encryption_options.get('vector'), - uploader_class is not PageBlobChunkUploader) - kwargs['encryptor'] = encryptor - kwargs['padder'] = padder - parallel = max_concurrency > 1 if parallel and 'modified_access_conditions' in kwargs: # Access conditions do not work with parallelism @@ -152,7 +137,6 @@ def __init__( self.parallel = parallel # Stream management - self.stream_start = stream.tell() if parallel else None self.stream_lock = threading.Lock() if parallel else None # Progress feedback @@ -217,7 +201,7 @@ async def _update_progress(self, length): self.progress_total += length if self.progress_hook: - self.progress_hook(self.progress_total, self.total_size) + await self.progress_hook(self.progress_total, self.total_size) async def _upload_chunk(self, chunk_offset, chunk_data): raise NotImplementedError("Must be implemented by child class.") diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_client_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_client_async.py index 747b3ffd935a..f9a920bdf7d5 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_client_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_client_async.py @@ -8,43 +8,36 @@ import functools import warnings from typing import ( # pylint: disable=unused-import - Optional, - Any, - Dict, - List, - TYPE_CHECKING, -) + Any, Dict, List, Optional, + TYPE_CHECKING) +from azure.core.async_paging import AsyncItemPaged from azure.core.exceptions import HttpResponseError from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async -from azure.core.async_paging import AsyncItemPaged - from .._serialize import get_api_version from .._shared.base_client_async import AsyncStorageAccountHostsMixin +from .._shared.policies_async import ExponentialRetry from .._shared.request_handlers import add_metadata_headers, serialize_iso from .._shared.response_handlers import ( return_response_headers, process_storage_error, return_headers_and_deserialized, ) -from .._deserialize import deserialize_queue_properties, deserialize_queue_creation from .._generated.aio import AzureQueueStorage -from .._generated.models import SignedIdentifier -from .._generated.models import QueueMessage as GenQueueMessage - +from .._generated.models import SignedIdentifier, QueueMessage as GenQueueMessage +from .._deserialize import deserialize_queue_properties, deserialize_queue_creation +from .._encryption import StorageEncryptionMixin from .._models import QueueMessage, AccessPolicy -from ._models import MessagesPaged -from .._shared.policies_async import ExponentialRetry from .._queue_client import QueueClient as QueueClientBase - +from ._models import MessagesPaged if TYPE_CHECKING: from .._models import QueueProperties -class QueueClient(AsyncStorageAccountHostsMixin, QueueClientBase): +class QueueClient(AsyncStorageAccountHostsMixin, QueueClientBase, StorageEncryptionMixin): """A client to interact with a specific Queue. :param str account_url: @@ -103,6 +96,7 @@ def __init__( pipeline=self._pipeline, loop=loop) # type: ignore self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access self._loop = loop + self.configure_encryption(kwargs) @distributed_trace_async async def create_queue(self, **kwargs): diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_service_client_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_service_client_async.py index f286504c83aa..3062644d22ec 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_service_client_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/aio/_queue_service_client_async.py @@ -7,48 +7,41 @@ import functools from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, + Any, Dict, List, Optional, Union, TYPE_CHECKING) -try: - from urllib.parse import urlparse # pylint: disable=unused-import -except ImportError: - from urlparse import urlparse # type: ignore -from azure.core.exceptions import HttpResponseError from azure.core.async_paging import AsyncItemPaged -from azure.core.tracing.decorator import distributed_trace +from azure.core.exceptions import HttpResponseError from azure.core.pipeline import AsyncPipeline +from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async from .._serialize import get_api_version +from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper from .._shared.policies_async import ExponentialRetry -from .._queue_service_client import QueueServiceClient as QueueServiceClientBase from .._shared.models import LocationMode -from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper from .._shared.response_handlers import process_storage_error from .._generated.aio import AzureQueueStorage from .._generated.models import StorageServiceProperties - -from ._models import QueuePropertiesPaged -from ._queue_client_async import QueueClient +from .._encryption import StorageEncryptionMixin from .._models import ( service_stats_deserialize, service_properties_deserialize, ) +from .._queue_service_client import QueueServiceClient as QueueServiceClientBase +from ._models import QueuePropertiesPaged +from ._queue_client_async import QueueClient if TYPE_CHECKING: - from datetime import datetime - from azure.core.configuration import Configuration - from azure.core.pipeline.policies import HTTPPolicy from .._models import ( + CorsRule, + Metrics, QueueProperties, QueueAnalyticsLogging, - Metrics, - CorsRule, ) -class QueueServiceClient(AsyncStorageAccountHostsMixin, QueueServiceClientBase): +class QueueServiceClient(AsyncStorageAccountHostsMixin, QueueServiceClientBase, StorageEncryptionMixin): """A client to interact with the Queue Service at the account level. This client provides operations to retrieve and configure the account properties @@ -104,6 +97,7 @@ def __init__( self._client = AzureQueueStorage(self.url, base_url=self.url, pipeline=self._pipeline, loop=loop) # type: ignore self._client._config.version = get_api_version(kwargs) # pylint: disable=protected-access self._loop = loop + self.configure_encryption(kwargs) @distributed_trace_async async def get_service_stats(self, **kwargs): diff --git a/sdk/storage/azure-storage-queue/tests/test_queue_encryption.py b/sdk/storage/azure-storage-queue/tests/test_queue_encryption.py index 1a07d581177b..4e6052ed128b 100644 --- a/sdk/storage/azure-storage-queue/tests/test_queue_encryption.py +++ b/sdk/storage/azure-storage-queue/tests/test_queue_encryption.py @@ -3,10 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + import os import unittest -import pytest -import six from base64 import ( b64decode, b64encode, @@ -16,15 +15,17 @@ dumps, ) -from cryptography.hazmat import backends -from cryptography.hazmat.primitives.ciphers import Cipher -from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.hazmat.primitives.ciphers.algorithms import AES -from cryptography.hazmat.primitives.ciphers.modes import CBC -from cryptography.hazmat.primitives.padding import PKCS7 +import pytest +import six from azure.core.exceptions import HttpResponseError, ResourceExistsError +from azure.storage.queue import ( + VERSION, + QueueServiceClient, + BinaryBase64EncodePolicy, + BinaryBase64DecodePolicy, +) from azure.storage.queue._shared import decode_base64_to_bytes -from azure.storage.queue._shared.encryption import ( +from azure.storage.queue._encryption import ( _ERROR_OBJECT_INVALID, _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, @@ -34,20 +35,21 @@ _validate_and_unwrap_cek, _WrappedContentKey, ) +from cryptography.hazmat import backends +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.ciphers.modes import CBC +from cryptography.hazmat.primitives.padding import PKCS7 -from azure.storage.queue import ( - VERSION, - QueueServiceClient, - BinaryBase64EncodePolicy, - BinaryBase64DecodePolicy, -) +from devtools_testutils.storage import StorageTestCase from encryption_test_helper import ( KeyWrapper, KeyResolver, RSAKeyWrapper, ) from settings.testcase import QueuePreparer -from devtools_testutils.storage import StorageTestCase + # ------------------------------------------------------------------------------ TEST_QUEUE_PREFIX = 'encryptionqueue' diff --git a/sdk/storage/azure-storage-queue/tests/test_queue_encryption_async.py b/sdk/storage/azure-storage-queue/tests/test_queue_encryption_async.py index eb370a8fceeb..be9de0aeb895 100644 --- a/sdk/storage/azure-storage-queue/tests/test_queue_encryption_async.py +++ b/sdk/storage/azure-storage-queue/tests/test_queue_encryption_async.py @@ -3,10 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + import os import unittest -import pytest -import six from base64 import ( b64decode, b64encode, @@ -16,16 +15,17 @@ dumps, ) -from cryptography.hazmat import backends -from cryptography.hazmat.primitives.ciphers import Cipher -from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.hazmat.primitives.ciphers.algorithms import AES -from cryptography.hazmat.primitives.ciphers.modes import CBC -from cryptography.hazmat.primitives.padding import PKCS7 - +import pytest +import six from azure.core.exceptions import HttpResponseError, ResourceExistsError -from multidict import CIMultiDict, CIMultiDictProxy -from azure.storage.queue._shared.encryption import ( +from azure.core.pipeline.transport import AioHttpTransport +from azure.storage.queue import ( + VERSION, + BinaryBase64EncodePolicy, + BinaryBase64DecodePolicy, +) +from azure.storage.queue.aio import QueueServiceClient +from azure.storage.queue._encryption import ( _ERROR_OBJECT_INVALID, _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, @@ -35,24 +35,21 @@ _validate_and_unwrap_cek, _WrappedContentKey, ) -from azure.core.pipeline.transport import AioHttpTransport -from azure.storage.queue import ( - VERSION, - BinaryBase64EncodePolicy, - BinaryBase64DecodePolicy, -) -from azure.storage.queue.aio import ( - QueueServiceClient, - QueueClient -) +from cryptography.hazmat import backends +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.ciphers.modes import CBC +from cryptography.hazmat.primitives.padding import PKCS7 +from multidict import CIMultiDict, CIMultiDictProxy +from devtools_testutils.storage.aio import AsyncStorageTestCase from encryption_test_helper import ( KeyWrapper, KeyResolver, RSAKeyWrapper, ) -from devtools_testutils.storage.aio import AsyncStorageTestCase from settings.testcase import QueuePreparer # ------------------------------------------------------------------------------