From 8e20e5be7feb70471f10e6d4ab11ce69e6e5cb67 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Mon, 18 May 2020 09:57:37 -0700 Subject: [PATCH 01/21] Port clientside encryption to mono-dll Ported blob client-side encryption work from long-diverged branch. Was an inheritance approach, is now part of the main package internals. Ported over WindowStream from stg73base, which allows us to prematurely end streams. Removed RollingBufferStream, as uploads no longer require seekable streams as input. --- ...e.Storage.Blobs.Batch.Samples.Tests.csproj | 1 - .../Azure.Storage.Blobs.Batch.Tests.csproj | 1 + .../BreakingChanges.txt | 6 - .../CHANGELOG.md | 6 - .../README.md | 109 ----- ...orage.Blobs.Cryptography.netstandard2.0.cs | 12 - .../src/AssemblyInfo.cs | 11 - .../Azure.Storage.Blobs.Cryptography.csproj | 24 -- .../src/EncryptedBlockBlobClient.cs | 142 ------- .../Azure.Storage.Blobs.Samples.Tests.csproj | 1 - .../src/AdvancedBlobClientOptions.cs | 38 ++ .../src/AppendBlobClient.cs | 9 +- .../src/Azure.Storage.Blobs.csproj | 8 +- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 156 ++++++- .../Azure.Storage.Blobs/src/BlobClient.cs | 87 +++- .../src/BlobClientOptions.cs | 4 + .../src/BlobContainerClient.cs | 26 +- .../src/BlobServiceClient.cs | 30 +- .../src/BlockBlobClient.cs | 19 +- .../src/Models/ContentRange.cs | 149 +++++++ .../Azure.Storage.Blobs/src/PageBlobClient.cs | 9 +- .../tests/Azure.Storage.Blobs.Tests.csproj | 6 +- .../Azure.Storage.Blobs/tests/BlobTestBase.cs | 17 + .../tests/ClientSideEncryptionTests.cs | 388 ++++++++++++++++++ .../tests/CryptoraphyTestsExtensionMethods.cs | 13 + .../tests/EncryptedBlockBlobClientTests.cs | 48 --- .../tests/MockKeyEncryptionKey.cs | 164 ++++++++ .../Azure.Storage.Blobs/tests/MockStream.cs | 54 +++ .../tests/SasQueryParametersTests.cs | 1 - .../src/Azure.Storage.Common.csproj | 1 + .../src/ClientSideEncryptionVersion.cs | 18 + .../src/ClientsideEncryptionOptions.cs | 63 +++ .../ClientSideEncryptionVersionExtensions.cs | 37 ++ .../EncryptionConstants.cs | 28 ++ .../ClientsideEncryption/EncryptionErrors.cs | 27 ++ .../Models/ClientSideEncryptionAlgorithm.cs | 68 +++ .../Models/EncryptedBlobRange.cs | 72 ++++ .../Models/EncryptionAgent.cs | 21 + .../Models/EncryptionData.cs | 102 +++++ .../Models/EncryptionDataSerializer.cs | 165 ++++++++ .../ClientsideEncryption/Models/WrappedKey.cs | 26 ++ .../Shared/ClientsideEncryption/Utility.cs | 295 +++++++++++++ .../src/Shared/Errors.Clients.cs | 14 +- .../Azure.Storage.Common/src/Shared/Errors.cs | 4 + .../src/Shared/WindowStream.cs | 81 ++++ .../tests/Shared/KeyVaultConfiguration.cs | 40 ++ .../tests/Shared/TestConfigurations.cs | 48 ++- .../Shared/TestConfigurationsTemplate.xml | 11 + .../src/AdvancedQueueClientOptions.cs | 38 ++ .../src/QueueClientOptions.cs | 4 + sdk/storage/Azure.Storage.sln | 8 +- 51 files changed, 2312 insertions(+), 398 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/README.md delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj delete mode 100644 sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs delete mode 100644 sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs create mode 100644 sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj index c4fa15a3c5a23..739f325f6efb0 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj @@ -9,7 +9,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj index cb4a73b1257ef..82561843c497d 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt b/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt deleted file mode 100644 index 54c406eff995d..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt +++ /dev/null @@ -1,6 +0,0 @@ -Breaking Changes -================ - -Next Preview --------------------------- -- New Azure.Storage.Blobs.Cryptography client library. diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md b/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md deleted file mode 100644 index 341f38fc0934e..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release History - -## 12.0.0-preview.5 - -- This preview is the first release supporting client-side encryption for Azure - Storage blobs. diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md b/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md deleted file mode 100644 index 1a6d44429b0f3..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Azure Storage Blobs Cryptography client library for .NET - -> Server Version: 2019-02-02 - -Azure Blob storage is Microsoft's object storage solution for the cloud. Blob -storage is optimized for storing massive amounts of unstructured data. -Unstructured data is data that does not adhere to a particular data model or -definition, such as text or binary data. - -[Source code][source] | [Package (NuGet)][package] | [API reference documentation][docs] | [REST API documentation][rest_docs] | [Product documentation][product_docs] - -## Getting started - -### Install the package - -Install the Azure Storage Blobs Cryptography client library for .NET with [NuGet][nuget]: - -```Powershell -dotnet add package Azure.Storage.Blobs.Cryptography --version 12.0.0-preview.4 -``` - -### Prerequisites - -You need an [Azure subscription][azure_sub] and a -[Storage Account][storage_account_docs] to use this package. - -To create a new Storage Account, you can use the [Azure Portal][storage_account_create_portal], -[Azure PowerShell][storage_account_create_ps], or the [Azure CLI][storage_account_create_cli]. -Here's an example using the Azure CLI: - -```Powershell -az storage account create --name MyStorageAccount --resource-group MyResourceGroup --location westus --sku Standard_LRS -``` - -## Key concepts - -TODO: Add Key Concepts - -## Examples - -TODO: Add Examples - -## Troubleshooting - -All Blob service operations will throw a -[RequestFailedException][RequestFailedException] on failure with -helpful [`ErrorCode`s][error_codes]. Many of these errors are recoverable. - -TODO: Update sample - -```c# -string connectionString = ""; - -// Try to create a container named "sample-container" and avoid any potential race -// conditions that might arise by checking if the container exists before creating -BlobContainerClient container = new BlobContainerClient(connectionString, "sample-container"); -try -{ - container.Create(); -} -catch (RequestFailedException ex) - when (ex.ErrorCode == BlobErrorCode.ContainerAlreadyExists) -{ - // Ignore any errors if the container already exists -} -``` - -## Next steps - -TODO: Link Samples - -## Contributing - -See the [Storage CONTRIBUTING.md][storage_contrib] for details on building, -testing, and contributing to this library. - -This project welcomes contributions and suggestions. Most contributions require -you to agree to a Contributor License Agreement (CLA) declaring that you have -the right to, and actually do, grant us the rights to use your contribution. For -details, visit [cla.microsoft.com][cla]. - -This project has adopted the [Microsoft Open Source Code of Conduct][coc]. -For more information see the [Code of Conduct FAQ][coc_faq] -or contact [opencode@microsoft.com][coc_contact] with any -additional questions or comments. - -![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-net%2Fsdk%2Fstorage%2FAzure.Storage.Blobs.Cryptography%2FREADME.png) - - -[source]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/storage/Azure.Storage.Blobs.Cryptography/src -[package]: https://www.nuget.org/packages/Azure.Storage.Blobs.Cryptography/ -[docs]: https://azure.github.io/azure-sdk-for-net/storage.html -[rest_docs]: https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api -[product_docs]: https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview -[nuget]: https://www.nuget.org/ -[storage_account_docs]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview -[storage_account_create_ps]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-powershell -[storage_account_create_cli]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-cli -[storage_account_create_portal]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-portal -[azure_cli]: https://docs.microsoft.com/cli/azure -[azure_sub]: https://azure.microsoft.com/free/ -[identity]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/identity/Azure.Identity/README.md -[RequestFailedException]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/core/Azure.Core/src/RequestFailedException.cs -[error_codes]: https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes -[storage_contrib]: ../CONTRIBUTING.md -[cla]: https://cla.microsoft.com -[coc]: https://opensource.microsoft.com/codeofconduct/ -[coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ -[coc_contact]: mailto:opencode@microsoft.com diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs deleted file mode 100644 index a50a41a71bd94..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Azure.Storage.Blobs.Specialized -{ - public partial class EncryptedBlockBlobClient : Azure.Storage.Blobs.BlobClient - { - protected EncryptedBlockBlobClient() { } - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName) { } - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName, Azure.Storage.Blobs.BlobClientOptions options) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Core.TokenCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Storage.Blobs.BlobClientOptions options = null) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { } - } -} diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs deleted file mode 100644 index 813f19b67cc4e..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("Azure.Storage.Blobs.Tests, PublicKey=" + - "0024000004800000940000000602000000240000525341310004000001000100d15ddcb2968829" + - "5338af4b7686603fe614abd555e09efba8fb88ee09e1f7b1ccaeed2e8f823fa9eef3fdd60217fc" + - "012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265" + - "e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593d" + - "aa7b11b4")] diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj deleted file mode 100644 index 3edd0054cacdd..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - $(RequiredTargetFrameworks) - - - Microsoft Azure.Storage.Blobs.Cryptography client library - 12.0.0-preview.5 - BlobSDK;$(DefineConstants) - Microsoft Azure Storage Blobs Cryptography;Encrypted blob;client-side encryption;client side ecnryption;Microsoft;Azure;Blobs;Blob;Storage;StorageScalable;$(PackageCommonTags) - - This client library supports client-side encryption for the Microsoft Azure Storage Blob service. - For this release see notes - https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md and https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md - in addition to the breaking changes https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt - Microsoft Azure Storage quickstarts and tutorials - https://docs.microsoft.com/en-us/azure/storage/ - Microsoft Azure Storage REST API Reference - https://docs.microsoft.com/en-us/rest/api/storageservices/ - REST API Reference for Blob Service - https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api - - false - - - - - - diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs deleted file mode 100644 index 618931a4f77d2..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using Azure.Core; -using Azure.Core.Pipeline; -using Azure.Storage.Blobs.Models; - -#pragma warning disable SA1402 // File may only contain a single type - -namespace Azure.Storage.Blobs.Specialized -{ - /// - /// The allows you to manipulate - /// Azure Storage block blobs with client-side encryption. - /// - public class EncryptedBlockBlobClient : BlobClient - { - #region ctors - /// - /// Initializes a new instance of the - /// class for mocking. - /// - protected EncryptedBlockBlobClient() - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A connection string includes the authentication information - /// required for your application to access data in an Azure Storage - /// account at runtime. - /// - /// For more information, . - /// - /// - /// The name of the container containing this encrypted block blob. - /// - /// - /// The name of this encrypted block blob. - /// - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName) - : base(connectionString, containerName, blobName) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A connection string includes the authentication information - /// required for your application to access data in an Azure Storage - /// account at runtime. - /// - /// For more information, . - /// - /// - /// The name of the container containing this encrypted block blob. - /// - /// - /// The name of this encrypted block blob. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName, BlobClientOptions options) - : base(connectionString, containerName, blobName, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the encrypted block blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, BlobClientOptions options = default) - : base(blobUri, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// The shared key credential used to sign requests. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) - : base(blobUri, credential, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// The token credential used to sign requests. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) - : base(blobUri, credential, options) - { - } - - #endregion ctors - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj index 62f992b5a1221..27e450932b380 100644 --- a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj @@ -9,7 +9,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs new file mode 100644 index 0000000000000..cee3882031c42 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Blobs.Specialized +{ + /// + /// Provides advanced client configuration options for connecting to Azure Blob + /// Storage. + /// +#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. + public class AdvancedBlobClientOptions : BlobClientOptions +#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion + { + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// The of the service API used when + /// making requests. + /// + public AdvancedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) + { + } + + /// + /// Settings for data encryption when uploading and downloading with a . + /// Client-side encryption adds metadata to your blob which is necessary for decryption. + /// + /// For more information, see . + /// + public ClientSideEncryptionOptions ClientSideEncryption + { + get => _clientSideEncryptionOptions; + set => _clientSideEncryptionOptions = value; + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 57233499a3456..08416a63cf4f9 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -192,7 +192,14 @@ internal AppendBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } #endregion ctors diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index dee2a351b326d..e92aad6a91721 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) @@ -16,6 +16,9 @@ REST API Reference for Blob Service - https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api + + + @@ -35,6 +38,8 @@ + + @@ -61,5 +66,6 @@ + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index ab5bae28a1042..40822ff11b1e0 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -9,6 +9,10 @@ using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized.Models; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Shared; using Metadata = System.Collections.Generic.IDictionary; #pragma warning disable SA1402 // File may only contain a single type @@ -75,6 +79,18 @@ public class BlobBaseClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + + internal bool UsingClientSideEncryption => ClientSideEncryption != default; + /// /// The name of the Encryption Scope to be used when sending requests. /// @@ -204,6 +220,7 @@ public BlobBaseClient(string connectionString, string blobContainerName, string _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -302,6 +319,7 @@ internal BlobBaseClient(Uri blobUri, HttpPipelinePolicy authentication, BlobClie _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -325,6 +343,7 @@ internal BlobBaseClient(Uri blobUri, HttpPipelinePolicy authentication, BlobClie /// /// Client diagnostics. /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. internal BlobBaseClient( Uri blobUri, @@ -332,6 +351,7 @@ internal BlobBaseClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) { _uri = blobUri; @@ -339,6 +359,7 @@ internal BlobBaseClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -370,7 +391,7 @@ internal BlobBaseClient( protected virtual BlobBaseClient WithSnapshotCore(string snapshot) { var builder = new BlobUriBuilder(Uri) { Snapshot = snapshot }; - return new BlobBaseClient(builder.ToUri(), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + return new BlobBaseClient(builder.ToUri(), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); } /// @@ -643,9 +664,11 @@ private async Task> DownloadInternal( Pipeline.LogMethodEnter(nameof(BlobBaseClient), message: $"{nameof(Uri)}: {Uri}"); try { + EncryptedBlobRange encryptedRange = new EncryptedBlobRange(range); + // Start downloading the blob (Response response, Stream stream) = await StartDownloadAsync( - range, + UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, conditions, rangeGetContentHash, async: async, @@ -661,11 +684,11 @@ private async Task> DownloadInternal( // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - response.Value.Content = RetriableStream.Create( + stream = RetriableStream.Create( stream, startOffset => StartDownloadAsync( - range, + UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, conditions, rangeGetContentHash, startOffset, @@ -675,7 +698,7 @@ private async Task> DownloadInternal( .Item2, async startOffset => (await StartDownloadAsync( - range, + UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, conditions, rangeGetContentHash, startOffset, @@ -686,6 +709,15 @@ private async Task> DownloadInternal( Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); + // if using clientside encryption, wrap the auto-retry stream in a decryptor + // we already return a nonseekable stream; returning a crypto stream is fine + if (UsingClientSideEncryption) + { + stream = await ClientSideDecryptInternal(stream, response.Value.Metadata, encryptedRange.OriginalRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); + } + + response.Value.Content = stream; + // Wrap the FlattenedDownloadProperties into a BlobDownloadOperation // to make the Content easier to find return Response.FromValue(new BlobDownloadInfo(response.Value), response.GetRawResponse()); @@ -1201,7 +1233,7 @@ internal async Task StagedDownloadAsync( bool async = true, CancellationToken cancellationToken = default) { - var client = new BlobBaseClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + var client = new BlobBaseClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); PartitionedDownloader downloader = new PartitionedDownloader(client, transferOptions); @@ -2950,6 +2982,117 @@ private async Task SetAccessTierInternal( } } #endregion SetAccessTier + + private async Task ClientSideDecryptInternal( + Stream content, + Metadata metadata, + HttpRange originalRange, + string receivedContentRange, + bool async, + CancellationToken cancellationToken) + { + ContentRange? contentRange = string.IsNullOrWhiteSpace(receivedContentRange) + ? default + : ContentRange.Parse(receivedContentRange); + + EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata); + if (encryptionData == default) + { + return content; // TODO readjust range + } + + bool ivInStream = originalRange.Offset >= 16; + + var plaintext = await Utility.DecryptInternal( + content, + encryptionData, + ivInStream, + ClientSideEncryption.KeyResolver, + ClientSideEncryption.KeyEncryptionKey, + CanIgnorePadding(contentRange), + async, + cancellationToken).ConfigureAwait(false); + + // retrim start of stream to original requested location + // keeping in mind whether we already pulled the IV out of the stream as well + int gap = (int)(originalRange.Offset - (contentRange?.Start ?? 0)) + - (ivInStream ? EncryptionConstants.EncryptionBlockSize : 0); + if (gap > 0) + { + // throw away initial bytes we want to trim off; stream cannot seek into future + if (async) + { + await plaintext.ReadAsync(new byte[gap], 0, gap, cancellationToken).ConfigureAwait(false); + } + else + { + plaintext.Read(new byte[gap], 0, gap); + } + } + + if (originalRange.Length.HasValue) + { + plaintext = new WindowStream(plaintext, originalRange.Length.Value); + } + + return plaintext; + } + + internal static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata metadata) + { + if (metadata == default) + { + return default; + } + if (!metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string encryptedDataString)) + { + return default; + } + + EncryptionData encryptionData = EncryptionData.Deserialize(encryptedDataString); + + _ = encryptionData.ContentEncryptionIV ?? throw EncryptionErrors.MissingEncryptionMetadata( + nameof(EncryptionData.ContentEncryptionIV)); + _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw EncryptionErrors.MissingEncryptionMetadata( + nameof(EncryptionData.WrappedContentKey.EncryptedKey)); + + return encryptionData; + } + + /// + /// Gets whether to ignore padding options for decryption. + /// + /// Downloaded content range. + /// True if we should ignore padding. + /// + /// If the last cipher block of the blob was returned, we need the padding. Otherwise, we can ignore it. + /// + private static bool CanIgnorePadding(ContentRange? contentRange) + { + // if Content-Range not present, we requested the whole blob + if (!contentRange.HasValue) + { + return false; + } + + // if range is wildcard, we requested the whole blob + if (!contentRange.Value.End.HasValue) + { + return false; + } + + // blob storage will always return ContentRange.Size + // we don't have to worry about the impossible decision of what to do if it doesn't + + // did we request the last block? + // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block + if (contentRange.Value.Size - contentRange.Value.End == 1) + { + return false; + } + + return true; + } } /// @@ -2977,6 +3120,7 @@ public static BlobBaseClient GetBlobBaseClient( client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, + client.ClientSideEncryption, client.EncryptionScope); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index 21994d631e8f2..75124926da863 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -11,6 +11,9 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Metadata = System.Collections.Generic.IDictionary; +using Azure.Storage.Cryptography.Models; +using System.Collections.Generic; +using Azure.Storage.Cryptography; namespace Azure.Storage.Blobs { @@ -162,6 +165,7 @@ public BlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions opt /// /// Client diagnostics. /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. internal BlobClient( Uri blobUri, @@ -169,8 +173,9 @@ internal BlobClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, clientSideEncryption, encryptionScope) { } #endregion ctors @@ -946,6 +951,12 @@ internal async Task> StagedUploadAsync( bool async = true, CancellationToken cancellationToken = default) { + if (UsingClientSideEncryption) + { + // content is now unseekable, so PartitionedUploader will be forced to do a buffered multipart upload + (content, metadata) = await ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false); + } + var client = new BlockBlobClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); PartitionedUploader uploader = new PartitionedUploader( @@ -1020,6 +1031,26 @@ internal async Task> StagedUploadAsync( bool async = true, CancellationToken cancellationToken = default) { + // TODO uncomment when upload from file gets its own implementation + //// if clientside encryption, upload from stream, where our crypto logic is + //if (ClientSideEncryption != default) + //{ + // using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read)) + // { + // return await StagedUploadAsync( + // stream, + // blobHttpHeaders, + // metadata, + // conditions, + // progressHandler, + // accessTier, + // transferOptions: transferOptions, + // async: async, + // cancellationToken: cancellationToken) + // .ConfigureAwait(false); + // } + //} + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read)) { return await StagedUploadAsync( @@ -1037,20 +1068,54 @@ internal async Task> StagedUploadAsync( } #endregion Upload - // NOTE: TransformContent is no longer called by the new implementation - // of parallel upload. Leaving the virtual stub in for now to avoid - // any confusion. Will need to be added back for encryption work per - // #7127. - /// - /// Performs a transform on the data for uploads. It is a no-op by default. + /// Applies client-side encryption to the data for upload. /// - /// Content to transform. - /// Content metadata to transform. + /// + /// Content to encrypt. + /// + /// + /// Metadata to add encryption metadata to. + /// + /// + /// Whether to perform this operation asynchronously. + /// + /// + /// Cancellation token. + /// /// Transformed content stream and metadata. - internal virtual (Stream, Metadata) TransformContent(Stream content, Metadata metadata) + private async Task<(Stream, Metadata)> ClientSideEncryptInternal( + Stream content, + Metadata metadata, + bool async, + CancellationToken cancellationToken) { - return (content, metadata); // no-op + if (ClientSideEncryption?.KeyEncryptionKey == default || ClientSideEncryption?.KeyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(ClientSideEncryption.KeyEncryptionKey), nameof(ClientSideEncryption.KeyWrapAlgorithm)); + } + + //long originalLength = content.Length; + + (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await Utility.EncryptInternal( + content, + ClientSideEncryption.KeyEncryptionKey, + ClientSideEncryption.KeyWrapAlgorithm, + async, + cancellationToken).ConfigureAwait(false); + + //Stream seekableCiphertext = new RollingBufferStream( + // nonSeekableCiphertext, + // EncryptionConstants.DefaultRollingBufferSize, + // GetExpectedCryptoStreamLength(originalLength)); + + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + metadata.Add(EncryptionConstants.EncryptionDataKey, encryptionData.Serialize()); + + return (nonSeekableCiphertext, metadata); } + + private static long GetExpectedCryptoStreamLength(long originalLength) + => originalLength + (EncryptionConstants.EncryptionBlockSize - originalLength % EncryptionConstants.EncryptionBlockSize); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index 26eb63f5cf05d..e6f39e9204a63 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -69,6 +69,10 @@ public enum ServiceVersion /// public Uri GeoRedundantSecondaryUri { get; set; } + #region Advanced Options + internal ClientSideEncryptionOptions _clientSideEncryptionOptions; + #endregion + /// /// Initializes a new instance of the /// class. diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs index 17003636389f5..d24c5dedf3044 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs @@ -11,6 +11,7 @@ using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Blobs @@ -92,6 +93,16 @@ public class BlobContainerClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The to be used when sending requests. /// @@ -287,6 +298,7 @@ internal BlobContainerClient(Uri blobContainerUri, HttpPipelinePolicy authentica _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -309,6 +321,7 @@ internal BlobContainerClient(Uri blobContainerUri, HttpPipelinePolicy authentica /// /// /// Customer provided key. + /// /// Encryption scope. internal BlobContainerClient( Uri containerUri, @@ -316,6 +329,7 @@ internal BlobContainerClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) { _uri = containerUri; @@ -323,6 +337,7 @@ internal BlobContainerClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -350,7 +365,14 @@ internal BlobContainerClient( /// protected static BlobContainerClient CreateClient(Uri containerUri, BlobClientOptions options, HttpPipeline pipeline) { - return new BlobContainerClient(containerUri, pipeline, options.Version, new ClientDiagnostics(options), null, null); + return new BlobContainerClient( + containerUri, + pipeline, + options.Version, + new ClientDiagnostics(options), + customerProvidedKey: null, + clientSideEncryption: null, + encryptionScope: null); } #endregion ctor @@ -363,7 +385,7 @@ protected static BlobContainerClient CreateClient(Uri containerUri, BlobClientOp /// The name of the blob. /// A new instance. public virtual BlobClient GetBlobClient(string blobName) => - new BlobClient(Uri.AppendToPath(blobName), _pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + new BlobClient(Uri.AppendToPath(blobName), _pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); /// /// Sets the various name fields if they are currently null. diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs index 59ea8716fec12..43dcb30b0149e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs @@ -8,6 +8,7 @@ using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Blobs @@ -82,6 +83,16 @@ public class BlobServiceClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The name of the Encryption Scope to be used when sending request. /// @@ -163,6 +174,7 @@ public BlobServiceClient(string connectionString, BlobClientOptions options) _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -252,6 +264,7 @@ internal BlobServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Bl options?.Version ?? BlobClientOptions.LatestVersion, new ClientDiagnostics(options), options?.CustomerProvidedKey, + options?._clientSideEncryptionOptions?.Clone(), options?.EncryptionScope, options.Build(authentication)) { @@ -277,6 +290,7 @@ internal BlobServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Bl /// diagnostic scopes every request. /// /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. /// /// The transport pipeline used to send every request. @@ -287,6 +301,7 @@ internal BlobServiceClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope, HttpPipeline pipeline) { @@ -296,12 +311,15 @@ internal BlobServiceClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); } /// + /// Intended for DataLake to create a backing blob client. + /// /// Initializes a new instance of the /// class. /// @@ -330,7 +348,15 @@ protected static BlobServiceClient CreateClient( HttpPipelinePolicy authentication, HttpPipeline pipeline) { - return new BlobServiceClient(serviceUri, authentication, options.Version, new ClientDiagnostics(options), null, null, pipeline); + return new BlobServiceClient( + serviceUri, + authentication, + options.Version, + new ClientDiagnostics(options), + customerProvidedKey: null, + clientSideEncryption: null, + encryptionScope: null, + pipeline); } #endregion ctors @@ -347,7 +373,7 @@ protected static BlobServiceClient CreateClient( /// A for the desired container. /// public virtual BlobContainerClient GetBlobContainerClient(string blobContainerName) => - new BlobContainerClient(Uri.AppendToPath(blobContainerName), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + new BlobContainerClient(Uri.AppendToPath(blobContainerName), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); #region protected static accessors for Azure.Storage.Blobs.Batch /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 1e2969d21d360..796217ff44054 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -243,7 +243,14 @@ internal BlockBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } @@ -1542,13 +1549,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static BlockBlobClient GetBlockBlobClient( this BlobContainerClient client, - string blobName) => - new BlockBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + return new BlockBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs new file mode 100644 index 0000000000000..da712dd19c871 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Text; + +namespace Azure.Storage.Blobs.Models +{ + internal struct ContentRange + { + private const string WildcardMarker = "*"; + + public struct RangeUnit + { + internal const string BytesValue = "bytes"; + + private readonly string _value; + + /// + /// Initializes a new instance of the structure. + /// + /// The string value of the instance. + public RangeUnit(string value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// AES-CBC using a 256 bit key. + /// + public static RangeUnit Bytes { get; } = new RangeUnit(BytesValue); + + /// + /// Determines if two values are the same. + /// + /// The first to compare. + /// The second to compare. + /// True if and are the same; otherwise, false. + public static bool operator ==(RangeUnit left, RangeUnit right) => left.Equals(right); + + /// + /// Determines if two values are different. + /// + /// The first to compare. + /// The second to compare. + /// True if and are different; otherwise, false. + public static bool operator !=(RangeUnit left, RangeUnit right) => !left.Equals(right); + + /// + /// Converts a string to a . + /// + /// The string value to convert. + public static implicit operator RangeUnit(string value) => new RangeUnit(value); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => obj is RangeUnit other && Equals(other); + + /// + public bool Equals(RangeUnit other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => _value?.GetHashCode() ?? 0; + + /// + public override string ToString() => _value; + } + + /// + /// Inclusive index where the range starts, measured in this instance's . + /// + public long? Start { get; } + + /// + /// Inclusive index where the range ends, measured in this instance's . + /// + public long? End { get; } + + /// + /// Size of this range, measured in this instance's . + /// + public long? Size { get; } + + /// + /// Unit this range is measured in. Generally "bytes". + /// + public RangeUnit Unit { get; } + + public ContentRange(RangeUnit unit, long? start, long? end, long? size) + { + Start = start; + End = end; + Size = size; + Unit = unit; + } + + public static ContentRange Parse(string headerValue) + { + /* Parse header value (e.g. " -/") + * Either side of the "/" can be an asterisk, so possible results include: + * [, , , ] + * [, "*", ] + * [, , , "*"] + * [, "*", "*"] (unsure if possible but not hard to support) + * "End" is the inclusive last byte; e.g. header "bytes 0-7/8" is the entire 8-byte blob + */ + var tokens = headerValue.Split(new char[] { ' ', '-', '/' }); // ["bytes", "", "", ""] + string unit = default; + long? start = default; + long? end = default; + long? size = default; + + try + { + // unit always first and always present + unit = tokens[0]; + + int blobSizeIndex; + if (tokens[1] == WildcardMarker) // [, "*", ( | "*")] + { + blobSizeIndex = 2; + } + else // [, , , ( | "*")] + { + blobSizeIndex = 3; + + start = int.Parse(tokens[1], CultureInfo.InvariantCulture); + end = int.Parse(tokens[2], CultureInfo.InvariantCulture); + } + + var rawSize = tokens[blobSizeIndex]; + if (rawSize != WildcardMarker) + { + size = int.Parse(rawSize, CultureInfo.InvariantCulture); + } + + return new ContentRange(unit, start, end, size); + } + catch (IndexOutOfRangeException) + { + throw Errors.ParsingHttpRangeFailed(); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index f324d547918c5..24843f1b27e6c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -188,7 +188,14 @@ internal PageBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } #endregion ctors diff --git a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj index 535e1e5a6f15d..e7a8aedd3d7e7 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj @@ -8,9 +8,13 @@ + + + + + - diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs index 6ec79b4460f8b..3b6690b820d10 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs @@ -225,6 +225,23 @@ public BlobServiceClient GetServiceClient_BlobServiceSas_Snapshot( new Uri($"{TestConfigDefault.BlobServiceEndpoint}?{sasCredentials ?? GetNewBlobServiceSasCredentialsSnapshot(containerName: containerName, blobName: blobName, snapshot: snapshot, sharedKeyCredentials: sharedKeyCredentials ?? GetNewSharedKeyCredentials())}"), GetOptions())); + public Security.KeyVault.Keys.KeyClient GetKeyClient_TargetKeyClient() + => GetKeyClient(TestConfigurations.DefaultTargetKeyVault); + + public TokenCredential GetTokenCredential_TargetKeyClient() + => GetKeyClientTokenCredential(TestConfigurations.DefaultTargetKeyVault); + + private static Security.KeyVault.Keys.KeyClient GetKeyClient(KeyVaultConfiguration config) + => new Security.KeyVault.Keys.KeyClient( + new Uri(config.VaultEndpoint), + GetKeyClientTokenCredential(config)); + + private static TokenCredential GetKeyClientTokenCredential(KeyVaultConfiguration config) + => new Identity.ClientSecretCredential( + config.ActiveDirectoryTenantId, + config.ActiveDirectoryApplicationId, + config.ActiveDirectoryApplicationSecret); + public async Task GetTestContainerAsync( BlobServiceClient service = default, string containerName = default, diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs new file mode 100644 index 0000000000000..7c2c85cfe1f72 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -0,0 +1,388 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Core.TestFramework; +using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Test.Shared; +using Azure.Storage.Tests.Shared; +using NUnit.Framework; + +namespace Azure.Storage.Blobs.Test +{ + public class ClientSideEncryptionTests : BlobTestBase + { + private const string ThrowawayAlgorithmName = "blah"; + public ClientSideEncryptionTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) + : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) + { + } + + + #region Utility + + private byte[] LocalManualEncryption(byte[] data, byte[] key, byte[] iv) + { + using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) + using (var encryptor = aesProvider.CreateEncryptor()) + using (var memStream = new MemoryStream()) + using (var cryptoStream = new CryptoStream(memStream, encryptor, CryptoStreamMode.Write)) + { + cryptoStream.Write(data, 0, data.Length); + cryptoStream.FlushFinalBlock(); + return memStream.ToArray(); + } + } + + private async Task GetKeyvaultIKeyEncryptionKey() + { + var keyClient = GetKeyClient_TargetKeyClient(); + Security.KeyVault.Keys.KeyVaultKey key = await keyClient.CreateRsaKeyAsync( + new Security.KeyVault.Keys.CreateRsaKeyOptions($"CloudRsaKey-{Guid.NewGuid()}", false)); + return new CryptographyClient(key.Id, GetTokenCredential_TargetKeyClient()); + } + + private async Task GetTestContainerEncryptionAsync( + ClientSideEncryptionOptions encryptionOptions, + string containerName = default, + IDictionary metadata = default) + { + // normally set through property on subclass; this is easier to hook up in current test infra with internals access + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; + + containerName ??= GetNewContainerName(); + var service = GetServiceClient_SharedKey(options); + + BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(containerName)); + await container.CreateAsync(metadata: metadata); + return new DisposingContainer(container); + } + + #endregion + + [TestCase(16)] // a single cipher block + [TestCase(14)] // a single unalligned cipher block + [TestCase(Constants.KB)] // multiple blocks + [TestCase(Constants.KB - 4)] // multiple unalligned blocks + [TestCase(Constants.MB)] // larger test, increasing likelihood to trigger async extension usage bugs + [LiveOnly] // cannot seed content encryption key + public async Task UploadAsync(long dataSize) + { + var data = GetRandomBuffer(dataSize); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var blobName = GetNewBlobName(); + var blob = disposable.Container.GetBlobClient(blobName); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data)); + + // download without decrypting + var encryptedDataStream = new MemoryStream(); + await new BlobClient(blob.Uri, GetNewSharedKeyCredentials()).DownloadToAsync(encryptedDataStream); + var encryptedData = encryptedDataStream.ToArray(); + + // encrypt original data manually for comparison + if (!(await blob.GetPropertiesAsync()).Value.Metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string serialEncryptionData)) + { + Assert.Fail("No encryption metadata present."); + } + EncryptionData encryptionMetadata = EncryptionDataSerializer.Deserialize(serialEncryptionData); + Assert.NotNull(encryptionMetadata, "Never encrypted data."); + byte[] expectedEncryptedData = LocalManualEncryption( + data, + (await mockKey.UnwrapKeyAsync(null, encryptionMetadata.WrappedContentKey.EncryptedKey) + .ConfigureAwait(false)).ToArray(), + encryptionMetadata.ContentEncryptionIV); + + // compare data + Assert.AreEqual(expectedEncryptedData, encryptedData); + } + } + + [TestCase(16)] // a single cipher block + [TestCase(14)] // a single unalligned cipher block + [TestCase(Constants.KB)] // multiple blocks + [TestCase(Constants.KB - 4)] // multiple unalligned blocks + [LiveOnly] // cannot seed content encryption key + public async Task RoundtripAsync(long dataSize) + { + var data = GetRandomBuffer(dataSize); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data)); + + // download with decryption + byte[] downloadData; + using (var stream = new MemoryStream()) + { + await blob.DownloadToAsync(stream); + downloadData = stream.ToArray(); + } + + // compare data + Assert.AreEqual(data, downloadData); + } + } + + [Test] // multiple unalligned blocks + [LiveOnly] // cannot seed content encryption key + public async Task KeyResolverKicksIn() + { + var data = GetRandomBuffer(Constants.KB); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + string blobName = GetNewBlobName(); + // upload with encryption + await disposable.Container.GetBlobClient(blobName).UploadAsync(new MemoryStream(data)); + + // download with decryption and no cached key + byte[] downloadData; + using (var stream = new MemoryStream()) + { + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKey + }; + await new BlobContainerClient(disposable.Container.Uri, GetNewSharedKeyCredentials(), options).GetBlobClient(blobName).DownloadToAsync(stream); + downloadData = stream.ToArray(); + } + + // compare data + Assert.AreEqual(data, downloadData); + } + } + + [TestCase(0, 16)] // first block + [TestCase(16, 16)] // not first block + [TestCase(32, 32)] // multiple blocks; IV not at blob start + [TestCase(16, 17)] // overlap end of block + [TestCase(32, 17)] // overlap end of block; IV not at blob start + [TestCase(15, 17)] // overlap beginning of block + [TestCase(31, 17)] // overlap beginning of block; IV not at blob start + [TestCase(15, 18)] // overlap both sides + [TestCase(31, 18)] // overlap both sides; IV not at blob start + [TestCase(16, null)] + [LiveOnly] // cannot seed content encryption key + public async Task PartialDownloadAsync(int offset, int? count) + { + var data = GetRandomBuffer(offset + (count ?? 16) + 32); // ensure we have enough room in original data + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data)); + + // download range with decryption + byte[] downloadData; // no overload that takes Stream and HttpRange; we must buffer read + Stream downloadStream = (await blob.DownloadAsync(new HttpRange(offset, count))).Value.Content; + byte[] buffer = new byte[Constants.KB]; + using (MemoryStream stream = new MemoryStream()) + { + int read; + while ((read = downloadStream.Read(buffer, 0, buffer.Length)) > 0) + { + stream.Write(buffer, 0, read); + } + downloadData = stream.ToArray(); + } + + // compare range of original data to downloaded data + var slice = data.Skip(offset); + slice = count.HasValue + ? slice.Take(count.Value) + : slice; + var sliceArray = slice.ToArray(); + Assert.AreEqual(sliceArray, downloadData); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task Track2DownloadTrack1Blob() + { + var data = GetRandomBuffer(Constants.KB); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var track2Blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + // upload with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Blob = new Microsoft.Azure.Storage.Blob.CloudBlockBlob( + track2Blob.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + await track1Blob.UploadFromByteArrayAsync( + data, 0, data.Length, default, + new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, mockKey) + }, + default, default); + + // download with track 2 + var downloadStream = new MemoryStream(); + await track2Blob.DownloadToAsync(downloadStream); + + // compare original data to downloaded data + Assert.AreEqual(data, downloadStream.ToArray()); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task Track1DownloadTrack2Blob() + { + var data = GetRandomBuffer(Constants.KB); // ensure we have enough room in original data + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var track2Blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + // upload with track 2 + await track2Blob.UploadAsync(new MemoryStream(data)); + + // download with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Blob = new Microsoft.Azure.Storage.Blob.CloudBlockBlob( + track2Blob.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var downloadData = new byte[data.Length]; + await track1Blob.DownloadToByteArrayAsync(downloadData, 0, default, + new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, mockKey) + }, + default, default); + + // compare original data to downloaded data + Assert.AreEqual(data, downloadData); + } + } + + [Test] + [LiveOnly] // need access to keyvault service && cannot seed content encryption key + public async Task RoundtripWithKeyvaultProvider() + { + var data = GetRandomBuffer(Constants.KB); + IKeyEncryptionKey key = await GetKeyvaultIKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = key, + KeyWrapAlgorithm = "RSA-OAEP-256" + })) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + await blob.UploadAsync(new MemoryStream(data)); + + var downloadStream = new MemoryStream(); + await blob.DownloadToAsync(downloadStream); + + Assert.AreEqual(data, downloadStream.ToArray()); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + [Ignore("stress test")] + public async Task StressAsync() + { + static async Task RoundTripDataHelper(BlobClient client, byte[] data) + { + using (var dataStream = new MemoryStream(data)) + { + await client.UploadAsync(dataStream); + } + + using (var downloadStream = new MemoryStream()) + { + await client.DownloadToAsync(downloadStream); + return downloadStream.ToArray(); + } + } + + var data = GetRandomBuffer(10 * Constants.MB); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var downloadTasks = new List>(); + foreach (var _ in Enumerable.Range(0, 10)) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + downloadTasks.Add(RoundTripDataHelper(blob, data)); + } + + var downloads = await Task.WhenAll(downloadTasks); + + foreach (byte[] downloadData in downloads) + { + Assert.AreEqual(data, downloadData); + } + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs b/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs new file mode 100644 index 0000000000000..e8afa156876f5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Core; +using Azure.Storage.Blobs.Specialized; + +namespace Azure.Storage.Blobs.Cryptography.Tests +{ + internal static class CryptoraphyTestsExtensionMethods + { + + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs deleted file mode 100644 index f0b1d054a9005..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.IO; -using System.Threading.Tasks; -using Azure.Core.TestFramework; -using Azure.Storage.Blobs.Specialized; -using Azure.Storage.Test.Shared; -using NUnit.Framework; - -namespace Azure.Storage.Blobs.Test -{ - public class EncryptedBlockBlobClientTests : BlobTestBase - { - public EncryptedBlockBlobClientTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) - : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) - { - } - - // Placeholder test to verify things work end-to-end - [Test] - public async Task DeleteAsync() - { - await using DisposingContainer test = await GetTestContainerAsync(); - - // First upload a regular block blob - var blobName = GetNewBlobName(); - var blob = InstrumentClient(test.Container.GetBlockBlobClient(blobName)); - var data = GetRandomBuffer(Constants.KB); - using var stream = new MemoryStream(data); - await blob.UploadAsync(stream); - - // Create an EncryptedBlockBlobClient pointing at the same blob - var encryptedBlob = InstrumentClient( - new EncryptedBlockBlobClient( - blob.Uri, - new StorageSharedKeyCredential( - TestConfigDefault.AccountName, - TestConfigDefault.AccountKey), - GetOptions())); - - // Delete the blob - var response = await encryptedBlob.DeleteAsync(); - Assert.IsNotNull(response.Headers.RequestId); - } - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs b/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs new file mode 100644 index 0000000000000..94bee871d2bef --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Microsoft.Azure.KeyVault.Core; + +namespace Azure.Storage.Tests.Shared +{ + /// + /// Mock for a key encryption key. Not meant for production use. + /// + internal class MockKeyEncryptionKey : IKeyEncryptionKey, IKeyEncryptionKeyResolver, + IKey, IKeyResolver // for track 1 compatibility tests + { + public ReadOnlyMemory KeyEncryptionKey { get; } + + public string KeyId { get; } + + #region Counters + public int WrappedSync { get; private set; } + public int WrappedAsync { get; private set; } + public int UnwrappedSync { get; private set; } + public int UnwrappedAsync { get; private set; } + public int ResolvedSync { get; private set; } + public int ResolvedAsync { get; private set; } + + public void ResetCounters() + { + WrappedSync = 0; + WrappedAsync = 0; + UnwrappedSync = 0; + UnwrappedAsync = 0; + ResolvedSync = 0; + ResolvedAsync = 0; + } + #endregion + + /// + /// Generates a key encryption key with the given properties. + /// + public MockKeyEncryptionKey(int keySizeBits = 256, string keyId = default) + { + KeyId = keyId ?? Guid.NewGuid().ToString(); + using (var random = new RNGCryptoServiceProvider()) + { + var bytes = new byte[keySizeBits >> 3]; + random.GetBytes(bytes); + KeyEncryptionKey = bytes; + } + } + + public byte[] UnwrapKey(string algorithm, ReadOnlyMemory encryptedKey, CancellationToken cancellationToken = default) + { + UnwrappedSync++; + return Xor(encryptedKey.ToArray(), KeyEncryptionKey.ToArray()); + } + + public Task UnwrapKeyAsync(string algorithm, ReadOnlyMemory encryptedKey, CancellationToken cancellationToken = default) + { + UnwrappedAsync++; + return Task.FromResult(Xor(encryptedKey.ToArray(), KeyEncryptionKey.ToArray())); + } + + public byte[] WrapKey(string algorithm, ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + WrappedSync++; + return Xor(key.ToArray(), KeyEncryptionKey.ToArray()); + } + + public Task WrapKeyAsync(string algorithm, ReadOnlyMemory key, CancellationToken cancellationToken = default) + { + WrappedAsync++; + return Task.FromResult(Xor(key.ToArray(), KeyEncryptionKey.ToArray())); + } + + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new BitArray(a); + var bBits = new BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + + public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationToken = default) + { + if (keyId != this.KeyId.ToString()) + { + throw new ArgumentException("Mock key resolver cannot find this keyId."); + } + + ResolvedSync++; + return this; + } + + public Task ResolveAsync(string keyId, CancellationToken cancellationToken = default) + { + if (keyId != this.KeyId.ToString()) + { + throw new ArgumentException("Mock key resolver cannot find this keyId."); + } + + ResolvedAsync++; + return Task.FromResult((IKeyEncryptionKey)this); + } + + #region Track 1 Impl + + string IKey.DefaultEncryptionAlgorithm => throw new NotImplementedException(); + + string IKey.DefaultKeyWrapAlgorithm => throw new NotImplementedException(); + + string IKey.DefaultSignatureAlgorithm => throw new NotImplementedException(); + + string IKey.Kid => KeyId; + + Task IKey.DecryptAsync(byte[] ciphertext, byte[] iv, byte[] authenticationData, byte[] authenticationTag, string algorithm, CancellationToken token) + { + throw new NotImplementedException(); + } + + Task> IKey.EncryptAsync(byte[] plaintext, byte[] iv, byte[] authenticationData, string algorithm, CancellationToken token) + { + throw new NotImplementedException(); + } + + async Task> IKey.WrapKeyAsync(byte[] key, string algorithm, CancellationToken token) + => new Tuple((await WrapKeyAsync(algorithm, key, token)), null); + + async Task IKey.UnwrapKeyAsync(byte[] encryptedKey, string algorithm, CancellationToken token) + => await UnwrapKeyAsync(algorithm, encryptedKey, token); + + Task> IKey.SignAsync(byte[] digest, string algorithm, CancellationToken token) + { + throw new NotImplementedException(); + } + + Task IKey.VerifyAsync(byte[] digest, byte[] signature, string algorithm, CancellationToken token) + { + throw new NotImplementedException(); + } + + void IDisposable.Dispose() + { + // no-op + } + + async Task IKeyResolver.ResolveKeyAsync(string kid, CancellationToken token) + => (MockKeyEncryptionKey)await ResolveAsync(kid, token); // we know we returned `this`; + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs b/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs new file mode 100644 index 0000000000000..534feb1877b92 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; + +namespace Azure.Storage.Blobs.Cryptography.Tests +{ + /// + /// Read-only, unseekable Stream where the 0-indexed nth byte read will have the value n. + /// + internal class MockStream : Stream + { + private byte _next = 0; + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _next; + + public override long Position { get => _next; set => throw new NotSupportedException(); } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) + { + for (int i = offset; i < offset + count; i++) + { + buffer[i] = _next++; + } + return count; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs index 1b8cf3faa6ddb..2585c1acda246 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs @@ -3,7 +3,6 @@ using System; using Azure.Storage.Sas; -using Azure.Storage.Test.Shared; using NUnit.Framework; namespace Azure.Storage.Blobs.Test diff --git a/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj b/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj index 91aa97b8b54f5..ed768db4babec 100644 --- a/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj +++ b/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj @@ -21,6 +21,7 @@ + diff --git a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs new file mode 100644 index 0000000000000..b209b1ff28bde --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage +{ +#pragma warning disable CA1707 // Identifiers should not contain underscores + /// + /// The version of clientside encryption to use. + /// + public enum ClientSideEncryptionVersion + { + /// + /// 1.0 + /// + V1_0 + } +#pragma warning restore CA1707 // Identifiers should not contain underscores +} diff --git a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs new file mode 100644 index 0000000000000..a51d569f4fb26 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Core.Cryptography; + +namespace Azure.Storage +{ + /// + /// Provides the client configuration options for connecting to Azure Blob using clientside encryption. + /// + public class ClientSideEncryptionOptions + { + /// + /// The version of clientside encryption to use. + /// + public ClientSideEncryptionVersion Version { get; } + + /// + /// Required for upload operations. + /// The key used to wrap the generated content encryption key. + /// For more information, see https://docs.microsoft.com/en-us/azure/storage/common/storage-client-side-encryption. + /// + public IKeyEncryptionKey KeyEncryptionKey { get; set; } + + /// + /// Required for download operations. + /// Fetches the correct key encryption key to unwrap the downloaded content encryption key. + /// For more information, see https://docs.microsoft.com/en-us/azure/storage/common/storage-client-side-encryption. + /// + public IKeyEncryptionKeyResolver KeyResolver { get; set; } + + /// + /// Required for upload operations. + /// The algorithm identifier to use when wrapping the content encryption key. This is passed into + /// + /// and its async counterpart. + /// + public string KeyWrapAlgorithm { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The version of clientside encryption to use. + public ClientSideEncryptionOptions(ClientSideEncryptionVersion version) + { + Version = version; + } + + /// + /// Copy constructor to keep these options grouped in clients while stopping users from + /// accidentally altering our configs out from under us. + /// + /// + internal ClientSideEncryptionOptions(ClientSideEncryptionOptions other) + { + Version = other.Version; + KeyEncryptionKey = other.KeyEncryptionKey; + KeyResolver = other.KeyResolver; + KeyWrapAlgorithm = other.KeyWrapAlgorithm; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs new file mode 100644 index 0000000000000..d284a5dd4cb9a --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + internal static class ClientSideEncryptionVersionExtensions + { + public static class ClientSideEncryptionVersionString + { + public const string V1_0 = "1.0"; + } + + public static string Serialize(this ClientSideEncryptionVersion version) + { + switch (version) + { + case ClientSideEncryptionVersion.V1_0: + return ClientSideEncryptionVersionString.V1_0; + default: + // sanity check; serialize is in this file to make it easy to add the serialization cases + throw Errors.ClientSideEncryptionVersionNotSupported(); + } + } + + public static ClientSideEncryptionVersion ToClientSideEncryptionVersion(this string versionString) + { + switch (versionString) + { + case ClientSideEncryptionVersionString.V1_0: + return ClientSideEncryptionVersion.V1_0; + default: + // This library doesn't support the stated encryption version + throw Errors.ClientSideEncryptionVersionNotSupported(versionString); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs new file mode 100644 index 0000000000000..679a39e5b05d2 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography +{ + internal static class EncryptionConstants + { + public const ClientSideEncryptionVersion CurrentVersion = ClientSideEncryptionVersion.V1_0; + + public const string AgentMetadataKey = "EncryptionLibrary"; + + public const string AesCbcPkcs5Padding = "AES/CBC/PKCS5Padding"; + + public const string AesCbcNoPadding = "AES/CBC/NoPadding"; + + public const string Aes = "AES"; + + public const string EncryptionDataKey = "encryptiondata"; + + public const string EncryptionMode = "FullBlob"; + + public const int EncryptionBlockSize = 16; + + public const int EncryptionKeySizeBits = 256; + + public const string XMsRange = "x-ms-range"; + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs new file mode 100644 index 0000000000000..48dccc622fa58 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Storage.Cryptography +{ + internal static class EncryptionErrors + { + public static ArgumentException KeyNotFound(string keyId) + => new ArgumentException($"Could not resolve key of id `{keyId}`."); + + public static ArgumentException BadEncryptionAgent(string agent) + => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + + $"the Encryption Agent protocol \"{agent}\" set on the blob."); + + public static ArgumentException BadEncryptionAlgorithm(string algorithm) + => new ArgumentException($"Invalid Encryption Algorithm \"{algorithm}\" found on the resource. This version of the client" + + "library does not support the given encryption algorithm."); + + public static ArgumentException NoKeyAccessor() + => new ArgumentException("No key to decrypt data with."); + + public static InvalidOperationException MissingEncryptionMetadata(string field) + => new InvalidOperationException($"Missing field \"{field}\" in encryption metadata"); + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs new file mode 100644 index 0000000000000..f54bacc82a6c7 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.ComponentModel; + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Specifies the encryption algorithm used to encrypt and decrypt a blob. + /// + internal readonly struct ClientSideEncryptionAlgorithm + { + internal const string AesCbc256Value = "AES_CBC_256"; + + private readonly string _value; + + /// + /// Initializes a new instance of the structure. + /// + /// The string value of the instance. + public ClientSideEncryptionAlgorithm(string value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// AES-CBC using a 256 bit key. + /// + public static ClientSideEncryptionAlgorithm AesCbc256 { get; } = new ClientSideEncryptionAlgorithm(AesCbc256Value); + + /// + /// Determines if two values are the same. + /// + /// The first to compare. + /// The second to compare. + /// True if and are the same; otherwise, false. + public static bool operator ==(ClientSideEncryptionAlgorithm left, ClientSideEncryptionAlgorithm right) => left.Equals(right); + + /// + /// Determines if two values are different. + /// + /// The first to compare. + /// The second to compare. + /// True if and are different; otherwise, false. + public static bool operator !=(ClientSideEncryptionAlgorithm left, ClientSideEncryptionAlgorithm right) => !left.Equals(right); + + /// + /// Converts a string to a . + /// + /// The string value to convert. + public static implicit operator ClientSideEncryptionAlgorithm(string value) => new ClientSideEncryptionAlgorithm(value); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => obj is ClientSideEncryptionAlgorithm other && Equals(other); + + /// + public bool Equals(ClientSideEncryptionAlgorithm other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => _value?.GetHashCode() ?? 0; + + /// + public override string ToString() => _value; + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs new file mode 100644 index 0000000000000..e4258c647338b --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Storage.Cryptography; + +namespace Azure.Storage.Blobs.Specialized.Models +{ + /// + /// This is a representation of a range of bytes on an encrypted blob, which may be expanded from the requested + /// range to include extra data needed for decryption. It contains the original range as well as the calculated + /// expanded range. + /// + internal struct EncryptedBlobRange + { + /// + /// The original blob range requested by the user. + /// + public HttpRange OriginalRange { get; } + + /// + /// The blob range to actually request from the service that will allow + /// decryption of the original range. + /// + public HttpRange AdjustedRange { get; } + + public EncryptedBlobRange(HttpRange originalRange) + { + OriginalRange = originalRange; + + int offsetAdjustment = 0; + long? adjustedDownloadCount = originalRange.Length; + + // Calculate offsetAdjustment. + if (OriginalRange.Offset != 0) + { + // Align with encryption block boundary. + int diff; + if ((diff = (int)(OriginalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) + { + offsetAdjustment += diff; + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += diff; + } + } + + // Account for IV. + if (OriginalRange.Offset >= EncryptionConstants.EncryptionBlockSize) + { + offsetAdjustment += EncryptionConstants.EncryptionBlockSize; + // Increment adjustedDownloadCount if necessary. + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; + } + } + } + + // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible + // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. + if (adjustedDownloadCount != null) + { + adjustedDownloadCount += ( + EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount + % EncryptionConstants.EncryptionBlockSize) + ) % EncryptionConstants.EncryptionBlockSize; + } + + AdjustedRange = new HttpRange(OriginalRange.Offset - offsetAdjustment, adjustedDownloadCount); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs new file mode 100644 index 0000000000000..347a3e8ff304a --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the encryption agent stored on the service. + /// + internal class EncryptionAgent + { + /// + /// The protocol version used for encryption. + /// + public ClientSideEncryptionVersion Protocol { get; set; } + + /// + /// The algorithm used for encryption. + /// + public ClientSideEncryptionAlgorithm EncryptionAlgorithm { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs new file mode 100644 index 0000000000000..7c2f8bd0a1437 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the encryption data that is stored on the service. + /// + internal class EncryptionData + { + /// + /// The blob encryption mode. + /// + public string EncryptionMode { get; set; } + + /// + /// A object that stores the wrapping algorithm, key identifier and the encrypted key. + /// + public WrappedKey WrappedContentKey { get; set; } + + /// + /// The encryption agent. + /// + public EncryptionAgent EncryptionAgent { get; set; } + + /// + /// The content encryption IV. + /// + public byte[] ContentEncryptionIV { get; set; } + +#pragma warning disable CA2227 // Collection properties should be read only + /// + /// Metadata for encryption. Currently used only for storing the encryption library, but may contain other data. + /// + public Metadata KeyWrappingMetadata { get; set; } +#pragma warning restore CA2227 // Collection properties should be read only + + /// + /// Serializes this object to JSON. + /// + /// + public string Serialize() + => EncryptionDataSerializer.Serialize(this); + + /// + /// Deserializes an from JSON. + /// + /// JSON to deserialize. + /// + public static EncryptionData Deserialize(string json) + => EncryptionDataSerializer.Deserialize(json); + + internal static async Task CreateInternalV1_0( + byte[] contentEncryptionIv, + string keyWrapAlgorithm, + byte[] contentEncryptionKey, + IKeyEncryptionKey keyEncryptionKey, + bool async, + CancellationToken cancellationToken) + => new EncryptionData() + { + EncryptionMode = EncryptionConstants.EncryptionMode, + ContentEncryptionIV = contentEncryptionIv, + EncryptionAgent = new EncryptionAgent() + { + EncryptionAlgorithm = ClientSideEncryptionAlgorithm.AesCbc256, + Protocol = ClientSideEncryptionVersion.V1_0 + }, + KeyWrappingMetadata = new Dictionary() + { + { EncryptionConstants.AgentMetadataKey, AgentString } + }, + WrappedContentKey = new WrappedKey() + { + Algorithm = keyWrapAlgorithm, + EncryptedKey = async + ? await keyEncryptionKey.WrapKeyAsync(keyWrapAlgorithm, contentEncryptionKey, cancellationToken).ConfigureAwait(false) + : keyEncryptionKey.WrapKey(keyWrapAlgorithm, contentEncryptionKey, cancellationToken), + KeyId = keyEncryptionKey.KeyId + } + }; + + /// + /// Singleton string identifying this encryption library. + /// + private static string AgentString { get; } = new Func(() => + { + Assembly assembly = typeof(EncryptionData).Assembly; + var platformInformation = $"({RuntimeInformation.FrameworkDescription}; {RuntimeInformation.OSDescription})"; + return $"azsdk-net-{assembly.GetName().Name}/{assembly.GetCustomAttribute().InformationalVersion} {platformInformation}"; + }).Invoke(); + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs new file mode 100644 index 0000000000000..a2bf0c418042f --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; + +namespace Azure.Storage.Cryptography.Models +{ + internal static class EncryptionDataSerializer + { + #region Serialize + public static string Serialize(EncryptionData data) + { + return Encoding.UTF8.GetString(SerializeEncryptionData(data).ToArray()); + } + + public static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) + { + var writer = new Core.ArrayBufferWriter(); + using var json = new Utf8JsonWriter(writer); + + json.WriteStartObject(); + WriteEncryptionData(json, data); + json.WriteEndObject(); + + json.Flush(); + return writer.WrittenMemory; + } + + public static void WriteEncryptionData(Utf8JsonWriter json, EncryptionData data) + { + json.WriteString(nameof(data.EncryptionMode), data.EncryptionMode); + + json.WriteStartObject(nameof(data.WrappedContentKey)); + WriteWrappedKey(json, data.WrappedContentKey); + json.WriteEndObject(); + + json.WriteStartObject(nameof(data.EncryptionAgent)); + WriteEncryptionAgent(json, data.EncryptionAgent); + json.WriteEndObject(); + + json.WriteString(nameof(data.ContentEncryptionIV), Convert.ToBase64String(data.ContentEncryptionIV)); + + json.WriteStartObject(nameof(data.KeyWrappingMetadata)); + WriteDictionary(json, data.KeyWrappingMetadata); + json.WriteEndObject(); + } + + private static void WriteWrappedKey(Utf8JsonWriter json, WrappedKey key) + { + json.WriteString(nameof(key.KeyId), key.KeyId); + json.WriteString(nameof(key.EncryptedKey), Convert.ToBase64String(key.EncryptedKey)); + json.WriteString(nameof(key.Algorithm), key.Algorithm); + } + + private static void WriteEncryptionAgent(Utf8JsonWriter json, EncryptionAgent encryptionAgent) + { + json.WriteString(nameof(encryptionAgent.Protocol), encryptionAgent.Protocol.Serialize()); + json.WriteString(nameof(encryptionAgent.EncryptionAlgorithm), encryptionAgent.EncryptionAlgorithm.ToString()); + } + + private static void WriteDictionary(Utf8JsonWriter json, IDictionary dictionary) + { + foreach (var entry in dictionary) + { + json.WriteString(entry.Key, entry.Value); + } + } + #endregion + + #region Deserialize + public static EncryptionData Deserialize(string serializedData) + { + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(serializedData)); + return DeserializeEncryptionData(ref reader); + } + + public static EncryptionData DeserializeEncryptionData(ref Utf8JsonReader reader) + { + using JsonDocument json = JsonDocument.ParseValue(ref reader); + JsonElement root = json.RootElement; + return ReadEncryptionData(root); + } + + public static EncryptionData ReadEncryptionData(JsonElement root) + { + var data = new EncryptionData(); + foreach (var property in root.EnumerateObject()) + { + ReadPropertyValue(data, property); + } + return data; + } + + private static void ReadPropertyValue(EncryptionData data, JsonProperty property) + { + if (property.NameEquals(nameof(data.EncryptionMode))) + { + data.EncryptionMode = property.Value.GetString(); + } + else if (property.NameEquals(nameof(data.WrappedContentKey))) + { + var key = new WrappedKey(); + foreach (var subProperty in property.Value.EnumerateObject()) + { + ReadPropertyValue(key, subProperty); + } + data.WrappedContentKey = key; + } + else if (property.NameEquals(nameof(data.EncryptionAgent))) + { + var agent = new EncryptionAgent(); + foreach (var subProperty in property.Value.EnumerateObject()) + { + ReadPropertyValue(agent, subProperty); + } + data.EncryptionAgent = agent; + } + else if (property.NameEquals(nameof(data.ContentEncryptionIV))) + { + data.ContentEncryptionIV = Convert.FromBase64String(property.Value.GetString()); + } + else if (property.NameEquals(nameof(data.KeyWrappingMetadata))) + { + var metadata = new Dictionary(); + foreach (var entry in property.Value.EnumerateObject()) + { + metadata.Add(entry.Name, entry.Value.GetString()); + } + data.KeyWrappingMetadata = metadata; + } + } + + private static void ReadPropertyValue(WrappedKey key, JsonProperty property) + { + if (property.NameEquals(nameof(key.Algorithm))) + { + key.Algorithm = property.Value.GetString(); + } + else if (property.NameEquals(nameof(key.EncryptedKey))) + { + key.EncryptedKey = Convert.FromBase64String(property.Value.GetString()); + } + else if (property.NameEquals(nameof(key.KeyId))) + { + key.KeyId = property.Value.GetString(); + } + } + + private static void ReadPropertyValue(EncryptionAgent agent, JsonProperty property) + { + if (property.NameEquals(nameof(agent.EncryptionAlgorithm))) + { + agent.EncryptionAlgorithm = new ClientSideEncryptionAlgorithm(property.Value.GetString()); + } + else if (property.NameEquals(nameof(agent.Protocol))) + { + agent.Protocol = property.Value.GetString().ToClientSideEncryptionVersion(); + } + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs new file mode 100644 index 0000000000000..34bcdf7590484 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the envelope key details stored on the service. + /// + internal class WrappedKey + { + /// + /// The key identifier string. + /// + public string KeyId { get; set; } + + /// + /// The encrypted content encryption key. + /// + public byte[] EncryptedKey { get; set; } + + /// + /// The algorithm used for wrapping. + /// + public string Algorithm { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs new file mode 100644 index 0000000000000..2e1825fa22add --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs @@ -0,0 +1,295 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Cryptography +{ + internal static class Utility + { + public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions other) + => new ClientSideEncryptionOptions(other.Version) + { + KeyEncryptionKey = other.KeyEncryptionKey, + KeyResolver = other.KeyResolver, + KeyWrapAlgorithm = other.KeyWrapAlgorithm + }; + + /// + /// Securely generate a key. + /// + /// Key size. + /// The generated key bytes. + public static byte[] CreateKey(int numBits) + { + using (var rng = new RNGCryptoServiceProvider()) + { + var buff = new byte[numBits / 8]; + rng.GetBytes(buff); + return buff; + } + } + + /// + /// Decrypts the given stream if decryption information is provided. + /// Does not shave off unwanted start/end bytes, but will shave off padding. + /// + /// Stream to decrypt. + /// + /// Encryption metadata and wrapped content encryption key. + /// + /// + /// Whether to use the first block of the stream for the IV instead of the value in + /// . Generally for partial blob downloads where the + /// previous block of the ciphertext is the IV for the next. + /// + /// + /// Resolver to fetch the key encryption key. + /// + /// + /// Clients that can upload data have a key encryption key stored on them. Checking if + /// a cached key exists and matches the saves a call + /// to the external key resolver implementation when available. + /// + /// + /// Whether to ignore padding. Generally for partial blob downloads where the end of + /// the blob (where the padding occurs) was not downloaded. + /// + /// Whether to perform this function asynchronously. + /// + public static async Task DecryptInternal( + Stream ciphertext, + EncryptionData encryptionData, + bool ivInStream, + IKeyEncryptionKeyResolver keyResolver, + IKeyEncryptionKey potentialCachedKeyWrapper, + bool noPadding, + bool async, + CancellationToken cancellationToken) + { + Stream plaintext; + //int read = 0; + if (encryptionData != default) + { + byte[] IV; + if (!ivInStream) + { + IV = encryptionData.ContentEncryptionIV; + } + else + { + IV = new byte[EncryptionConstants.EncryptionBlockSize]; + if (async) + { + await ciphertext.ReadAsync(IV, 0, IV.Length, cancellationToken).ConfigureAwait(false); + } + else + { + ciphertext.Read(IV, 0, IV.Length); + } + //read = IV.Length; + } + + var contentEncryptionKey = await GetContentEncryptionKeyAsync(encryptionData, keyResolver, potentialCachedKeyWrapper, async, cancellationToken).ConfigureAwait(false); + + plaintext = WrapStream( + ciphertext, + contentEncryptionKey.ToArray(), + encryptionData, + IV, + noPadding); + } + else + { + plaintext = ciphertext; + } + + return plaintext; + } + +#pragma warning disable CS1587 // XML comment is not placed on a valid language element + /// + /// Returns the content encryption key for blob. First tries to get the key encryption key from KeyResolver, + /// then falls back to IKey stored on this EncryptionPolicy. Unwraps the content encryption key with the + /// correct key wrapper. + /// + /// The encryption data. + /// + /// + /// Whether to perform asynchronously. + /// + /// Encryption key as a byte array. + private static async Task> GetContentEncryptionKeyAsync( +#pragma warning restore CS1587 // XML comment is not placed on a valid language element + EncryptionData encryptionData, + IKeyEncryptionKeyResolver keyResolver, + IKeyEncryptionKey potentiallyCachedKeyWrapper, + bool async, + CancellationToken cancellationToken) + { + IKeyEncryptionKey key; + + // If we already have a local key and it is the correct one, use that. + if (encryptionData.WrappedContentKey.KeyId == potentiallyCachedKeyWrapper?.KeyId) + { + key = potentiallyCachedKeyWrapper; + } + // Otherwise, use the resolver. + else if (keyResolver != null) + { + key = async + ? await keyResolver.ResolveAsync(encryptionData.WrappedContentKey.KeyId, cancellationToken).ConfigureAwait(false) + : keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); + } + else + { + throw EncryptionErrors.KeyNotFound(encryptionData.WrappedContentKey.KeyId); + } + + if (key == default) + { + throw EncryptionErrors.NoKeyAccessor(); + } + + return async + ? await key.UnwrapKeyAsync( + encryptionData.WrappedContentKey.Algorithm, + encryptionData.WrappedContentKey.EncryptedKey).ConfigureAwait(false) + : key.UnwrapKey( + encryptionData.WrappedContentKey.Algorithm, + encryptionData.WrappedContentKey.EncryptedKey); + } + +#pragma warning disable CS1587 // XML comment is not placed on a valid language element + /// + /// Wraps a stream of ciphertext to stream plaintext. + /// + /// + /// + /// + /// + /// + /// + private static Stream WrapStream(Stream contentStream, byte[] contentEncryptionKey, +#pragma warning restore CS1587 // XML comment is not placed on a valid language element + EncryptionData encryptionData, byte[] iv, bool noPadding) + { + if (encryptionData.EncryptionAgent.EncryptionAlgorithm == ClientSideEncryptionAlgorithm.AesCbc256) + { + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider()) + { + aesProvider.IV = iv ?? encryptionData.ContentEncryptionIV; + aesProvider.Key = contentEncryptionKey; + + if (noPadding) + { + aesProvider.Padding = PaddingMode.None; + } + + return new CryptoStream(contentStream, aesProvider.CreateDecryptor(), CryptoStreamMode.Read); + } + } + + throw EncryptionErrors.BadEncryptionAlgorithm(encryptionData.EncryptionAgent.EncryptionAlgorithm.ToString()); + } + + /// + /// Wraps the given read-stream in a CryptoStream and provides the metadata used to create + /// that stream. + /// + /// Stream to wrap. + /// Key encryption key (KEK). + /// Algorithm to encrypt the content encryption key (CEK) with. + /// Whether to wrap the CEK asynchronously. + /// Cancellation token. + /// The wrapped stream to read from and the encryption metadata for the wrapped stream. + public static async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( + Stream plaintext, + IKeyEncryptionKey keyWrapper, + string keyWrapAlgorithm, + bool async, + CancellationToken cancellationToken) + { + var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + Stream ciphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: keyWrapper, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + ciphertext = new CryptoStream( + plaintext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Read); + } + + return (ciphertext, encryptionData); + } + + /// + /// Encrypts the given stream and provides the metadata used to encrypt. + /// + /// Stream to encrypt. + /// Key encryption key (KEK). + /// Algorithm to encrypt the content encryption key (CEK) with. + /// Whether to wrap the CEK asynchronously. + /// Cancellation token. + /// The encrypted data and the encryption metadata for the wrapped stream. + public static async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( + Stream plaintext, + IKeyEncryptionKey keyWrapper, + string keyWrapAlgorithm, + bool async, + CancellationToken cancellationToken) + { + var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + var ciphertext = new MemoryStream(); + byte[] bufferedCiphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: keyWrapper, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var transformStream = new CryptoStream( + ciphertext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Write); + + if (async) + { + await plaintext.CopyToAsync(transformStream).ConfigureAwait(false); + } + else + { + plaintext.CopyTo(transformStream); + } + + transformStream.FlushFinalBlock(); + + bufferedCiphertext = ciphertext.ToArray(); + } + + return (bufferedCiphertext, encryptionData); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index d4bbcd239d49a..81fd09680741a 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -3,6 +3,7 @@ using System; using System.Globalization; +using System.Linq; using System.Security.Authentication; using Azure.Core.Pipeline; @@ -84,5 +85,16 @@ public static void VerifyHttpsTokenAuth(Uri uri) throw new ArgumentException("Cannot use TokenCredential without HTTPS."); } } + + public static class ClientSideEncryption + { + public static InvalidOperationException TypeNotSupported(Type type) + => new InvalidOperationException( + $"Client-side encryption is not supported for type \"{type.FullName}\". " + + "Please use a supported client type or create this client without specifying client-side encryption options."); + + public static InvalidOperationException MissingRequiredEncryptionResources(params string[] resourceNames) + => new InvalidOperationException("Cannot encrypt without specifying " + string.Join(",", resourceNames.AsEnumerable())); + } } -} \ No newline at end of file +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index 0ee6912e60d3c..454d68f1a1b31 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -42,5 +42,9 @@ public static ArgumentOutOfRangeException InvalidSasProtocol(string protocol, st public static ArgumentException InvalidService(char s) => new ArgumentException($"Invalid service: '{s}'"); + + public static InvalidOperationException ClientSideEncryptionVersionNotSupported(string versionString = default) + => new InvalidOperationException("This library does not support the given version of client-side encryption." + + versionString == default ? "" : $" Version ID = {versionString}"); } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs new file mode 100644 index 0000000000000..abbc03cc69485 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Storage.Shared +{ + /// + /// Exposes a predetermined slice of a larger stream using the same Stream interface. + /// + internal class WindowStream : Stream + { + private Stream InnerStream { get; } + private long WindowLength { get; } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + private long _position = 0; + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + /// + /// Constructs a window of an underlying stream. While we can construct windows of unseekable + /// streams (no access to Length or Position), we must know the stream length to create a valid window. + /// + /// + /// Potentialy unseekable stream to expose a window of. + /// + /// + /// Maximum size of this window. + /// + public WindowStream(Stream stream, long windowLength) + { + InnerStream = stream; + WindowLength = windowLength; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + count = (int)Math.Min(count, WindowLength - _position); + if (count == 0) + { + return 0; + } + int result = InnerStream.Read(buffer, offset, count); + _position += result; + return result; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + count = (int)Math.Min(count, WindowLength - _position); + if (count == 0) + { + return 0; + } + int result = await InnerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + _position += result; + return result; + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs new file mode 100644 index 0000000000000..aa329692928b3 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Azure.Storage.Test +{ + public class KeyVaultConfiguration + { + public string VaultName { get; private set; } + public string VaultEndpoint { get; private set; } + public string ActiveDirectoryApplicationId { get; private set; } + public string ActiveDirectoryTenantId { get; private set; } + public string ActiveDirectoryApplicationSecret { get; private set; } + public string ActiveDirectoryAuthEndpoint { get; private set; } + + /// + /// Parse an XML representation into a TenantConfiguration value. + /// + /// The XML element to parse. + /// A TenantConfiguration value. + public static KeyVaultConfiguration Parse(XElement tenant) + { + string Get(string name) => (string)tenant.Element(name); + + return new KeyVaultConfiguration + { + VaultName = Get("VaultName"), + VaultEndpoint = Get("VaultEndpoint"), + ActiveDirectoryApplicationId = Get("ActiveDirectoryApplicationId"), + ActiveDirectoryApplicationSecret = Get("ActiveDirectoryApplicationSecret"), + ActiveDirectoryTenantId = Get("ActiveDirectoryTenantId"), + ActiveDirectoryAuthEndpoint = Get("ActiveDirectoryAuthEndpoint") + }; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs index 523bb68b7da47..9366d333bb5b1 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs @@ -36,6 +36,15 @@ public class TestConfigurations /// private IDictionary Tenants { get; set; } + /// + /// Gets or sets a mapping of keyvault names to definitions. The + /// Target*TenantName properties define the keys to use with this + /// dictionary. You should only access the tenants via the GetTenant + /// method that will Assert.Inconclusive if the desired tenant wasn't + /// defined in this configuration. + /// + private IDictionary KeyVaults { get; set; } + /// /// Gets the name of the tenant in the Tenants dictionary to use by /// default for our tests. @@ -66,6 +75,12 @@ public class TestConfigurations /// private string TargetOAuthTenantName { get; set; } + /// + /// Gets the name of the tenant in the keyvaults dictionary to use for + /// any tests that require integration with key vault. + /// + private string TargetKeyVaultName { get; set; } + /// /// Gets the name of the tenant in the Tenants dictionary to use for /// any tests that require hierarchical namespace. @@ -110,6 +125,12 @@ public class TestConfigurations public static TenantConfiguration DefaultTargetOAuthTenant => GetTenant("TargetOAuthTenant", s_configurations.Value.TargetOAuthTenantName); + /// + /// Gets a keyvault to use for any tests that require keyvault access. + /// + public static KeyVaultConfiguration DefaultTargetKeyVault => + GetKeyVault("TargetKeyVault", s_configurations.Value.TargetKeyVaultName); + /// /// Gets a tenant to use for any tests that require hierarchical namespace. /// @@ -168,6 +189,26 @@ private static TenantConfiguration GetTenant(string type, string name) return config; } + /// + /// Get the live test configuration for a specific key vault type, or + /// stop running the test via Assert.Inconclusive if not found. + /// + /// + /// The name of the key vault type (XML element) to get. + /// + /// The name of the keyvault. + /// + /// The live test configuration for a specific tenant type. + /// + private static KeyVaultConfiguration GetKeyVault(string type, string name) + { + if (!s_configurations.Value.KeyVaults.TryGetValue(name, out KeyVaultConfiguration config)) + { + Assert.Inconclusive($"Live test configuration key vault type '{type}' named '{name}' was not found in file {TestConfigurationsPath}!"); + } + return config; + } + /// /// Load the test configurations file from the path pointed to by the /// AZ_STORAGE_CONFIG_PATH environment variable or the local copy of @@ -217,12 +258,17 @@ private static TestConfigurations ReadFromXml(XDocument doc) TargetPremiumBlobTenantName = Get("TargetPremiumBlobTenant"), TargetPreviewBlobTenantName = Get("TargetPreviewBlobTenant"), TargetOAuthTenantName = Get("TargetOAuthTenant"), + TargetKeyVaultName = Get("TargetKeyVault"), TargetHierarchicalNamespaceTenantName = Get("TargetHierarchicalNamespaceTenant"), TargetManagedDiskTenantName = Get("TargetManagedDiskTenant"), Tenants = config.Element("TenantConfigurations").Elements("TenantConfiguration") .Select(TenantConfiguration.Parse) - .ToDictionary(tenant => tenant.TenantName) + .ToDictionary(tenant => tenant.TenantName), + KeyVaults = + config.Element("KeyVaultConfigurations").Elements("KeyVaultConfiguration") + .Select(KeyVaultConfiguration.Parse) + .ToDictionary(keyvault => keyvault.VaultName) }; } } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml index d626b7f1da3b2..f5a5ddad108b0 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml @@ -4,6 +4,7 @@ SecondaryTenant NotInPreview OAuthTenant + ClientsideEncryptionKeyvault NamespaceTenant ManagedDiskTenant @@ -124,4 +125,14 @@ http://[ACCOUNT]-secondary.table.core.windows.net + + + ClientsideEncryptionKeyvault + https://[KEYVAULT NAME].vault.azure.net/ + [ActiveDirectoryApplicationId] + [ActiveDirectoryTenantId] + [ActiveDirectoryApplicationSecret] + https://login.microsoftonline.com/ + + diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs new file mode 100644 index 0000000000000..82302d2319757 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Provides advanced client configuration options for connecting to Azure Queue + /// Storage. + /// +#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. + public class AdvancedQueueClientOptions : QueueClientOptions +#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion + { + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// The of the service API used when + /// making requests. + /// + public AdvancedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) + { + } + + /// + /// Settings for data encryption within the SDK. Client-side encryption adds metadata to your queue + /// messages which is necessary to maintain for decryption. + /// + /// For more information, see . + /// + public ClientSideEncryptionOptions ClientSideEncryption + { + get => _clientSideEncryptionOptions; + set => _clientSideEncryptionOptions = value; + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index 4330118d84c4e..cba507c1e6784 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -82,6 +82,10 @@ public QueueClientOptions(ServiceVersion version = LatestVersion) /// public Uri GeoRedundantSecondaryUri { get; set; } + #region Advanced Options + internal ClientSideEncryptionOptions _clientSideEncryptionOptions; + #endregion + /// /// Add headers and query parameters in and /// diff --git a/sdk/storage/Azure.Storage.sln b/sdk/storage/Azure.Storage.sln index 83df1576f3187..e78b608452723 100644 --- a/sdk/storage/Azure.Storage.sln +++ b/sdk/storage/Azure.Storage.sln @@ -85,8 +85,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.Shares. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Common.Samples.Tests", "Azure.Storage.Common\samples\Azure.Storage.Common.Samples.Tests.csproj", "{9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Cryptography", "Azure.Storage.Blobs.Cryptography\src\Azure.Storage.Blobs.Cryptography.csproj", "{4384B232-1222-44C7-9946-C23FE94DF6A9}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Cryptography", "Cryptography", "{9B235463-7D89-471E-87CA-A392661255E4}" ProjectSection(SolutionItems) = preProject Azure.Storage.Blobs.Cryptography\BreakingChanges.txt = Azure.Storage.Blobs.Cryptography\BreakingChanges.txt @@ -115,7 +113,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApiCompat", "..\..\eng\ApiC EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.Samples.Tests", "Azure.Storage.Blobs.Batch\samples\Azure.Storage.Blobs.Batch.Samples.Tests.csproj", "{73F575E6-FA87-40B0-9D36-6D9FAAC1E1C1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Core.TestFramework", "..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{23B3D5C8-3160-4BD6-8B25-0D33C98ABE70}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{23B3D5C8-3160-4BD6-8B25-0D33C98ABE70}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -179,10 +177,6 @@ Global {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Release|Any CPU.Build.0 = Release|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Release|Any CPU.Build.0 = Release|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Debug|Any CPU.Build.0 = Debug|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Release|Any CPU.ActiveCfg = Release|Any CPU From 2df8a86483bd434d7b2171b3b04787c6552d9527 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 19 May 2020 09:42:21 -0700 Subject: [PATCH 02/21] Removed crypto package from ci.yml --- sdk/storage/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index a88d44a0589a3..a241a4a193997 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -42,8 +42,6 @@ stages: safeName: AzureStorageBlobs - name: Azure.Storage.Blobs.Batch safeName: AzureStorageBlobsBatch - - name: Azure.Storage.Blobs.Cryptography - safeName: AzureStorageBlobsCryptography - name: Azure.Storage.Common safeName: AzureStorageCommon - name: Azure.Storage.Files.Shares From 7906d8e1dae9f2eca8ebdd0c353e5644ffef902c Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 19 May 2020 10:23:16 -0700 Subject: [PATCH 03/21] regenerate/export-api --- .../api/Azure.Storage.Blobs.netstandard2.0.cs | 5 + .../src/Generated/BlobRestClient.cs | 112 +++++++++--------- .../src/StorageRequestFailedException.cs | 2 +- .../Azure.Storage.Common.netstandard2.0.cs | 12 ++ .../Azure.Storage.Queues.netstandard2.0.cs | 8 ++ 5 files changed, 82 insertions(+), 57 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 31a6d1b16ef05..0754f8c60c755 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -914,6 +914,11 @@ internal UserDelegationKey() { } } namespace Azure.Storage.Blobs.Specialized { + public partial class AdvancedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions + { + public AdvancedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } public partial class AppendBlobClient : Azure.Storage.Blobs.Specialized.BlobBaseClient { protected AppendBlobClient() { } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs index 2b83efa4156f0..d1d8de36f7f22 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs @@ -18048,7 +18048,7 @@ internal partial class DataLakeStorageError /// /// The service error response object. /// - public Azure.Storage.Blobs.Models.Error Error { get; internal set; } + public Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails DataLakeStorageErrorDetails { get; internal set; } /// /// Creates a new DataLakeStorageError instance @@ -18066,7 +18066,7 @@ internal DataLakeStorageError(bool skipInitialization) { if (!skipInitialization) { - Error = new Azure.Storage.Blobs.Models.Error(); + DataLakeStorageErrorDetails = new Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails(); } } @@ -18083,7 +18083,7 @@ internal static Azure.Storage.Blobs.Models.DataLakeStorageError FromXml(System.X _child = element.Element(System.Xml.Linq.XName.Get("error", "")); if (_child != null) { - _value.Error = Azure.Storage.Blobs.Models.Error.FromXml(_child); + _value.DataLakeStorageErrorDetails = Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails.FromXml(_child); } CustomizeFromXml(element, _value); return _value; @@ -18094,6 +18094,59 @@ internal static Azure.Storage.Blobs.Models.DataLakeStorageError FromXml(System.X } #endregion class DataLakeStorageError +#region class DataLakeStorageErrorDetails +namespace Azure.Storage.Blobs.Models +{ + /// + /// The service error response object. + /// + internal partial class DataLakeStorageErrorDetails + { + /// + /// The service error code. + /// + public string Code { get; internal set; } + + /// + /// The service error message. + /// + public string Message { get; internal set; } + + /// + /// Prevent direct instantiation of DataLakeStorageErrorDetails instances. + /// You can use BlobsModelFactory.DataLakeStorageErrorDetails instead. + /// + internal DataLakeStorageErrorDetails() { } + + /// + /// Deserializes XML into a new DataLakeStorageErrorDetails instance. + /// + /// The XML element to deserialize. + /// A deserialized DataLakeStorageErrorDetails instance. + internal static Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails FromXml(System.Xml.Linq.XElement element) + { + System.Diagnostics.Debug.Assert(element != null); + System.Xml.Linq.XElement _child; + Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails _value = new Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails(); + _child = element.Element(System.Xml.Linq.XName.Get("Code", "")); + if (_child != null) + { + _value.Code = _child.Value; + } + _child = element.Element(System.Xml.Linq.XName.Get("Message", "")); + if (_child != null) + { + _value.Message = _child.Value; + } + CustomizeFromXml(element, _value); + return _value; + } + + static partial void CustomizeFromXml(System.Xml.Linq.XElement element, Azure.Storage.Blobs.Models.DataLakeStorageErrorDetails value); + } +} +#endregion class DataLakeStorageErrorDetails + #region enum DeleteSnapshotsOption namespace Azure.Storage.Blobs.Models { @@ -19849,58 +19902,5 @@ public static UserDelegationKey UserDelegationKey( } } #endregion class UserDelegationKey - -#region class Error -namespace Azure.Storage.Blobs.Models -{ - /// - /// The service error response object. - /// - internal partial class Error - { - /// - /// The service error code. - /// - public string Code { get; internal set; } - - /// - /// The service error message. - /// - public string Message { get; internal set; } - - /// - /// Prevent direct instantiation of Error instances. - /// You can use BlobsModelFactory.Error instead. - /// - internal Error() { } - - /// - /// Deserializes XML into a new Error instance. - /// - /// The XML element to deserialize. - /// A deserialized Error instance. - internal static Azure.Storage.Blobs.Models.Error FromXml(System.Xml.Linq.XElement element) - { - System.Diagnostics.Debug.Assert(element != null); - System.Xml.Linq.XElement _child; - Azure.Storage.Blobs.Models.Error _value = new Azure.Storage.Blobs.Models.Error(); - _child = element.Element(System.Xml.Linq.XName.Get("Code", "")); - if (_child != null) - { - _value.Code = _child.Value; - } - _child = element.Element(System.Xml.Linq.XName.Get("Message", "")); - if (_child != null) - { - _value.Message = _child.Value; - } - CustomizeFromXml(element, _value); - return _value; - } - - static partial void CustomizeFromXml(System.Xml.Linq.XElement element, Azure.Storage.Blobs.Models.Error value); - } -} -#endregion class Error #endregion Models diff --git a/sdk/storage/Azure.Storage.Blobs/src/StorageRequestFailedException.cs b/sdk/storage/Azure.Storage.Blobs/src/StorageRequestFailedException.cs index f65937353bf42..534d37ddb6377 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/StorageRequestFailedException.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/StorageRequestFailedException.cs @@ -78,6 +78,6 @@ internal partial class DataLakeStorageError /// The failed response. /// A RequestFailedException. public Exception CreateException(ClientDiagnostics clientDiagnostics, Azure.Response response) - => clientDiagnostics.CreateRequestFailedExceptionWithContent(response, message: Error.Message, content: null, response.GetErrorCode(Error.Code)); + => clientDiagnostics.CreateRequestFailedExceptionWithContent(response, message: DataLakeStorageErrorDetails.Message, content: null, response.GetErrorCode(DataLakeStorageErrorDetails.Code)); } } diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index 9768ccbc8f975..ed381d3506e36 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -1,5 +1,17 @@ namespace Azure.Storage { + public partial class ClientSideEncryptionOptions + { + public ClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) { } + public Azure.Core.Cryptography.IKeyEncryptionKey KeyEncryptionKey { get { throw null; } set { } } + public Azure.Core.Cryptography.IKeyEncryptionKeyResolver KeyResolver { get { throw null; } set { } } + public string KeyWrapAlgorithm { get { throw null; } set { } } + public Azure.Storage.ClientSideEncryptionVersion Version { get { throw null; } } + } + public enum ClientSideEncryptionVersion + { + V1_0 = 0, + } public partial class StorageSharedKeyCredential { public StorageSharedKeyCredential(string accountName, string accountKey) { } diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index 209865123d36c..e6de63848eb14 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -315,6 +315,14 @@ internal UpdateReceipt() { } public string PopReceipt { get { throw null; } } } } +namespace Azure.Storage.Queues.Specialized +{ + public partial class AdvancedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions + { + public AdvancedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } +} namespace Azure.Storage.Sas { [System.FlagsAttribute] From 4d717593388fff7a82a791daef9ba121e98d6498 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 19 May 2020 14:40:02 -0700 Subject: [PATCH 04/21] Ported clientside encryption for queues --- eng/Packages.Data.props | 1 + .../src/Azure.Storage.Queues.csproj | 7 +- .../src/Models/EncryptedMessage.cs | 14 + .../src/Models/EncryptedMessageSerializer.cs | 93 +++++ .../Azure.Storage.Queues/src/QueueClient.cs | 117 +++++- .../src/QueueServiceClient.cs | 15 +- .../tests/Azure.Storage.Queues.Tests.csproj | 4 + .../tests/ClientSideEncryptionTests.cs | 372 ++++++++++++++++++ .../tests/EncryptedMessageSerializerTests.cs | 139 +++++++ .../tests/QueueTestBase.cs | 21 +- 10 files changed, 772 insertions(+), 11 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs create mode 100644 sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs create mode 100644 sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs create mode 100644 sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs diff --git a/eng/Packages.Data.props b/eng/Packages.Data.props index 7b6f1a558630d..1b570f33ff856 100755 --- a/eng/Packages.Data.props +++ b/eng/Packages.Data.props @@ -44,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj b/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj index 22499d097f263..efc3150cc7835 100644 --- a/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj +++ b/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) @@ -16,6 +16,9 @@ REST API Reference for Queue Service - https://docs.microsoft.com/en-us/rest/api/storageservices/queue-service-rest-api + + + @@ -31,6 +34,8 @@ + + diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs new file mode 100644 index 0000000000000..79ed4f5dde82f --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Queues.Specialized.Models +{ + internal class EncryptedMessage + { + public string EncryptedMessageContents { get; set; } + + public EncryptionData EncryptionData { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs new file mode 100644 index 0000000000000..05b7de79c5e05 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Text; +using System.Text.Json; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Queues.Specialized.Models +{ + internal static class EncryptedMessageSerializer + { + #region Serialize + public static string Serialize(EncryptedMessage data) + { + return Encoding.UTF8.GetString(SerializeEncryptedMessage(data).ToArray()); + } + + public static ReadOnlyMemory SerializeEncryptedMessage(EncryptedMessage message) + { + var writer = new Core.ArrayBufferWriter(); + using var json = new Utf8JsonWriter(writer); + + json.WriteStartObject(); + WriteEncryptedMessage(json, message); + json.WriteEndObject(); + + json.Flush(); + return writer.WrittenMemory; + } + + public static void WriteEncryptedMessage(Utf8JsonWriter json, EncryptedMessage message) + { + json.WriteString(nameof(message.EncryptedMessageContents), message.EncryptedMessageContents); + + json.WriteStartObject(nameof(message.EncryptionData)); + EncryptionDataSerializer.WriteEncryptionData(json, message.EncryptionData); + json.WriteEndObject(); + } + #endregion + + #region Deserialize + public static bool TryDeserialize(string serializedData, out EncryptedMessage encryptedMessage) + { + try + { + encryptedMessage = Deserialize(serializedData); + return true; + } + catch (JsonException) + { + encryptedMessage = default; + return false; + } + } + + public static EncryptedMessage Deserialize(string serializedData) + { + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(serializedData)); + return DeserializeEncryptedMessage(ref reader); + } + + public static EncryptedMessage DeserializeEncryptedMessage(ref Utf8JsonReader reader) + { + using JsonDocument json = JsonDocument.ParseValue(ref reader); + JsonElement root = json.RootElement; + return ReadEncryptionData(root); + } + + private static EncryptedMessage ReadEncryptionData(JsonElement root) + { + var data = new EncryptedMessage(); + foreach (var property in root.EnumerateObject()) + { + ReadPropertyValue(data, property); + } + return data; + } + + private static void ReadPropertyValue(EncryptedMessage data, JsonProperty property) + { + if (property.NameEquals(nameof(data.EncryptedMessageContents))) + { + data.EncryptedMessageContents = property.Value.GetString(); + } + else if (property.NameEquals(nameof(data.EncryptionData))) + { + data.EncryptionData = EncryptionDataSerializer.ReadEncryptionData(property.Value); + } + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index f972d89da457e..35aad93c1d21b 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -4,13 +4,18 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Net; +using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized.Models; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Queues @@ -72,6 +77,18 @@ public class QueueClient /// internal virtual ClientDiagnostics ClientDiagnostics => _clientDiagnostics; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + + internal bool UsingClientSideEncryption => ClientSideEncryption != default; + /// /// QueueMaxMessagesPeek indicates the maximum number of messages /// you can retrieve with each call to Peek. @@ -178,6 +195,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); } /// @@ -269,6 +287,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); } /// @@ -290,13 +309,17 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien /// The instance used to create /// diagnostic scopes every request. /// - internal QueueClient(Uri queueUri, HttpPipeline pipeline, QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics) + /// + /// Options for client-side encryption. + /// + internal QueueClient(Uri queueUri, HttpPipeline pipeline, QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, ClientSideEncryptionOptions encryptionOptions) { _uri = queueUri; _messagesUri = queueUri.AppendToPath(Constants.Queue.MessagesUri); _pipeline = pipeline; _version = version; _clientDiagnostics = clientDiagnostics; + _clientSideEncryption = encryptionOptions?.Clone(); } #endregion ctors @@ -1472,6 +1495,10 @@ private async Task> SendMessageInternal( $"{nameof(timeToLive)}: {timeToLive}"); try { + messageText = UsingClientSideEncryption + ? await ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) + : messageText; + Response> messages = await QueueRestClient.Messages.EnqueueAsync( ClientDiagnostics, @@ -1659,9 +1686,20 @@ private async Task> ReceiveMessagesInternal( .ConfigureAwait(false); // Return an exploding Response on 304 - return response.IsUnavailable() ? - response.GetRawResponse().AsNoBodyResponse() : - Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + if (response.IsUnavailable()) + { + return response.GetRawResponse().AsNoBodyResponse(); + } + else if (UsingClientSideEncryption) + { + return Response.FromValue( + await ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + response.GetRawResponse()); + } + else + { + return Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + } } catch (Exception ex) { @@ -1766,9 +1804,20 @@ private async Task> PeekMessagesInternal( .ConfigureAwait(false); // Return an exploding Response on 304 - return response.IsUnavailable() ? - response.GetRawResponse().AsNoBodyResponse() : - Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + if (response.IsUnavailable()) + { + return response.GetRawResponse().AsNoBodyResponse(); + } + else if (UsingClientSideEncryption) + { + return Response.FromValue( + await ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + response.GetRawResponse()); + } + else + { + return Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + } } catch (Exception ex) { @@ -2043,5 +2092,59 @@ private async Task> UpdateMessageInternal( } } #endregion UpdateMessage + + private async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) + { + var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); + (byte[] ciphertext, EncryptionData encryptionData) = await Utility.BufferedEncryptInternal( + new MemoryStream(bytesToEncrypt), + ClientSideEncryption.KeyEncryptionKey, + ClientSideEncryption.KeyWrapAlgorithm, + async, + cancellationToken).ConfigureAwait(false); + + return EncryptedMessageSerializer.Serialize(new EncryptedMessage + { + EncryptedMessageContents = Convert.ToBase64String(ciphertext), + EncryptionData = encryptionData + }); + } + + private async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) + { + foreach (var message in messages) + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + } + return messages; + } + private async Task ClientSideDecryptMessagesInternal(PeekedMessage[] messages, bool async, CancellationToken cancellationToken) + { + foreach (var message in messages) + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + } + return messages; + } + + private async Task ClientSideDecryptInternal(string downloadedMessage, bool async, CancellationToken cancellationToken) + { + if (!EncryptedMessageSerializer.TryDeserialize(downloadedMessage, out var encryptedMessage)) + { + return downloadedMessage; // not recognized as client-side encrypted message + } + + var decryptedMessageStream = await Utility.DecryptInternal( + new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)), + encryptedMessage.EncryptionData, + ivInStream: false, + ClientSideEncryption.KeyResolver, + ClientSideEncryption.KeyEncryptionKey, + noPadding: false, + async: async, + cancellationToken).ConfigureAwait(false); + + return new StreamReader(decryptedMessageStream, Encoding.UTF8).ReadToEnd(); + } } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs index 3aa8eae603aa8..2d99ec33eaeaa 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; using Azure.Storage.Queues.Models; namespace Azure.Storage.Queues @@ -59,6 +60,16 @@ public class QueueServiceClient /// internal virtual ClientDiagnostics ClientDiagnostics => _clientDiagnostics; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The Storage account name corresponding to the service client. /// @@ -128,6 +139,7 @@ public QueueServiceClient(string connectionString, QueueClientOptions options) _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); } /// @@ -214,6 +226,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); } #endregion ctors @@ -230,7 +243,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q /// A for the desired queue. /// public virtual QueueClient GetQueueClient(string queueName) - => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics); + => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption); #region GetQueues /// diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index e767284f654c6..c3751a40aa40f 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -13,7 +13,11 @@ PreserveNewest + + + + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs new file mode 100644 index 0000000000000..42e97983d5c4b --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -0,0 +1,372 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Core.TestFramework; +using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Specialized.Models; +using Azure.Storage.Queues.Tests; +using Azure.Storage.Tests.Shared; +using NUnit.Framework; + +namespace Azure.Storage.Queues.Test +{ + public class ClientSideEncryptionTests : QueueTestBase + { + private readonly string SampleUTF8String = Encoding.UTF8.GetString( + new byte[] { 0xe1, 0x9a, 0xa0, 0xe1, 0x9b, 0x87, 0xe1, 0x9a, 0xbb, 0x0a }); // valid UTF-8 bytes + + public ClientSideEncryptionTests(bool async) + : base(async, null /* RecordedTestMode.Record /* to re-record */) + { + } + + #region Utility + + private string LocalManualEncryption(string message, byte[] key, byte[] iv) + { + using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) + using (var encryptor = aesProvider.CreateEncryptor()) + using (var memStream = new MemoryStream()) + using (var cryptoStream = new CryptoStream(memStream, encryptor, CryptoStreamMode.Write)) + { + var messageBytes = Encoding.UTF8.GetBytes(message); + cryptoStream.Write(messageBytes, 0, messageBytes.Length); + cryptoStream.FlushFinalBlock(); + return Convert.ToBase64String(memStream.ToArray()); + } + } + + private async Task GetKeyvaultIKeyEncryptionKey() + { + var keyClient = GetKeyClient_TargetKeyClient(); + Security.KeyVault.Keys.KeyVaultKey key = await keyClient.CreateRsaKeyAsync( + new Security.KeyVault.Keys.CreateRsaKeyOptions($"CloudRsaKey-{Guid.NewGuid()}", false)); + return new CryptographyClient(key.Id, GetTokenCredential_TargetKeyClient()); + } + + /// + /// Creates an encrypted queue client from a normal queue client. Note that this method does not copy over any + /// client options from the container client. You must pass in your own options. These options will be mutated. + /// + public async Task GetTestEncryptedQueueAsync( + ClientSideEncryptionOptions encryptionOptions, + string queueName = default, + IDictionary metadata = default) + { + // normally set through property on subclass; this is easier to hook up in current test infra with internals access + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; + + var service = GetServiceClient_SharedKey(options); + + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + QueueClient queue = InstrumentClient(service.GetQueueClient(GetNewQueueName())); + return await DisposingQueue.CreateAsync(queue, metadata); + } + + /// + /// Generates a random string of the given size. + /// For implementation simplicity, this generates an ASCII string. + /// + /// Size of the string IN BYTES, not chars. + /// + public string GetRandomMessage(int size) + { + var buf = new byte[size]; + var random = new Random(); + for (int i = 0; i < size; i++) + { + buf[i] = (byte)random.Next(32, 127); // printable ASCII has values [32, 127) + } + + return Encoding.ASCII.GetString(buf); + } + + #endregion + + [TestCase(16, false)] // a single cipher block + [TestCase(14, false)] // a single unalligned cipher block + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message); + + // download without decrypting + var receivedMessages = (await new QueueClient(queue.Uri, GetNewSharedKeyCredentials()).ReceiveMessagesAsync()).Value; + Assert.AreEqual(1, receivedMessages.Length); + var encryptedMessage = receivedMessages[0].MessageText; // json of message and metadata + var parsedEncryptedMessage = EncryptedMessageSerializer.Deserialize(encryptedMessage); + + // encrypt original data manually for comparison + EncryptionData encryptionMetadata = parsedEncryptedMessage.EncryptionData; + Assert.NotNull(encryptionMetadata, "Never encrypted data."); + string expectedEncryptedMessage = LocalManualEncryption( + message, + (await mockKey.UnwrapKeyAsync(null, encryptionMetadata.WrappedContentKey.EncryptedKey) + .ConfigureAwait(false)).ToArray(), + encryptionMetadata.ContentEncryptionIV); + + // compare data + Assert.AreEqual(expectedEncryptedMessage, parsedEncryptedMessage.EncryptedMessageContents); + } + } + + [TestCase(16, false)] // a single cipher block + [TestCase(14, false)] // a single unalligned cipher block + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task RoundtripAsync(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message); + + // download with decryption + var receivedMessages = (await queue.ReceiveMessagesAsync()).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message, downloadedMessage); + } + } + + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task Track2DownloadTrack1Blob(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var track2Queue = disposable.Queue; + + // upload with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Queue = new Microsoft.Azure.Storage.Queue.CloudQueue( + track2Queue.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + await track1Queue.AddMessageAsync( + new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), + null, + null, + new Microsoft.Azure.Storage.Queue.QueueRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, mockKey) + }, + null); + + // download with track 2 + var receivedMessages = (await track2Queue.ReceiveMessagesAsync()).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare original data to downloaded data + Assert.AreEqual(message, downloadedMessage); + } + } + + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task Track1DownloadTrack2Blob(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var track2Queue = disposable.Queue; + + // upload with track 2 + await track2Queue.SendMessageAsync(message); + + // download with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Queue = new Microsoft.Azure.Storage.Queue.CloudQueue( + track2Queue.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var response = await track1Queue.GetMessageAsync( + null, + new Microsoft.Azure.Storage.Queue.QueueRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, mockKey) + }, + null); + + // compare original data to downloaded data + Assert.AreEqual(message, response.AsString); + } + } + + [Test] + [LiveOnly] // need access to keyvault service && cannot seed content encryption key + public async Task RoundtripWithKeyvaultProvider() + { + var message = GetRandomMessage(Constants.KB); + IKeyEncryptionKey key = await GetKeyvaultIKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = key, + KeyWrapAlgorithm = "RSA-OAEP-256" + })) + { + var queue = disposable.Queue; + + await queue.SendMessageAsync(message); + + var receivedMessages = (await queue.ReceiveMessagesAsync()).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + Assert.AreEqual(message, downloadedMessage); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task ReadPlaintextMessage() + { + var message = "any old message"; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var encryptedQueueClient = disposable.Queue; + var plainQueueClient = new QueueClient(encryptedQueueClient.Uri, GetNewSharedKeyCredentials()); + + // upload with encryption + await plainQueueClient.SendMessageAsync(message); + + // download with decryption + var receivedMessages = (await encryptedQueueClient.ReceiveMessagesAsync()).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message, downloadedMessage); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task OnlyOneKeyWrapCall() + { + var message = "any old message"; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var queue = disposable.Queue; + + await queue.SendMessageAsync(message).ConfigureAwait(false); + + Assert.AreEqual(1, IsAsync ? mockKey.WrappedAsync : mockKey.WrappedSync); + Assert.AreEqual(0, IsAsync ? mockKey.WrappedSync : mockKey.WrappedAsync); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task OnlyOneKeyResolveAndUnwrapCall() + { + var message = "any old message"; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var queue = disposable.Queue; + await queue.SendMessageAsync(message).ConfigureAwait(false); + mockKey.ResetCounters(); + + // replace with client that has only key resolver + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = default, // we want the key resolver to trigger; no cached key + KeyResolver = mockKey + }; + queue = InstrumentClient(new QueueClient( + queue.Uri, + GetNewSharedKeyCredentials(), + options)); + + await queue.ReceiveMessagesAsync(); + + Assert.AreEqual(1, IsAsync ? mockKey.ResolvedAsync : mockKey.ResolvedSync); + Assert.AreEqual(0, IsAsync ? mockKey.ResolvedSync : mockKey.ResolvedAsync); + + Assert.AreEqual(1, IsAsync ? mockKey.UnwrappedAsync : mockKey.UnwrappedSync); + Assert.AreEqual(0, IsAsync ? mockKey.UnwrappedSync : mockKey.UnwrappedAsync); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs new file mode 100644 index 0000000000000..3d72973c7a3fa --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Specialized.Models; +using Azure.Storage.Tests.Shared; +using NUnit.Framework; + +namespace Azure.Storage.Queues.Test +{ + public class EncryptedMessageSerializerTests // doesn't inherit our test base because there are no network tests + { + private const string TestMessage = "This can technically be a valid encrypted message."; + private const string KeyWrapAlgorithm = "my_key_wrap_algorithm"; + + [Test] + public void SerializeEncryptedMessage() + { + var result = Utility.BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + new MockKeyEncryptionKey(), + KeyWrapAlgorithm, + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + // success = don't throw. test values in another test with deserialization (can't control serialization order) + } + + [Test] + public void DeserializeEncryptedMessage() + { + var result = Utility.BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + new MockKeyEncryptionKey(), + KeyWrapAlgorithm, + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + var parsedEncryptedMessage = EncryptedMessageSerializer.Deserialize(serializedMessage); + + Assert.IsTrue(AreEqual(encryptedMessage, parsedEncryptedMessage)); + } + + [Test] + public void TryDeserializeEncryptedMessage() + { + var result = Utility.BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + new MockKeyEncryptionKey(), + KeyWrapAlgorithm, + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + bool tryResult = EncryptedMessageSerializer.TryDeserialize(serializedMessage, out var parsedEncryptedMessage); + + Assert.AreEqual(true, tryResult); + Assert.IsTrue(AreEqual(encryptedMessage, parsedEncryptedMessage)); + } + + [Test] + public void TryDeserializeEncryptedMessageFail() + { + bool tryResult = EncryptedMessageSerializer.TryDeserialize("this is not even valid json", out var parsedEncryptedMessage); + + Assert.AreEqual(false, tryResult); + Assert.AreEqual(default, parsedEncryptedMessage); + } + + #region ModelComparison + private static bool AreEqual(EncryptedMessage left, EncryptedMessage right) + => left.EncryptedMessageContents.Equals(right.EncryptedMessageContents, StringComparison.InvariantCulture) + && AreEqual(left.EncryptionData, right.EncryptionData); + + private static bool AreEqual(EncryptionData left, EncryptionData right) + => left.EncryptionMode.Equals(right.EncryptionMode, StringComparison.InvariantCulture) + && AreEqual(left.WrappedContentKey, right.WrappedContentKey) + && AreEqual(left.EncryptionAgent, right.EncryptionAgent) + && left.ContentEncryptionIV.SequenceEqual(right.ContentEncryptionIV) + && AreEqual(left.KeyWrappingMetadata, right.KeyWrappingMetadata); + + private static bool AreEqual(WrappedKey left, WrappedKey right) + => left.KeyId.Equals(right.KeyId, StringComparison.InvariantCulture) + && left.EncryptedKey.SequenceEqual(right.EncryptedKey) + && left.Algorithm.Equals(right.Algorithm, StringComparison.InvariantCulture); + + private static bool AreEqual(EncryptionAgent left, EncryptionAgent right) + => left.EncryptionAlgorithm.Equals(right.EncryptionAlgorithm) + && left.Protocol.Equals(right.Protocol); + + private static bool AreEqual(IDictionary left, IDictionary right) + { + if (left.Count != right.Count) + { + return false; + } + foreach (KeyValuePair leftPair in left) + { + if (!right.TryGetValue(leftPair.Key, out string rightValue)) + { + return false; + } + if (!leftPair.Value.Equals(rightValue, StringComparison.InvariantCulture)) + { + return false; + } + } + + return true; + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs b/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs index 601cfff3e5100..64ebbd250c93d 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs @@ -55,14 +55,14 @@ public QueueClientOptions GetOptions() return Recording.InstrumentClientOptions(options); } - public QueueServiceClient GetServiceClient_SharedKey() + public QueueServiceClient GetServiceClient_SharedKey(QueueClientOptions options = default) => InstrumentClient( new QueueServiceClient( new Uri(TestConfigDefault.QueueServiceEndpoint), new StorageSharedKeyCredential( TestConfigDefault.AccountName, TestConfigDefault.AccountKey), - GetOptions())); + options ?? GetOptions())); public QueueServiceClient GetServiceClient_AccountSas(StorageSharedKeyCredential sharedKeyCredentials = default, SasQueryParameters sasCredentials = default) => InstrumentClient( @@ -76,6 +76,23 @@ public QueueServiceClient GetServiceClient_QueueServiceSas(string queueName, Sto new Uri($"{TestConfigDefault.QueueServiceEndpoint}?{sasCredentials ?? GetNewQueueServiceSasCredentials(queueName, sharedKeyCredentials ?? GetNewSharedKeyCredentials())}"), GetOptions())); + public Security.KeyVault.Keys.KeyClient GetKeyClient_TargetKeyClient() + => GetKeyClient(TestConfigurations.DefaultTargetKeyVault); + + public TokenCredential GetTokenCredential_TargetKeyClient() + => GetKeyClientTokenCredential(TestConfigurations.DefaultTargetKeyVault); + + private static Security.KeyVault.Keys.KeyClient GetKeyClient(KeyVaultConfiguration config) + => new Security.KeyVault.Keys.KeyClient( + new Uri(config.VaultEndpoint), + GetKeyClientTokenCredential(config)); + + private static TokenCredential GetKeyClientTokenCredential(KeyVaultConfiguration config) + => new Identity.ClientSecretCredential( + config.ActiveDirectoryTenantId, + config.ActiveDirectoryApplicationId, + config.ActiveDirectoryApplicationSecret); + public QueueServiceClient GetServiceClient_OauthAccount() => GetServiceClientFromOauthConfig(TestConfigOAuth); From 5219aa560fcbe83f9bc95e37ac53f6f3ec0536fa Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Wed, 20 May 2020 15:38:32 -0700 Subject: [PATCH 05/21] Queues handles partial decryption errors. Queues now has a listener for when only some messages cannot be decrypted. these messages are filtered out of the response and sent to the listened instead. If no listener is provided, the whole fetch throws. Some PR comments addressed. --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 16 ++-- .../AlwaysFailsKeyEncryptionKeyResolver.cs | 38 +++++++++ .../tests/ClientSideEncryptionTests.cs | 50 ++++++++++++ .../tests/CryptoraphyTestsExtensionMethods.cs | 13 --- .../Azure.Storage.Common.netstandard2.0.cs | 5 ++ ...lientSideEncryptionKeyNotFoundException.cs | 28 +++++++ .../src/ClientsideEncryptionOptions.cs | 13 --- .../ClientsideEncryption/EncryptionErrors.cs | 4 +- .../Models/EncryptionData.cs | 6 +- .../Models/EncryptionDataSerializer.cs | 6 +- .../Models/{WrappedKey.cs => KeyEnvelope.cs} | 2 +- .../Shared/ClientsideEncryption/Utility.cs | 30 ++++--- .../src/Shared/WindowStream.cs | 18 +++++ .../Azure.Storage.Queues.netstandard2.0.cs | 8 ++ .../src/AdvancedQueueClientOptions.cs | 45 +++++++++++ .../src/Models/EncryptedMessageSerializer.cs | 4 +- .../Azure.Storage.Queues/src/QueueClient.cs | 67 ++++++++++++++-- .../src/QueueClientOptions.cs | 2 + .../src/QueueServiceClient.cs | 7 +- .../tests/Azure.Storage.Queues.Tests.csproj | 1 + .../tests/ClientSideEncryptionTests.cs | 80 +++++++++++++++++++ .../tests/EncryptedMessageSerializerTests.cs | 12 ++- ...kMissingClientSideEncryptionKeyListener.cs | 36 +++++++++ 23 files changed, 426 insertions(+), 65 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs delete mode 100644 sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs rename sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/{WrappedKey.cs => KeyEnvelope.cs} (95%) create mode 100644 sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 40822ff11b1e0..840b004741aed 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -659,16 +658,20 @@ private async Task> DownloadInternal( bool async, CancellationToken cancellationToken) { + HttpRange requestedRange = range; using (Pipeline.BeginLoggingScope(nameof(BlobBaseClient))) { Pipeline.LogMethodEnter(nameof(BlobBaseClient), message: $"{nameof(Uri)}: {Uri}"); try { - EncryptedBlobRange encryptedRange = new EncryptedBlobRange(range); + if (UsingClientSideEncryption) + { + range = new EncryptedBlobRange(range).AdjustedRange; + } // Start downloading the blob (Response response, Stream stream) = await StartDownloadAsync( - UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, + range, conditions, rangeGetContentHash, async: async, @@ -688,7 +691,7 @@ private async Task> DownloadInternal( stream, startOffset => StartDownloadAsync( - UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, + range, conditions, rangeGetContentHash, startOffset, @@ -698,7 +701,7 @@ private async Task> DownloadInternal( .Item2, async startOffset => (await StartDownloadAsync( - UsingClientSideEncryption ? encryptedRange.AdjustedRange : range, + range, conditions, rangeGetContentHash, startOffset, @@ -713,7 +716,7 @@ private async Task> DownloadInternal( // we already return a nonseekable stream; returning a crypto stream is fine if (UsingClientSideEncryption) { - stream = await ClientSideDecryptInternal(stream, response.Value.Metadata, encryptedRange.OriginalRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); + stream = await ClientSideDecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); } response.Value.Content = stream; @@ -3003,6 +3006,7 @@ private async Task ClientSideDecryptInternal( bool ivInStream = originalRange.Offset >= 16; + // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. var plaintext = await Utility.DecryptInternal( content, encryptionData, diff --git a/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs b/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs new file mode 100644 index 0000000000000..688716dc1f9e6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; + +namespace Azure.Storage.Blobs.Tests +{ + internal class AlwaysFailsKeyEncryptionKeyResolver : IKeyEncryptionKeyResolver + { + /// + /// False means the resolver just can't find the key and returns null. + /// True means the resolver has an internal failure and throws. + /// + public bool ResolverInternalFailure { get; set; } = false; + + public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationToken = default) + { + if (ResolverInternalFailure) + { + throw new Exception(); + } + return default; + } + + public Task ResolveAsync(string keyId, CancellationToken cancellationToken = default) + { + if (ResolverInternalFailure) + { + throw new Exception(); + } + return Task.FromResult(default); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 7c2c85cfe1f72..33b0116dc485e 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -5,12 +5,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using Azure.Core.Cryptography; using Azure.Core.TestFramework; using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Blobs.Tests; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Test.Shared; @@ -339,6 +341,54 @@ public async Task RoundtripWithKeyvaultProvider() } } + [TestCase(true)] + [TestCase(false)] + [LiveOnly] + public async Task CannotFindKeyAsync(bool resolverFailure) + { + var data = GetRandomBuffer(Constants.KB); + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = ThrowawayAlgorithmName + })) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + await blob.UploadAsync(new MemoryStream(data)); + + bool threwKeyNotFound = false; + bool threwGeneral = false; + try + { + // download but can't find key + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ResolverInternalFailure = resolverFailure }, + KeyWrapAlgorithm = "test" + }; + var encryptedDataStream = new MemoryStream(); + await new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options).DownloadToAsync(encryptedDataStream); + } + catch (ClientSideEncryptionKeyNotFoundException) + { + threwKeyNotFound = true; + } + catch (Exception) + { + threwGeneral = true; + } + finally + { + Assert.AreEqual(resolverFailure, threwGeneral); + Assert.AreEqual(!resolverFailure, threwKeyNotFound); + } + } + } + [Test] [LiveOnly] // cannot seed content encryption key [Ignore("stress test")] diff --git a/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs b/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs deleted file mode 100644 index e8afa156876f5..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/tests/CryptoraphyTestsExtensionMethods.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using Azure.Core; -using Azure.Storage.Blobs.Specialized; - -namespace Azure.Storage.Blobs.Cryptography.Tests -{ - internal static class CryptoraphyTestsExtensionMethods - { - - } -} diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index ed381d3506e36..6e53eec22cd34 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -1,5 +1,10 @@ namespace Azure.Storage { + public partial class ClientSideEncryptionKeyNotFoundException : System.Exception + { + public ClientSideEncryptionKeyNotFoundException(string keyId) { } + public string KeyId { get { throw null; } } + } public partial class ClientSideEncryptionOptions { public ClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) { } diff --git a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs new file mode 100644 index 0000000000000..06c1b7e3ae61c --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Storage +{ + /// + /// Thrown when the client fails to resolve the necessary key to decrypt data using client-side encryption. + /// + public class ClientSideEncryptionKeyNotFoundException : Exception + { + /// + /// Key that could not be resolved. + /// + public string KeyId { get; } + + /// + /// Constructs the exception. + /// + /// Key id. + public ClientSideEncryptionKeyNotFoundException(string keyId) + : base($"Provided key resolver ould not resolve key of id `{keyId}`.") + { + KeyId = keyId; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs index a51d569f4fb26..6e50fd7a5d0ba 100644 --- a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs +++ b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs @@ -46,18 +46,5 @@ public ClientSideEncryptionOptions(ClientSideEncryptionVersion version) { Version = version; } - - /// - /// Copy constructor to keep these options grouped in clients while stopping users from - /// accidentally altering our configs out from under us. - /// - /// - internal ClientSideEncryptionOptions(ClientSideEncryptionOptions other) - { - Version = other.Version; - KeyEncryptionKey = other.KeyEncryptionKey; - KeyResolver = other.KeyResolver; - KeyWrapAlgorithm = other.KeyWrapAlgorithm; - } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs index 48dccc622fa58..3866c492fb8e0 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs @@ -7,8 +7,8 @@ namespace Azure.Storage.Cryptography { internal static class EncryptionErrors { - public static ArgumentException KeyNotFound(string keyId) - => new ArgumentException($"Could not resolve key of id `{keyId}`."); + public static ClientSideEncryptionKeyNotFoundException KeyNotFound(string keyId) + => new ClientSideEncryptionKeyNotFoundException(keyId); public static ArgumentException BadEncryptionAgent(string agent) => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs index 7c2f8bd0a1437..d7a3727fa05a5 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -23,9 +23,9 @@ internal class EncryptionData public string EncryptionMode { get; set; } /// - /// A object that stores the wrapping algorithm, key identifier and the encrypted key. + /// A object that stores the wrapping algorithm, key identifier and the encrypted key. /// - public WrappedKey WrappedContentKey { get; set; } + public KeyEnvelope WrappedContentKey { get; set; } /// /// The encryption agent. @@ -79,7 +79,7 @@ internal static async Task CreateInternalV1_0( { { EncryptionConstants.AgentMetadataKey, AgentString } }, - WrappedContentKey = new WrappedKey() + WrappedContentKey = new KeyEnvelope() { Algorithm = keyWrapAlgorithm, EncryptedKey = async diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs index a2bf0c418042f..e9b8af4c1ceb1 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -48,7 +48,7 @@ public static void WriteEncryptionData(Utf8JsonWriter json, EncryptionData data) json.WriteEndObject(); } - private static void WriteWrappedKey(Utf8JsonWriter json, WrappedKey key) + private static void WriteWrappedKey(Utf8JsonWriter json, KeyEnvelope key) { json.WriteString(nameof(key.KeyId), key.KeyId); json.WriteString(nameof(key.EncryptedKey), Convert.ToBase64String(key.EncryptedKey)); @@ -102,7 +102,7 @@ private static void ReadPropertyValue(EncryptionData data, JsonProperty property } else if (property.NameEquals(nameof(data.WrappedContentKey))) { - var key = new WrappedKey(); + var key = new KeyEnvelope(); foreach (var subProperty in property.Value.EnumerateObject()) { ReadPropertyValue(key, subProperty); @@ -133,7 +133,7 @@ private static void ReadPropertyValue(EncryptionData data, JsonProperty property } } - private static void ReadPropertyValue(WrappedKey key, JsonProperty property) + private static void ReadPropertyValue(KeyEnvelope key, JsonProperty property) { if (property.NameEquals(nameof(key.Algorithm))) { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs similarity index 95% rename from sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs rename to sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs index 34bcdf7590484..49c0cad646cfc 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/WrappedKey.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs @@ -6,7 +6,7 @@ namespace Azure.Storage.Cryptography.Models /// /// Represents the envelope key details stored on the service. /// - internal class WrappedKey + internal class KeyEnvelope { /// /// The key identifier string. diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs index 2e1825fa22add..0d6155c9f6dd8 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs @@ -18,7 +18,7 @@ public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions { KeyEncryptionKey = other.KeyEncryptionKey, KeyResolver = other.KeyResolver, - KeyWrapAlgorithm = other.KeyWrapAlgorithm + KeyWrapAlgorithm = other.KeyWrapAlgorithm, }; /// @@ -63,6 +63,10 @@ public static byte[] CreateKey(int numBits) /// /// Whether to perform this function asynchronously. /// + /// + /// Decrypted plaintext. If key could not be resolved, returns null. + /// + /// When key ID cannot be resolved. public static async Task DecryptInternal( Stream ciphertext, EncryptionData encryptionData, @@ -73,6 +77,13 @@ public static async Task DecryptInternal( bool async, CancellationToken cancellationToken) { + var contentEncryptionKey = await GetContentEncryptionKeyOrDefaultAsync( + encryptionData, + keyResolver, + potentialCachedKeyWrapper, + async, + cancellationToken).ConfigureAwait(false); + Stream plaintext; //int read = 0; if (encryptionData != default) @@ -96,8 +107,6 @@ public static async Task DecryptInternal( //read = IV.Length; } - var contentEncryptionKey = await GetContentEncryptionKeyAsync(encryptionData, keyResolver, potentialCachedKeyWrapper, async, cancellationToken).ConfigureAwait(false); - plaintext = WrapStream( ciphertext, contentEncryptionKey.ToArray(), @@ -124,8 +133,11 @@ public static async Task DecryptInternal( /// /// Whether to perform asynchronously. /// - /// Encryption key as a byte array. - private static async Task> GetContentEncryptionKeyAsync( + /// + /// Encryption key as a byte array. + /// + /// When key ID cannot be resolved. + private static async Task> GetContentEncryptionKeyOrDefaultAsync( #pragma warning restore CS1587 // XML comment is not placed on a valid language element EncryptionData encryptionData, IKeyEncryptionKeyResolver keyResolver, @@ -133,7 +145,7 @@ private static async Task> GetContentEncryptionKeyAsync( bool async, CancellationToken cancellationToken) { - IKeyEncryptionKey key; + IKeyEncryptionKey key = default; // If we already have a local key and it is the correct one, use that. if (encryptionData.WrappedContentKey.KeyId == potentiallyCachedKeyWrapper?.KeyId) @@ -147,14 +159,10 @@ private static async Task> GetContentEncryptionKeyAsync( ? await keyResolver.ResolveAsync(encryptionData.WrappedContentKey.KeyId, cancellationToken).ConfigureAwait(false) : keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); } - else - { - throw EncryptionErrors.KeyNotFound(encryptionData.WrappedContentKey.KeyId); - } if (key == default) { - throw EncryptionErrors.NoKeyAccessor(); + throw EncryptionErrors.KeyNotFound(encryptionData.WrappedContentKey.KeyId); } return async diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs index abbc03cc69485..60267b05bc23b 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs @@ -48,6 +48,22 @@ public override void Flush() throw new NotImplementedException(); } + public override int ReadByte() + { + if (WindowLength - _position == 0) + { + return -1; + } + + int val = InnerStream.ReadByte(); + if (val != -1) + { + _position++; + } + + return val; + } + public override int Read(byte[] buffer, int offset, int count) { count = (int)Math.Min(count, WindowLength - _position); @@ -77,5 +93,7 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void WriteByte(byte value) => throw new NotSupportedException(); } } diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index e6de63848eb14..2ef7af1d77bbc 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -321,6 +321,14 @@ public partial class AdvancedQueueClientOptions : Azure.Storage.Queues.QueueClie { public AdvancedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + public Azure.Storage.Queues.Specialized.IMissingClientSideEncryptionKeyListener OnMissingClientSideEncryptionKey { get { throw null; } set { } } + } + public partial interface IMissingClientSideEncryptionKeyListener + { + void OnMissingKey(Azure.Storage.Queues.Models.PeekedMessage message); + void OnMissingKey(Azure.Storage.Queues.Models.QueueMessage message); + System.Threading.Tasks.Task OnMissingKeyAsync(Azure.Storage.Queues.Models.PeekedMessage message); + System.Threading.Tasks.Task OnMissingKeyAsync(Azure.Storage.Queues.Models.QueueMessage message); } } namespace Azure.Storage.Sas diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs index 82302d2319757..f1c5f1747abf0 100644 --- a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs @@ -1,6 +1,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; + namespace Azure.Storage.Queues.Specialized { /// @@ -34,5 +37,47 @@ public ClientSideEncryptionOptions ClientSideEncryption get => _clientSideEncryptionOptions; set => _clientSideEncryptionOptions = value; } + + /// + /// Behavior when receiving a queue message that cannot be decrypted due to lack of key access. + /// Messages in the list of results that cannot be decrypted will be filtered out of the list and + /// sent to this listener. Default behavior, when no listener is provided, is for the overall message + /// fetch to throw. + /// + public IMissingClientSideEncryptionKeyListener OnMissingClientSideEncryptionKey + { + get => _missingClientSideEncryptionKeyListener; + set => _missingClientSideEncryptionKeyListener = value; + } + } + + /// + /// Describes a listener to handle queue messages who's client-side encryption keys cannot be resolved. + /// + public interface IMissingClientSideEncryptionKeyListener + { + /// + /// Handle an unresolved key in a call. + /// + /// Message that couldn't be decrypted. + void OnMissingKey(QueueMessage message); + + /// + /// Handle an unresolved key in a call. + /// + /// Message that couldn't be decrypted. + Task OnMissingKeyAsync(QueueMessage message); + + /// + /// Handle an unresolved key in a call. + /// + /// Message that couldn't be decrypted. + void OnMissingKey(PeekedMessage message); + + /// + /// Handle an unresolved key in a call. + /// + /// Message that couldn't be decrypted. + Task OnMissingKeyAsync(PeekedMessage message); } } diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs index 05b7de79c5e05..84a9eb6b2e948 100644 --- a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs @@ -47,7 +47,9 @@ public static bool TryDeserialize(string serializedData, out EncryptedMessage en encryptedMessage = Deserialize(serializedData); return true; } - catch (JsonException) + // JsonException does not actually cover everything. InvalidOperationException can be thrown + // on some string inputs, as we can't assume input is even JSON. + catch (Exception) { encryptedMessage = default; return false; diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index 35aad93c1d21b..fd50b372d0482 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -6,7 +6,6 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -15,6 +14,7 @@ using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; using Azure.Storage.Queues.Specialized.Models; using Metadata = System.Collections.Generic.IDictionary; @@ -89,6 +89,8 @@ public class QueueClient internal bool UsingClientSideEncryption => ClientSideEncryption != default; + private readonly IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; + /// /// QueueMaxMessagesPeek indicates the maximum number of messages /// you can retrieve with each call to Peek. @@ -196,6 +198,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); + _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; } /// @@ -288,6 +291,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); + _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; } /// @@ -312,7 +316,16 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien /// /// Options for client-side encryption. /// - internal QueueClient(Uri queueUri, HttpPipeline pipeline, QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, ClientSideEncryptionOptions encryptionOptions) + /// + /// Listener regarding partial decryption failures using clientside encryption. + /// + internal QueueClient( + Uri queueUri, + HttpPipeline pipeline, + QueueClientOptions.ServiceVersion version, + ClientDiagnostics clientDiagnostics, + ClientSideEncryptionOptions encryptionOptions, + IMissingClientSideEncryptionKeyListener listener) { _uri = queueUri; _messagesUri = queueUri.AppendToPath(Constants.Queue.MessagesUri); @@ -320,6 +333,7 @@ internal QueueClient(Uri queueUri, HttpPipeline pipeline, QueueClientOptions.Ser _version = version; _clientDiagnostics = clientDiagnostics; _clientSideEncryption = encryptionOptions?.Clone(); + _missingClientSideEncryptionKeyListener = listener; } #endregion ctors @@ -2112,19 +2126,51 @@ private async Task ClientSideEncryptInternal(string messageToUpload, boo private async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) { + var filteredMessages = new List(); foreach (var message in messages) { - message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (ClientSideEncryptionKeyNotFoundException) when (_missingClientSideEncryptionKeyListener != default) + { + if (async) + { + await _missingClientSideEncryptionKeyListener.OnMissingKeyAsync(message).ConfigureAwait(false); + } + else + { + _missingClientSideEncryptionKeyListener.OnMissingKey(message); + } + } } - return messages; + return filteredMessages.ToArray(); } private async Task ClientSideDecryptMessagesInternal(PeekedMessage[] messages, bool async, CancellationToken cancellationToken) { + var filteredMessages = new List(); foreach (var message in messages) { - message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (ClientSideEncryptionKeyNotFoundException) when (_missingClientSideEncryptionKeyListener != default) + { + if (async) + { + await _missingClientSideEncryptionKeyListener.OnMissingKeyAsync(message).ConfigureAwait(false); + } + else + { + _missingClientSideEncryptionKeyListener.OnMissingKey(message); + } + } } - return messages; + return filteredMessages.ToArray(); } private async Task ClientSideDecryptInternal(string downloadedMessage, bool async, CancellationToken cancellationToken) @@ -2134,8 +2180,9 @@ private async Task ClientSideDecryptInternal(string downloadedMessage, b return downloadedMessage; // not recognized as client-side encrypted message } + var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); var decryptedMessageStream = await Utility.DecryptInternal( - new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)), + encryptedMessageStream, encryptedMessage.EncryptionData, ivInStream: false, ClientSideEncryption.KeyResolver, @@ -2143,6 +2190,12 @@ private async Task ClientSideDecryptInternal(string downloadedMessage, b noPadding: false, async: async, cancellationToken).ConfigureAwait(false); + // if we got back the stream we put in, then we couldn't decrypt and are supposed to return the original + // message to the user + if (encryptedMessageStream == decryptedMessageStream) + { + return downloadedMessage; + } return new StreamReader(decryptedMessageStream, Encoding.UTF8).ReadToEnd(); } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index cba507c1e6784..574ff13becb24 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -4,6 +4,7 @@ using System; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Queues.Specialized; namespace Azure.Storage.Queues { @@ -84,6 +85,7 @@ public QueueClientOptions(ServiceVersion version = LatestVersion) #region Advanced Options internal ClientSideEncryptionOptions _clientSideEncryptionOptions; + internal IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; #endregion /// diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs index 2d99ec33eaeaa..94c8877d0ee1d 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs @@ -10,6 +10,7 @@ using Azure.Core.Pipeline; using Azure.Storage.Cryptography; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; namespace Azure.Storage.Queues { @@ -70,6 +71,8 @@ public class QueueServiceClient /// internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + private readonly IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; + /// /// The Storage account name corresponding to the service client. /// @@ -140,6 +143,7 @@ public QueueServiceClient(string connectionString, QueueClientOptions options) _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); + _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; } /// @@ -227,6 +231,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); + _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; } #endregion ctors @@ -243,7 +248,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q /// A for the desired queue. /// public virtual QueueClient GetQueueClient(string queueName) - => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption); + => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption, _missingClientSideEncryptionKeyListener); #region GetQueues /// diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index c3751a40aa40f..7315891066f3b 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -19,5 +19,6 @@ + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index 42e97983d5c4b..c8f08bc0fbef6 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -11,7 +11,10 @@ using Azure.Core.Cryptography; using Azure.Core.TestFramework; using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Blobs.Tests; using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; using Azure.Storage.Queues.Specialized.Models; using Azure.Storage.Queues.Tests; using Azure.Storage.Tests.Shared; @@ -368,5 +371,82 @@ public async Task OnlyOneKeyResolveAndUnwrapCall() Assert.AreEqual(0, IsAsync ? mockKey.UnwrappedSync : mockKey.UnwrappedAsync); } } + + [TestCase(true, true)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(false, false)] + [LiveOnly] + public async Task CannotFindKeyAsync(bool useListener, bool resolverFailure) + { + MockMissingClientSideEncryptionKeyListener listener = null; + if (useListener) + { + listener = new MockMissingClientSideEncryptionKeyListener(); + } + + const int numMessages = 5; + var message = "any old message"; + var mockKey = new MockKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKey, + KeyWrapAlgorithm = "mock" + })) + { + var queue = disposable.Queue; + foreach (var _ in Enumerable.Range(0, numMessages)) + { + await queue.SendMessageAsync(message).ConfigureAwait(false); + } + + bool threwKeyNotFound = false; + bool threwGeneral = false; + QueueMessage[] result = default; + try + { + // download but can't find key + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ResolverInternalFailure = resolverFailure }, + KeyWrapAlgorithm = "test" + }; + options._missingClientSideEncryptionKeyListener = listener; + result = await new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options).ReceiveMessagesAsync(numMessages); + } + catch (ClientSideEncryptionKeyNotFoundException) + { + threwKeyNotFound = true; + } + catch (Exception) + { + threwGeneral = true; + } + finally + { + if (resolverFailure) + { + Assert.True(threwGeneral); + } + else + { + Assert.False(threwGeneral); + + if (useListener) + { + Assert.AreEqual(numMessages, listener.TimesInvoked); + Assert.AreEqual(0, result.Length); // all messages should have been filtered out + } + else + { + Assert.True(threwKeyNotFound); + } + } + } + } + } } } diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs index 3d72973c7a3fa..58952ec2753db 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -12,6 +12,7 @@ using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Specialized.Models; using Azure.Storage.Tests.Shared; +using Newtonsoft.Json; using NUnit.Framework; namespace Azure.Storage.Queues.Test @@ -84,10 +85,13 @@ public void TryDeserializeEncryptedMessage() Assert.IsTrue(AreEqual(encryptedMessage, parsedEncryptedMessage)); } - [Test] - public void TryDeserializeEncryptedMessageFail() + [TestCase("")] + [TestCase("\"aa\"")] // real world example + [TestCase("this is not even valid json")] + [TestCase("ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ")] + public void TryDeserializeGracefulOnBadInput(string input) { - bool tryResult = EncryptedMessageSerializer.TryDeserialize("this is not even valid json", out var parsedEncryptedMessage); + bool tryResult = EncryptedMessageSerializer.TryDeserialize(input, out var parsedEncryptedMessage); Assert.AreEqual(false, tryResult); Assert.AreEqual(default, parsedEncryptedMessage); @@ -105,7 +109,7 @@ private static bool AreEqual(EncryptionData left, EncryptionData right) && left.ContentEncryptionIV.SequenceEqual(right.ContentEncryptionIV) && AreEqual(left.KeyWrappingMetadata, right.KeyWrappingMetadata); - private static bool AreEqual(WrappedKey left, WrappedKey right) + private static bool AreEqual(KeyEnvelope left, KeyEnvelope right) => left.KeyId.Equals(right.KeyId, StringComparison.InvariantCulture) && left.EncryptedKey.SequenceEqual(right.EncryptedKey) && left.Algorithm.Equals(right.Algorithm, StringComparison.InvariantCulture); diff --git a/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs b/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs new file mode 100644 index 0000000000000..803249fabef80 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; + +namespace Azure.Storage.Queues.Tests +{ + internal class MockMissingClientSideEncryptionKeyListener : IMissingClientSideEncryptionKeyListener + { + public int TimesInvoked { get; private set; } = 0; + + public void OnMissingKey(QueueMessage message) + { + TimesInvoked++; + } + + public void OnMissingKey(PeekedMessage message) + { + TimesInvoked++; + } + + public Task OnMissingKeyAsync(QueueMessage message) + { + TimesInvoked++; + return Task.CompletedTask; + } + + public Task OnMissingKeyAsync(PeekedMessage message) + { + TimesInvoked++; + return Task.CompletedTask; + } + } +} From 295f1e8c669d67bfaeb096d197b16272e5c456bd Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Wed, 20 May 2020 16:21:44 -0700 Subject: [PATCH 06/21] Removed nonexistent project from sln --- sdk/storage/Azure.Storage.sln | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sdk/storage/Azure.Storage.sln b/sdk/storage/Azure.Storage.sln index e78b608452723..7d220db0125a4 100644 --- a/sdk/storage/Azure.Storage.sln +++ b/sdk/storage/Azure.Storage.sln @@ -85,13 +85,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.Shares. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Common.Samples.Tests", "Azure.Storage.Common\samples\Azure.Storage.Common.Samples.Tests.csproj", "{9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Cryptography", "Cryptography", "{9B235463-7D89-471E-87CA-A392661255E4}" - ProjectSection(SolutionItems) = preProject - Azure.Storage.Blobs.Cryptography\BreakingChanges.txt = Azure.Storage.Blobs.Cryptography\BreakingChanges.txt - Azure.Storage.Blobs.Cryptography\Changelog.txt = Azure.Storage.Blobs.Cryptography\Changelog.txt - Azure.Storage.Blobs.Cryptography\README.md = Azure.Storage.Blobs.Cryptography\README.md - EndProjectSection -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake", "Azure.Storage.Files.DataLake\src\Azure.Storage.Files.DataLake.csproj", "{DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake.Tests", "Azure.Storage.Files.DataLake\tests\Azure.Storage.Files.DataLake.Tests.csproj", "{EDBF4B6B-5D28-47AF-A9BE-B70568067407}" From df0ccc90e137d36ffcc3e0e36589d73aabbcc2bd Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Wed, 20 May 2020 19:39:58 -0700 Subject: [PATCH 07/21] Change in handling key resolution Clientside decryption will now either succeed or throw. Queues can redirect their throw to a listener. --- sdk/core/Azure.Core/Azure.Core.All.sln | 2 - .../AlwaysFailsKeyEncryptionKeyResolver.cs | 10 ++--- .../tests/ClientSideEncryptionTests.cs | 16 +++----- ...lientSideEncryptionKeyNotFoundException.cs | 28 ------------- .../ClientsideEncryption/EncryptionErrors.cs | 4 +- .../Shared/ClientsideEncryption/Utility.cs | 14 +++++-- .../src/AdvancedQueueClientOptions.cs | 29 +++++++------ .../Azure.Storage.Queues/src/QueueClient.cs | 20 ++++----- .../src/QueueClientOptions.cs | 2 +- .../src/QueueServiceClient.cs | 6 +-- .../tests/ClientSideEncryptionTests.cs | 41 ++++++------------- ...kMissingClientSideEncryptionKeyListener.cs | 16 +++++--- 12 files changed, 78 insertions(+), 110 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs diff --git a/sdk/core/Azure.Core/Azure.Core.All.sln b/sdk/core/Azure.Core/Azure.Core.All.sln index 3aa3735c4c1ef..6102e98b28202 100644 --- a/sdk/core/Azure.Core/Azure.Core.All.sln +++ b/sdk/core/Azure.Core/Azure.Core.All.sln @@ -85,8 +85,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.S EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.Tests", "..\..\storage\Azure.Storage.Blobs.Batch\tests\Azure.Storage.Blobs.Batch.Tests.csproj", "{EA651D86-70EE-4F32-AF1D-E5EB85ACF79F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Cryptography", "..\..\storage\Azure.Storage.Blobs.Cryptography\src\Azure.Storage.Blobs.Cryptography.csproj", "{908FBA6A-12B2-44DA-BCFB-11750B613FFA}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Samples.Tests", "..\..\storage\Azure.Storage.Blobs\samples\Azure.Storage.Blobs.Samples.Tests.csproj", "{83DC827C-9AD6-4C76-883F-53AB4A027E82}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Tests", "..\..\storage\Azure.Storage.Blobs\tests\Azure.Storage.Blobs.Tests.csproj", "{02708856-2D84-47E2-80DD-65A3A31E01C8}" diff --git a/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs b/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs index 688716dc1f9e6..0cb9666182d61 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs @@ -12,14 +12,14 @@ namespace Azure.Storage.Blobs.Tests internal class AlwaysFailsKeyEncryptionKeyResolver : IKeyEncryptionKeyResolver { /// - /// False means the resolver just can't find the key and returns null. - /// True means the resolver has an internal failure and throws. + /// False means the resolver returns null. + /// True means the resolver throws. /// - public bool ResolverInternalFailure { get; set; } = false; + public bool ShouldThrow { get; set; } = false; public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationToken = default) { - if (ResolverInternalFailure) + if (ShouldThrow) { throw new Exception(); } @@ -28,7 +28,7 @@ public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationTok public Task ResolveAsync(string keyId, CancellationToken cancellationToken = default) { - if (ResolverInternalFailure) + if (ShouldThrow) { throw new Exception(); } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 33b0116dc485e..93f0cf56a467b 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -344,7 +344,7 @@ public async Task RoundtripWithKeyvaultProvider() [TestCase(true)] [TestCase(false)] [LiveOnly] - public async Task CannotFindKeyAsync(bool resolverFailure) + public async Task CannotFindKeyAsync(bool resolverThrows) { var data = GetRandomBuffer(Constants.KB); var mockKey = new MockKeyEncryptionKey(); @@ -359,32 +359,26 @@ public async Task CannotFindKeyAsync(bool resolverFailure) var blob = disposable.Container.GetBlobClient(GetNewBlobName()); await blob.UploadAsync(new MemoryStream(data)); - bool threwKeyNotFound = false; - bool threwGeneral = false; + bool threw = false; try { // download but can't find key var options = GetOptions(); options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ResolverInternalFailure = resolverFailure }, + KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ShouldThrow = resolverThrows }, KeyWrapAlgorithm = "test" }; var encryptedDataStream = new MemoryStream(); await new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options).DownloadToAsync(encryptedDataStream); } - catch (ClientSideEncryptionKeyNotFoundException) - { - threwKeyNotFound = true; - } catch (Exception) { - threwGeneral = true; + threw = true; } finally { - Assert.AreEqual(resolverFailure, threwGeneral); - Assert.AreEqual(!resolverFailure, threwKeyNotFound); + Assert.IsTrue(threw); } } } diff --git a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs deleted file mode 100644 index 06c1b7e3ae61c..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionKeyNotFoundException.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; - -namespace Azure.Storage -{ - /// - /// Thrown when the client fails to resolve the necessary key to decrypt data using client-side encryption. - /// - public class ClientSideEncryptionKeyNotFoundException : Exception - { - /// - /// Key that could not be resolved. - /// - public string KeyId { get; } - - /// - /// Constructs the exception. - /// - /// Key id. - public ClientSideEncryptionKeyNotFoundException(string keyId) - : base($"Provided key resolver ould not resolve key of id `{keyId}`.") - { - KeyId = keyId; - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs index 3866c492fb8e0..48303644b4fdb 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs @@ -7,8 +7,8 @@ namespace Azure.Storage.Cryptography { internal static class EncryptionErrors { - public static ClientSideEncryptionKeyNotFoundException KeyNotFound(string keyId) - => new ClientSideEncryptionKeyNotFoundException(keyId); + public static ArgumentException KeyNotFound(string keyId) + => new ArgumentException($"Resolution of id {keyId} returned null."); public static ArgumentException BadEncryptionAgent(string agent) => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs index 0d6155c9f6dd8..6991299fe6f6a 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs @@ -64,9 +64,12 @@ public static byte[] CreateKey(int numBits) /// Whether to perform this function asynchronously. /// /// - /// Decrypted plaintext. If key could not be resolved, returns null. + /// Decrypted plaintext. /// - /// When key ID cannot be resolved. + /// + /// Exceptions thrown based on implementations of and + /// . + /// public static async Task DecryptInternal( Stream ciphertext, EncryptionData encryptionData, @@ -136,7 +139,10 @@ public static async Task DecryptInternal( /// /// Encryption key as a byte array. /// - /// When key ID cannot be resolved. + /// + /// Exceptions thrown based on implementations of and + /// . + /// private static async Task> GetContentEncryptionKeyOrDefaultAsync( #pragma warning restore CS1587 // XML comment is not placed on a valid language element EncryptionData encryptionData, @@ -160,6 +166,8 @@ private static async Task> GetContentEncryptionKeyOrDefaultAsync( : keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); } + // We throw for every other reason that decryption couldn't happen. Throw a reasonable + // exception here instead of nullref. if (key == default) { throw EncryptionErrors.KeyNotFound(encryptionData.WrappedContentKey.KeyId); diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs index f1c5f1747abf0..e91707ada33ec 100644 --- a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.Threading.Tasks; using Azure.Storage.Queues.Models; @@ -44,40 +45,44 @@ public ClientSideEncryptionOptions ClientSideEncryption /// sent to this listener. Default behavior, when no listener is provided, is for the overall message /// fetch to throw. /// - public IMissingClientSideEncryptionKeyListener OnMissingClientSideEncryptionKey + public IClientSideDecryptionFailureListener OnClientSideDecryptionFailure { - get => _missingClientSideEncryptionKeyListener; - set => _missingClientSideEncryptionKeyListener = value; + get => _onClientSideDecryptionFailure; + set => _onClientSideDecryptionFailure = value; } } /// /// Describes a listener to handle queue messages who's client-side encryption keys cannot be resolved. /// - public interface IMissingClientSideEncryptionKeyListener + public interface IClientSideDecryptionFailureListener { /// - /// Handle an unresolved key in a call. + /// Handle a decryption failure in a call. /// /// Message that couldn't be decrypted. - void OnMissingKey(QueueMessage message); + /// Exception of the failure. + void OnFailure(QueueMessage message, Exception exception); /// - /// Handle an unresolved key in a call. + /// Handle a decryption failure in a call. /// /// Message that couldn't be decrypted. - Task OnMissingKeyAsync(QueueMessage message); + /// Exception of the failure. + Task OnFailureAsync(QueueMessage message, Exception exception); /// - /// Handle an unresolved key in a call. + /// Handle a decryption failure in a call. /// /// Message that couldn't be decrypted. - void OnMissingKey(PeekedMessage message); + /// Exception of the failure. + void OnFailure(PeekedMessage message, Exception exception); /// - /// Handle an unresolved key in a call. + /// Handle a decryption failure in a call. /// /// Message that couldn't be decrypted. - Task OnMissingKeyAsync(PeekedMessage message); + /// Exception of the failure. + Task OnFailureAsync(PeekedMessage message, Exception exception); } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index fd50b372d0482..17fcd163caa09 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -89,7 +89,7 @@ public class QueueClient internal bool UsingClientSideEncryption => ClientSideEncryption != default; - private readonly IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; + private readonly IClientSideDecryptionFailureListener _missingClientSideEncryptionKeyListener; /// /// QueueMaxMessagesPeek indicates the maximum number of messages @@ -198,7 +198,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; + _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; } /// @@ -291,7 +291,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; + _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; } /// @@ -325,7 +325,7 @@ internal QueueClient( QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, ClientSideEncryptionOptions encryptionOptions, - IMissingClientSideEncryptionKeyListener listener) + IClientSideDecryptionFailureListener listener) { _uri = queueUri; _messagesUri = queueUri.AppendToPath(Constants.Queue.MessagesUri); @@ -2134,15 +2134,15 @@ private async Task ClientSideDecryptMessagesInternal(QueueMessag message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); filteredMessages.Add(message); } - catch (ClientSideEncryptionKeyNotFoundException) when (_missingClientSideEncryptionKeyListener != default) + catch (Exception e) when (_missingClientSideEncryptionKeyListener != default) { if (async) { - await _missingClientSideEncryptionKeyListener.OnMissingKeyAsync(message).ConfigureAwait(false); + await _missingClientSideEncryptionKeyListener.OnFailureAsync(message, e).ConfigureAwait(false); } else { - _missingClientSideEncryptionKeyListener.OnMissingKey(message); + _missingClientSideEncryptionKeyListener.OnFailure(message, e); } } } @@ -2158,15 +2158,15 @@ private async Task ClientSideDecryptMessagesInternal(PeekedMess message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); filteredMessages.Add(message); } - catch (ClientSideEncryptionKeyNotFoundException) when (_missingClientSideEncryptionKeyListener != default) + catch (Exception e) when (_missingClientSideEncryptionKeyListener != default) { if (async) { - await _missingClientSideEncryptionKeyListener.OnMissingKeyAsync(message).ConfigureAwait(false); + await _missingClientSideEncryptionKeyListener.OnFailureAsync(message, e).ConfigureAwait(false); } else { - _missingClientSideEncryptionKeyListener.OnMissingKey(message); + _missingClientSideEncryptionKeyListener.OnFailure(message, e); } } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index 574ff13becb24..2923b3fc94e38 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -85,7 +85,7 @@ public QueueClientOptions(ServiceVersion version = LatestVersion) #region Advanced Options internal ClientSideEncryptionOptions _clientSideEncryptionOptions; - internal IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; + internal IClientSideDecryptionFailureListener _onClientSideDecryptionFailure; #endregion /// diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs index 94c8877d0ee1d..107aecc1c4e7d 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs @@ -71,7 +71,7 @@ public class QueueServiceClient /// internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; - private readonly IMissingClientSideEncryptionKeyListener _missingClientSideEncryptionKeyListener; + private readonly IClientSideDecryptionFailureListener _missingClientSideEncryptionKeyListener; /// /// The Storage account name corresponding to the service client. @@ -143,7 +143,7 @@ public QueueServiceClient(string connectionString, QueueClientOptions options) _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; + _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; } /// @@ -231,7 +231,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._missingClientSideEncryptionKeyListener; + _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; } #endregion ctors diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index c8f08bc0fbef6..a7afc58355e93 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -372,12 +372,12 @@ public async Task OnlyOneKeyResolveAndUnwrapCall() } } - [TestCase(true, true)] [TestCase(true, false)] - [TestCase(false, true)] [TestCase(false, false)] + [TestCase(true, true)] + [TestCase(false, true)] [LiveOnly] - public async Task CannotFindKeyAsync(bool useListener, bool resolverFailure) + public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) { MockMissingClientSideEncryptionKeyListener listener = null; if (useListener) @@ -402,8 +402,7 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverFailure) await queue.SendMessageAsync(message).ConfigureAwait(false); } - bool threwKeyNotFound = false; - bool threwGeneral = false; + bool threw = false; QueueMessage[] result = default; try { @@ -411,39 +410,25 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverFailure) var options = GetOptions(); options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ResolverInternalFailure = resolverFailure }, + // note decryption will throw whether the resolver throws or just returns null + KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ShouldThrow = resolverThrows }, KeyWrapAlgorithm = "test" }; - options._missingClientSideEncryptionKeyListener = listener; + options._onClientSideDecryptionFailure = listener; result = await new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options).ReceiveMessagesAsync(numMessages); } - catch (ClientSideEncryptionKeyNotFoundException) - { - threwKeyNotFound = true; - } catch (Exception) { - threwGeneral = true; + threw = true; } finally { - if (resolverFailure) - { - Assert.True(threwGeneral); - } - else + Assert.AreNotEqual(useListener, threw); + + if (useListener) { - Assert.False(threwGeneral); - - if (useListener) - { - Assert.AreEqual(numMessages, listener.TimesInvoked); - Assert.AreEqual(0, result.Length); // all messages should have been filtered out - } - else - { - Assert.True(threwKeyNotFound); - } + Assert.AreEqual(numMessages, listener.TimesInvoked); + Assert.AreEqual(0, result.Length); // all messages should have been filtered out } } } diff --git a/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs b/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs index 803249fabef80..5497c699cac85 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs @@ -1,34 +1,40 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.Threading.Tasks; using Azure.Storage.Queues.Models; using Azure.Storage.Queues.Specialized; +using NUnit.Framework; namespace Azure.Storage.Queues.Tests { - internal class MockMissingClientSideEncryptionKeyListener : IMissingClientSideEncryptionKeyListener + internal class MockMissingClientSideEncryptionKeyListener : IClientSideDecryptionFailureListener { public int TimesInvoked { get; private set; } = 0; - public void OnMissingKey(QueueMessage message) + public void OnFailure(QueueMessage message, Exception e) { + Assert.IsNotNull(e); TimesInvoked++; } - public void OnMissingKey(PeekedMessage message) + public void OnFailure(PeekedMessage message, Exception e) { + Assert.IsNotNull(e); TimesInvoked++; } - public Task OnMissingKeyAsync(QueueMessage message) + public Task OnFailureAsync(QueueMessage message, Exception e) { + Assert.IsNotNull(e); TimesInvoked++; return Task.CompletedTask; } - public Task OnMissingKeyAsync(PeekedMessage message) + public Task OnFailureAsync(PeekedMessage message, Exception e) { + Assert.IsNotNull(e); TimesInvoked++; return Task.CompletedTask; } From cada706eaeb4d6e1b1bde3da926a3c75bb79478b Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Thu, 28 May 2020 11:50:22 -0700 Subject: [PATCH 08/21] Many PR comments addressed; key-substitution added --- .../src/AdvancedBlobClientExtensions.cs | 28 ++++ .../src/AppendBlobClient.cs | 10 +- .../src/Azure.Storage.Blobs.csproj | 1 + .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 51 ++++++- .../Azure.Storage.Blobs/src/BlobClient.cs | 4 +- .../Azure.Storage.Blobs/src/PageBlobClient.cs | 10 +- .../{Utility.cs => ClientSideDecryptor.cs} | 122 +---------------- .../ClientSideEncryptionOptionsExtensions.cs | 16 +++ .../ClientSideEncryptor.cs | 124 ++++++++++++++++++ .../Models/EncryptionData.cs | 21 +-- .../Models/KeyEnvelope.cs | 7 +- .../src/AdvancedQueueClientExtensions.cs | 28 ++++ .../Azure.Storage.Queues/src/QueueClient.cs | 4 +- .../tests/EncryptedMessageSerializerTests.cs | 6 +- 14 files changed, 280 insertions(+), 152 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs rename sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/{Utility.cs => ClientSideDecryptor.cs} (61%) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs create mode 100644 sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs b/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs new file mode 100644 index 0000000000000..42f6626d8fb01 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Blobs.Specialized +{ + /// + /// Provides advanced extensions for blob service clients. + /// + public static class AdvancedBlobClientExtensions + { + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// New instance with provided options and same internals otherwise. + public static BlobClient WithClientSideEncryptionOptions(this BlobClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) + => new BlobClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + client.CustomerProvidedKey, + clientSideEncryptionOptions, + client.EncryptionScope); + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 08416a63cf4f9..4f6e9e7b8b745 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -1047,13 +1047,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static AppendBlobClient GetAppendBlobClient( this BlobContainerClient client, - string blobName) => - new AppendBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + return new AppendBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index e92aad6a91721..fa73decae81f6 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -40,6 +40,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 840b004741aed..6a1bca0c159cc 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -8,7 +8,6 @@ using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; -using Azure.Storage.Blobs.Specialized.Models; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Shared; @@ -666,7 +665,7 @@ private async Task> DownloadInternal( { if (UsingClientSideEncryption) { - range = new EncryptedBlobRange(range).AdjustedRange; + range = GetEncryptedBlobRange(range); } // Start downloading the blob @@ -3007,7 +3006,7 @@ private async Task ClientSideDecryptInternal( bool ivInStream = originalRange.Offset >= 16; // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. - var plaintext = await Utility.DecryptInternal( + var plaintext = await ClientSideDecryptor.DecryptInternal( content, encryptionData, ivInStream, @@ -3053,7 +3052,7 @@ internal static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata me return default; } - EncryptionData encryptionData = EncryptionData.Deserialize(encryptedDataString); + EncryptionData encryptionData = EncryptionDataSerializer.Deserialize(encryptedDataString); _ = encryptionData.ContentEncryptionIV ?? throw EncryptionErrors.MissingEncryptionMetadata( nameof(EncryptionData.ContentEncryptionIV)); @@ -3097,6 +3096,50 @@ private static bool CanIgnorePadding(ContentRange? contentRange) return true; } + + internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) + { + int offsetAdjustment = 0; + long? adjustedDownloadCount = originalRange.Length; + + // Calculate offsetAdjustment. + if (originalRange.Offset != 0) + { + // Align with encryption block boundary. + int diff; + if ((diff = (int)(originalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) + { + offsetAdjustment += diff; + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += diff; + } + } + + // Account for IV. + if (originalRange.Offset >= EncryptionConstants.EncryptionBlockSize) + { + offsetAdjustment += EncryptionConstants.EncryptionBlockSize; + // Increment adjustedDownloadCount if necessary. + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; + } + } + } + + // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible + // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. + if (adjustedDownloadCount != null) + { + adjustedDownloadCount += ( + EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount + % EncryptionConstants.EncryptionBlockSize) + ) % EncryptionConstants.EncryptionBlockSize; + } + + return new HttpRange(originalRange.Offset - offsetAdjustment, adjustedDownloadCount); + } } /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index 75124926da863..d051b9a6ecdc4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -1097,7 +1097,7 @@ internal async Task> StagedUploadAsync( //long originalLength = content.Length; - (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await Utility.EncryptInternal( + (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await ClientSideEncryptor.EncryptInternal( content, ClientSideEncryption.KeyEncryptionKey, ClientSideEncryption.KeyWrapAlgorithm, @@ -1110,7 +1110,7 @@ internal async Task> StagedUploadAsync( // GetExpectedCryptoStreamLength(originalLength)); metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); - metadata.Add(EncryptionConstants.EncryptionDataKey, encryptionData.Serialize()); + metadata.Add(EncryptionConstants.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); return (nonSeekableCiphertext, metadata); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index 24843f1b27e6c..d23fe6e7aa7c9 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -2617,13 +2617,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static PageBlobClient GetPageBlobClient( this BlobContainerClient client, - string blobName) => - new PageBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + return new PageBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs similarity index 61% rename from sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs rename to sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs index 6991299fe6f6a..5ae4c9d2dd806 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Utility.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -11,31 +11,8 @@ namespace Azure.Storage.Cryptography { - internal static class Utility + internal static class ClientSideDecryptor { - public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions other) - => new ClientSideEncryptionOptions(other.Version) - { - KeyEncryptionKey = other.KeyEncryptionKey, - KeyResolver = other.KeyResolver, - KeyWrapAlgorithm = other.KeyWrapAlgorithm, - }; - - /// - /// Securely generate a key. - /// - /// Key size. - /// The generated key bytes. - public static byte[] CreateKey(int numBits) - { - using (var rng = new RNGCryptoServiceProvider()) - { - var buff = new byte[numBits / 8]; - rng.GetBytes(buff); - return buff; - } - } - /// /// Decrypts the given stream if decryption information is provided. /// Does not shave off unwanted start/end bytes, but will shave off padding. @@ -80,7 +57,7 @@ public static async Task DecryptInternal( bool async, CancellationToken cancellationToken) { - var contentEncryptionKey = await GetContentEncryptionKeyOrDefaultAsync( + var contentEncryptionKey = await GetContentEncryptionKeyAsync( encryptionData, keyResolver, potentialCachedKeyWrapper, @@ -143,7 +120,7 @@ public static async Task DecryptInternal( /// Exceptions thrown based on implementations of and /// . /// - private static async Task> GetContentEncryptionKeyOrDefaultAsync( + private static async Task> GetContentEncryptionKeyAsync( #pragma warning restore CS1587 // XML comment is not placed on a valid language element EncryptionData encryptionData, IKeyEncryptionKeyResolver keyResolver, @@ -214,98 +191,5 @@ private static Stream WrapStream(Stream contentStream, byte[] contentEncryptionK throw EncryptionErrors.BadEncryptionAlgorithm(encryptionData.EncryptionAgent.EncryptionAlgorithm.ToString()); } - - /// - /// Wraps the given read-stream in a CryptoStream and provides the metadata used to create - /// that stream. - /// - /// Stream to wrap. - /// Key encryption key (KEK). - /// Algorithm to encrypt the content encryption key (CEK) with. - /// Whether to wrap the CEK asynchronously. - /// Cancellation token. - /// The wrapped stream to read from and the encryption metadata for the wrapped stream. - public static async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( - Stream plaintext, - IKeyEncryptionKey keyWrapper, - string keyWrapAlgorithm, - bool async, - CancellationToken cancellationToken) - { - var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); - EncryptionData encryptionData = default; - Stream ciphertext = default; - - using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) - { - encryptionData = await EncryptionData.CreateInternalV1_0( - contentEncryptionIv: aesProvider.IV, - keyWrapAlgorithm: keyWrapAlgorithm, - contentEncryptionKey: generatedKey, - keyEncryptionKey: keyWrapper, - async: async, - cancellationToken: cancellationToken).ConfigureAwait(false); - - ciphertext = new CryptoStream( - plaintext, - aesProvider.CreateEncryptor(), - CryptoStreamMode.Read); - } - - return (ciphertext, encryptionData); - } - - /// - /// Encrypts the given stream and provides the metadata used to encrypt. - /// - /// Stream to encrypt. - /// Key encryption key (KEK). - /// Algorithm to encrypt the content encryption key (CEK) with. - /// Whether to wrap the CEK asynchronously. - /// Cancellation token. - /// The encrypted data and the encryption metadata for the wrapped stream. - public static async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( - Stream plaintext, - IKeyEncryptionKey keyWrapper, - string keyWrapAlgorithm, - bool async, - CancellationToken cancellationToken) - { - var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); - EncryptionData encryptionData = default; - var ciphertext = new MemoryStream(); - byte[] bufferedCiphertext = default; - - using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) - { - encryptionData = await EncryptionData.CreateInternalV1_0( - contentEncryptionIv: aesProvider.IV, - keyWrapAlgorithm: keyWrapAlgorithm, - contentEncryptionKey: generatedKey, - keyEncryptionKey: keyWrapper, - async: async, - cancellationToken: cancellationToken).ConfigureAwait(false); - - var transformStream = new CryptoStream( - ciphertext, - aesProvider.CreateEncryptor(), - CryptoStreamMode.Write); - - if (async) - { - await plaintext.CopyToAsync(transformStream).ConfigureAwait(false); - } - else - { - plaintext.CopyTo(transformStream); - } - - transformStream.FlushFinalBlock(); - - bufferedCiphertext = ciphertext.ToArray(); - } - - return (bufferedCiphertext, encryptionData); - } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs new file mode 100644 index 0000000000000..7fe2b3bc5dfbd --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography +{ + internal static class ClientSideEncryptionOptionsExtensions + { + public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions options) + => new ClientSideEncryptionOptions(options.Version) + { + KeyEncryptionKey = options.KeyEncryptionKey, + KeyResolver = options.KeyResolver, + KeyWrapAlgorithm = options.KeyWrapAlgorithm, + }; + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs new file mode 100644 index 0000000000000..999b1a48433ee --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Cryptography +{ + internal static class ClientSideEncryptor + { + /// + /// Wraps the given read-stream in a CryptoStream and provides the metadata used to create + /// that stream. + /// + /// Stream to wrap. + /// Key encryption key (KEK). + /// Algorithm to encrypt the content encryption key (CEK) with. + /// Whether to wrap the CEK asynchronously. + /// Cancellation token. + /// The wrapped stream to read from and the encryption metadata for the wrapped stream. + public static async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( + Stream plaintext, + IKeyEncryptionKey keyWrapper, + string keyWrapAlgorithm, + bool async, + CancellationToken cancellationToken) + { + var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + Stream ciphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: keyWrapper, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + ciphertext = new CryptoStream( + plaintext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Read); + } + + return (ciphertext, encryptionData); + } + + /// + /// Encrypts the given stream and provides the metadata used to encrypt. This method writes to a memory stream, + /// optimized for known-size data that will already be buffered in memory. + /// + /// Stream to encrypt. + /// Key encryption key (KEK). + /// Algorithm to encrypt the content encryption key (CEK) with. + /// Whether to wrap the CEK asynchronously. + /// Cancellation token. + /// The encrypted data and the encryption metadata for the wrapped stream. + public static async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( + Stream plaintext, + IKeyEncryptionKey keyWrapper, + string keyWrapAlgorithm, + bool async, + CancellationToken cancellationToken) + { + var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + var ciphertext = new MemoryStream(); + byte[] bufferedCiphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: keyWrapper, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var transformStream = new CryptoStream( + ciphertext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Write); + + if (async) + { + await plaintext.CopyToAsync(transformStream).ConfigureAwait(false); + } + else + { + plaintext.CopyTo(transformStream); + } + + transformStream.FlushFinalBlock(); + + bufferedCiphertext = ciphertext.ToArray(); + } + + return (bufferedCiphertext, encryptionData); + } + + /// + /// Securely generate a key. + /// + /// Key size. + /// The generated key bytes. + private static byte[] CreateKey(int numBits) + { + using (var secureRng = new RNGCryptoServiceProvider()) + { + var buff = new byte[numBits / 8]; + secureRng.GetBytes(buff); + return buff; + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs index d7a3727fa05a5..decbaa460bac9 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -44,21 +44,6 @@ internal class EncryptionData public Metadata KeyWrappingMetadata { get; set; } #pragma warning restore CA2227 // Collection properties should be read only - /// - /// Serializes this object to JSON. - /// - /// - public string Serialize() - => EncryptionDataSerializer.Serialize(this); - - /// - /// Deserializes an from JSON. - /// - /// JSON to deserialize. - /// - public static EncryptionData Deserialize(string json) - => EncryptionDataSerializer.Deserialize(json); - internal static async Task CreateInternalV1_0( byte[] contentEncryptionIv, string keyWrapAlgorithm, @@ -92,11 +77,13 @@ internal static async Task CreateInternalV1_0( /// /// Singleton string identifying this encryption library. /// - private static string AgentString { get; } = new Func(() => + private static string AgentString { get; } = GenerateAgentString(); + + private static string GenerateAgentString() { Assembly assembly = typeof(EncryptionData).Assembly; var platformInformation = $"({RuntimeInformation.FrameworkDescription}; {RuntimeInformation.OSDescription})"; return $"azsdk-net-{assembly.GetName().Name}/{assembly.GetCustomAttribute().InformationalVersion} {platformInformation}"; - }).Invoke(); + } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs index 49c0cad646cfc..9239e0510b9e7 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs @@ -5,6 +5,11 @@ namespace Azure.Storage.Cryptography.Models { /// /// Represents the envelope key details stored on the service. + /// In the envelope technique, a securely generated content encryption key (CEK) is generated + /// for every encryption operation. It is then encrypted (wrapped) with the user-provided key + /// encryption key (KEK), using a key-wrap algorithm. The wrapped CEK is stored with the + /// encrypted data, and needs the KEK to be unwrapped. The KEK and key-wrapping operation is + /// never seen by this SDK. /// internal class KeyEnvelope { @@ -19,7 +24,7 @@ internal class KeyEnvelope public byte[] EncryptedKey { get; set; } /// - /// The algorithm used for wrapping. + /// The algorithm used to wrap the content encryption key. /// public string Algorithm { get; set; } } diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs new file mode 100644 index 0000000000000..374b53420bc50 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Provides advanced extensions for queue service clients. + /// + public static class AdvancedQueueClientExtensions + { + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// + /// New instance with provided options and same internals otherwise. + public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions, IClientSideDecryptionFailureListener listener = default) + => new QueueClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + clientSideEncryptionOptions, + listener); + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index 17fcd163caa09..98d0ed60c54b6 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -2110,7 +2110,7 @@ private async Task> UpdateMessageInternal( private async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) { var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); - (byte[] ciphertext, EncryptionData encryptionData) = await Utility.BufferedEncryptInternal( + (byte[] ciphertext, EncryptionData encryptionData) = await ClientSideEncryptor.BufferedEncryptInternal( new MemoryStream(bytesToEncrypt), ClientSideEncryption.KeyEncryptionKey, ClientSideEncryption.KeyWrapAlgorithm, @@ -2181,7 +2181,7 @@ private async Task ClientSideDecryptInternal(string downloadedMessage, b } var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); - var decryptedMessageStream = await Utility.DecryptInternal( + var decryptedMessageStream = await ClientSideDecryptor.DecryptInternal( encryptedMessageStream, encryptedMessage.EncryptionData, ivInStream: false, diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs index 58952ec2753db..d9d4e5fabcc5d 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -25,7 +25,7 @@ public class EncryptedMessageSerializerTests // doesn't inherit our test base be [Test] public void SerializeEncryptedMessage() { - var result = Utility.BufferedEncryptInternal( + var result = ClientSideEncryptor.BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), new MockKeyEncryptionKey(), KeyWrapAlgorithm, @@ -45,7 +45,7 @@ public void SerializeEncryptedMessage() [Test] public void DeserializeEncryptedMessage() { - var result = Utility.BufferedEncryptInternal( + var result = ClientSideEncryptor.BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), new MockKeyEncryptionKey(), KeyWrapAlgorithm, @@ -66,7 +66,7 @@ public void DeserializeEncryptedMessage() [Test] public void TryDeserializeEncryptedMessage() { - var result = Utility.BufferedEncryptInternal( + var result = ClientSideEncryptor.BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), new MockKeyEncryptionKey(), KeyWrapAlgorithm, From 801fb3b73b9ceba0a9e137715e1f8da4cbdff0f7 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 29 May 2020 15:51:58 -0700 Subject: [PATCH 09/21] Refactors; Mocking some tests --- .../api/Azure.Storage.Blobs.netstandard2.0.cs | 14 +- ...tExtensions.cs => BlobClientExtensions.cs} | 2 +- ...ptions.cs => ExtendedBlobClientOptions.cs} | 4 +- .../tests/ClientSideEncryptionTests.cs | 267 ++++++++++++----- .../tests/MockKeyEncryptionKey.cs | 164 ----------- .../Azure.Storage.Common.netstandard2.0.cs | 5 - .../ClientSideDecryptor.cs | 17 +- .../Models/EncryptedBlobRange.cs | 72 ----- .../Models/KeyEnvelope.cs | 2 +- .../Azure.Storage.Queues.netstandard2.0.cs | 20 +- ...tions.cs => ExtendedQueueClientOptions.cs} | 4 +- ...Extensions.cs => QueueClientExtensions.cs} | 2 +- .../tests/Azure.Storage.Queues.Tests.csproj | 1 - .../tests/ClientSideEncryptionTests.cs | 273 +++++++++++++----- .../tests/EncryptedMessageSerializerTests.cs | 66 ++++- 15 files changed, 494 insertions(+), 419 deletions(-) rename sdk/storage/Azure.Storage.Blobs/src/{AdvancedBlobClientExtensions.cs => BlobClientExtensions.cs} (95%) rename sdk/storage/Azure.Storage.Blobs/src/{AdvancedBlobClientOptions.cs => ExtendedBlobClientOptions.cs} (92%) delete mode 100644 sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs rename sdk/storage/Azure.Storage.Queues/src/{AdvancedQueueClientOptions.cs => ExtendedQueueClientOptions.cs} (96%) rename sdk/storage/Azure.Storage.Queues/src/{AdvancedQueueClientExtensions.cs => QueueClientExtensions.cs} (95%) diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 0754f8c60c755..2ce20f1e07850 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -914,11 +914,6 @@ internal UserDelegationKey() { } } namespace Azure.Storage.Blobs.Specialized { - public partial class AdvancedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions - { - public AdvancedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } - public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } - } public partial class AppendBlobClient : Azure.Storage.Blobs.Specialized.BlobBaseClient { protected AppendBlobClient() { } @@ -994,6 +989,10 @@ public BlobBaseClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti public virtual Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshot(string snapshot) { throw null; } protected virtual Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } + public static partial class BlobClientExtensions + { + public static Azure.Storage.Blobs.BlobClient WithClientSideEncryptionOptions(this Azure.Storage.Blobs.BlobClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } + } public partial class BlobLeaseClient { public static readonly System.TimeSpan InfiniteLeaseDuration; @@ -1042,6 +1041,11 @@ public BlockBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredent public new Azure.Storage.Blobs.Specialized.BlockBlobClient WithSnapshot(string snapshot) { throw null; } protected sealed override Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } + public partial class ExtendedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions + { + public ExtendedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } public partial class PageBlobClient : Azure.Storage.Blobs.Specialized.BlobBaseClient { protected PageBlobClient() { } diff --git a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs similarity index 95% rename from sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs rename to sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs index 42f6626d8fb01..a515bbea3f195 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientExtensions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs @@ -6,7 +6,7 @@ namespace Azure.Storage.Blobs.Specialized /// /// Provides advanced extensions for blob service clients. /// - public static class AdvancedBlobClientExtensions + public static class BlobClientExtensions { /// /// Creates a new instance of the class, maintaining all the same diff --git a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs similarity index 92% rename from sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs rename to sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs index cee3882031c42..55f4f30fa916f 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AdvancedBlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs @@ -8,7 +8,7 @@ namespace Azure.Storage.Blobs.Specialized /// Storage. /// #pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. - public class AdvancedBlobClientOptions : BlobClientOptions + public class ExtendedBlobClientOptions : BlobClientOptions #pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion { /// @@ -19,7 +19,7 @@ public class AdvancedBlobClientOptions : BlobClientOptions /// The of the service API used when /// making requests. /// - public AdvancedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) + public ExtendedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) { } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 93f0cf56a467b..3fe25aeda8d6d 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -5,9 +5,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Security.Cryptography; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Azure.Core.Cryptography; using Azure.Core.TestFramework; @@ -16,14 +15,17 @@ using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Test.Shared; -using Azure.Storage.Tests.Shared; +using Moq; using NUnit.Framework; +using static Moq.It; namespace Azure.Storage.Blobs.Test { public class ClientSideEncryptionTests : BlobTestBase { - private const string ThrowawayAlgorithmName = "blah"; + private const string s_algorithmName = "some algorithm name"; + private static readonly CancellationToken s_cancellationToken = new CancellationTokenSource().Token; + public ClientSideEncryptionTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { @@ -70,6 +72,103 @@ private async Task GetTestContainerEncryptionAsync( return new DisposingContainer(container); } + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + if (IsAsync) + { + keyMock.Setup(k => k.WrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Task.FromResult(Xor(userKeyBytes, key.ToArray()))); + keyMock.Setup(k => k.UnwrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey.ToArray()))); + } + else + { + keyMock.Setup(k => k.WrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, wrappedKey.ToArray())); + } + + return keyMock; + } + + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + resolverMock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + } + else + { + resolverMock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? iKey : throw new Exception("Mock resolver couldn't resolve key id.")); + } + + return resolverMock; + } + + private Mock GetTrackOneIKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.Kid).Returns(keyId); + keyMock.SetupGet(k => k.DefaultKeyWrapAlgorithm).Returns(s_algorithmName); + // track one had async-only key wrapping + keyMock.Setup(k => k.WrapKeyAsync(IsNotNull(), IsAny(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + // track 1 doesn't pass in the algorithm name, it lets the implementation return the default algorithm it chose + .Returns((key, algorithm, cancellationToken) => Task.FromResult(Tuple.Create(Xor(userKeyBytes, key), s_algorithmName))); + keyMock.Setup(k => k.UnwrapKeyAsync(IsNotNull(), s_algorithmName, IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((wrappedKey, algorithm, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey))); + + return keyMock; + } + + private Mock GetTrackOneIKeyResolver(Microsoft.Azure.KeyVault.Core.IKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + resolverMock.Setup(r => r.ResolveKeyAsync(IsNotNull(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((keyId, cancellationToken) => iKey?.Kid == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + + return resolverMock; + } + + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + #endregion [TestCase(16)] // a single cipher block @@ -81,24 +180,23 @@ private async Task GetTestContainerEncryptionAsync( public async Task UploadAsync(long dataSize) { var data = GetRandomBuffer(dataSize); - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyWrapAlgorithm = s_algorithmName })) { var blobName = GetNewBlobName(); - var blob = disposable.Container.GetBlobClient(blobName); + var blob = InstrumentClient(disposable.Container.GetBlobClient(blobName)); // upload with encryption - await blob.UploadAsync(new MemoryStream(data)); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); // download without decrypting var encryptedDataStream = new MemoryStream(); - await new BlobClient(blob.Uri, GetNewSharedKeyCredentials()).DownloadToAsync(encryptedDataStream); + await InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials())).DownloadToAsync(encryptedDataStream, cancellationToken: s_cancellationToken); var encryptedData = encryptedDataStream.ToArray(); // encrypt original data manually for comparison @@ -108,10 +206,13 @@ public async Task UploadAsync(long dataSize) } EncryptionData encryptionMetadata = EncryptionDataSerializer.Deserialize(serialEncryptionData); Assert.NotNull(encryptionMetadata, "Never encrypted data."); + + var explicitlyUnwrappedKey = IsAsync // can't instrument this + ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) + : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); byte[] expectedEncryptedData = LocalManualEncryption( data, - (await mockKey.UnwrapKeyAsync(null, encryptionMetadata.WrappedContentKey.EncryptedKey) - .ConfigureAwait(false)).ToArray(), + explicitlyUnwrappedKey, encryptionMetadata.ContentEncryptionIV); // compare data @@ -127,25 +228,26 @@ public async Task UploadAsync(long dataSize) public async Task RoundtripAsync(long dataSize) { var data = GetRandomBuffer(dataSize); - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { - var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); // upload with encryption - await blob.UploadAsync(new MemoryStream(data)); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); // download with decryption byte[] downloadData; using (var stream = new MemoryStream()) { - await blob.DownloadToAsync(stream); + await blob.DownloadToAsync(stream, cancellationToken: s_cancellationToken); downloadData = stream.ToArray(); } @@ -159,18 +261,18 @@ public async Task RoundtripAsync(long dataSize) public async Task KeyResolverKicksIn() { var data = GetRandomBuffer(Constants.KB); - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyWrapAlgorithm = s_algorithmName })) { string blobName = GetNewBlobName(); // upload with encryption - await disposable.Container.GetBlobClient(blobName).UploadAsync(new MemoryStream(data)); + await InstrumentClient(disposable.Container.GetBlobClient(blobName)).UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); // download with decryption and no cached key byte[] downloadData; @@ -179,9 +281,9 @@ public async Task KeyResolverKicksIn() var options = GetOptions(); options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyResolver = mockKey + KeyResolver = mockKeyResolver }; - await new BlobContainerClient(disposable.Container.Uri, GetNewSharedKeyCredentials(), options).GetBlobClient(blobName).DownloadToAsync(stream); + await InstrumentClient(new BlobContainerClient(disposable.Container.Uri, GetNewSharedKeyCredentials(), options).GetBlobClient(blobName)).DownloadToAsync(stream, cancellationToken: s_cancellationToken); downloadData = stream.ToArray(); } @@ -204,23 +306,24 @@ public async Task KeyResolverKicksIn() public async Task PartialDownloadAsync(int offset, int? count) { var data = GetRandomBuffer(offset + (count ?? 16) + 32); // ensure we have enough room in original data - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { - var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); // upload with encryption - await blob.UploadAsync(new MemoryStream(data)); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); // download range with decryption byte[] downloadData; // no overload that takes Stream and HttpRange; we must buffer read - Stream downloadStream = (await blob.DownloadAsync(new HttpRange(offset, count))).Value.Content; + Stream downloadStream = (await blob.DownloadAsync(new HttpRange(offset, count), cancellationToken: s_cancellationToken)).Value.Content; byte[] buffer = new byte[Constants.KB]; using (MemoryStream stream = new MemoryStream()) { @@ -247,33 +350,43 @@ public async Task PartialDownloadAsync(int offset, int? count) public async Task Track2DownloadTrack1Blob() { var data = GetRandomBuffer(Constants.KB); - var mockKey = new MockKeyEncryptionKey(); - await using (var disposable = await GetTestContainerEncryptionAsync( - new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) - { - KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName - })) + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object).Object; + await using (var disposable = await GetTestContainerEncryptionAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) { - var track2Blob = disposable.Container.GetBlobClient(GetNewBlobName()); + var track2Blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); // upload with track 1 var creds = GetNewSharedKeyCredentials(); var track1Blob = new Microsoft.Azure.Storage.Blob.CloudBlockBlob( track2Blob.Uri, new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); - await track1Blob.UploadFromByteArrayAsync( - data, 0, data.Length, default, - new Microsoft.Azure.Storage.Blob.BlobRequestOptions() - { - EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, mockKey) - }, - default, default); + var track1RequestOptions = new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, default) + }; + if (IsAsync) // can't instrument track 1 + { + await track1Blob.UploadFromByteArrayAsync(data, 0, data.Length, default, track1RequestOptions, default, s_cancellationToken); + } + else + { + track1Blob.UploadFromByteArray(data, 0, data.Length, default, track1RequestOptions, default); + } // download with track 2 var downloadStream = new MemoryStream(); - await track2Blob.DownloadToAsync(downloadStream); + await track2Blob.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); // compare original data to downloaded data Assert.AreEqual(data, downloadStream.ToArray()); @@ -285,19 +398,25 @@ await track1Blob.UploadFromByteArrayAsync( public async Task Track1DownloadTrack2Blob() { var data = GetRandomBuffer(Constants.KB); // ensure we have enough room in original data - var mockKey = new MockKeyEncryptionKey(); + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetTrackOneIKeyResolver(GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object).Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyWrapAlgorithm = s_algorithmName })) { - var track2Blob = disposable.Container.GetBlobClient(GetNewBlobName()); + var track2Blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); // upload with track 2 - await track2Blob.UploadAsync(new MemoryStream(data)); + await track2Blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); // download with track 1 var creds = GetNewSharedKeyCredentials(); @@ -305,12 +424,18 @@ public async Task Track1DownloadTrack2Blob() track2Blob.Uri, new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); var downloadData = new byte[data.Length]; - await track1Blob.DownloadToByteArrayAsync(downloadData, 0, default, - new Microsoft.Azure.Storage.Blob.BlobRequestOptions() - { - EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, mockKey) - }, - default, default); + var track1RequestOptions = new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(default, mockKeyResolver) + }; + if (IsAsync) // can't instrument track 1 + { + await track1Blob.DownloadToByteArrayAsync(downloadData, 0, default, track1RequestOptions, default, s_cancellationToken); + } + else + { + track1Blob.DownloadToByteArray(downloadData, 0, default, track1RequestOptions, default); + } // compare original data to downloaded data Assert.AreEqual(data, downloadData); @@ -332,10 +457,10 @@ public async Task RoundtripWithKeyvaultProvider() { var blob = disposable.Container.GetBlobClient(GetNewBlobName()); - await blob.UploadAsync(new MemoryStream(data)); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); var downloadStream = new MemoryStream(); - await blob.DownloadToAsync(downloadStream); + await blob.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); Assert.AreEqual(data, downloadStream.ToArray()); } @@ -347,17 +472,16 @@ public async Task RoundtripWithKeyvaultProvider() public async Task CannotFindKeyAsync(bool resolverThrows) { var data = GetRandomBuffer(Constants.KB); - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyWrapAlgorithm = s_algorithmName })) { - var blob = disposable.Container.GetBlobClient(GetNewBlobName()); - await blob.UploadAsync(new MemoryStream(data)); + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); bool threw = false; try @@ -370,7 +494,7 @@ public async Task CannotFindKeyAsync(bool resolverThrows) KeyWrapAlgorithm = "test" }; var encryptedDataStream = new MemoryStream(); - await new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options).DownloadToAsync(encryptedDataStream); + await InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options)).DownloadToAsync(encryptedDataStream, cancellationToken: s_cancellationToken); } catch (Exception) { @@ -392,24 +516,25 @@ static async Task RoundTripDataHelper(BlobClient client, byte[] data) { using (var dataStream = new MemoryStream(data)) { - await client.UploadAsync(dataStream); + await client.UploadAsync(dataStream, cancellationToken: s_cancellationToken); } using (var downloadStream = new MemoryStream()) { - await client.DownloadToAsync(downloadStream); + await client.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); return downloadStream.ToArray(); } } var data = GetRandomBuffer(10 * Constants.MB); - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestContainerEncryptionAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = ThrowawayAlgorithmName + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { var downloadTasks = new List>(); diff --git a/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs b/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs deleted file mode 100644 index 94bee871d2bef..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/tests/MockKeyEncryptionKey.cs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Collections; -using System.Security.Cryptography; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core.Cryptography; -using Microsoft.Azure.KeyVault.Core; - -namespace Azure.Storage.Tests.Shared -{ - /// - /// Mock for a key encryption key. Not meant for production use. - /// - internal class MockKeyEncryptionKey : IKeyEncryptionKey, IKeyEncryptionKeyResolver, - IKey, IKeyResolver // for track 1 compatibility tests - { - public ReadOnlyMemory KeyEncryptionKey { get; } - - public string KeyId { get; } - - #region Counters - public int WrappedSync { get; private set; } - public int WrappedAsync { get; private set; } - public int UnwrappedSync { get; private set; } - public int UnwrappedAsync { get; private set; } - public int ResolvedSync { get; private set; } - public int ResolvedAsync { get; private set; } - - public void ResetCounters() - { - WrappedSync = 0; - WrappedAsync = 0; - UnwrappedSync = 0; - UnwrappedAsync = 0; - ResolvedSync = 0; - ResolvedAsync = 0; - } - #endregion - - /// - /// Generates a key encryption key with the given properties. - /// - public MockKeyEncryptionKey(int keySizeBits = 256, string keyId = default) - { - KeyId = keyId ?? Guid.NewGuid().ToString(); - using (var random = new RNGCryptoServiceProvider()) - { - var bytes = new byte[keySizeBits >> 3]; - random.GetBytes(bytes); - KeyEncryptionKey = bytes; - } - } - - public byte[] UnwrapKey(string algorithm, ReadOnlyMemory encryptedKey, CancellationToken cancellationToken = default) - { - UnwrappedSync++; - return Xor(encryptedKey.ToArray(), KeyEncryptionKey.ToArray()); - } - - public Task UnwrapKeyAsync(string algorithm, ReadOnlyMemory encryptedKey, CancellationToken cancellationToken = default) - { - UnwrappedAsync++; - return Task.FromResult(Xor(encryptedKey.ToArray(), KeyEncryptionKey.ToArray())); - } - - public byte[] WrapKey(string algorithm, ReadOnlyMemory key, CancellationToken cancellationToken = default) - { - WrappedSync++; - return Xor(key.ToArray(), KeyEncryptionKey.ToArray()); - } - - public Task WrapKeyAsync(string algorithm, ReadOnlyMemory key, CancellationToken cancellationToken = default) - { - WrappedAsync++; - return Task.FromResult(Xor(key.ToArray(), KeyEncryptionKey.ToArray())); - } - - private static byte[] Xor(byte[] a, byte[] b) - { - if (a.Length != b.Length) - { - throw new ArgumentException("Keys must be the same length for this mock implementation."); - } - - var aBits = new BitArray(a); - var bBits = new BitArray(b); - - var result = new byte[a.Length]; - aBits.Xor(bBits).CopyTo(result, 0); - - return result; - } - - public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationToken = default) - { - if (keyId != this.KeyId.ToString()) - { - throw new ArgumentException("Mock key resolver cannot find this keyId."); - } - - ResolvedSync++; - return this; - } - - public Task ResolveAsync(string keyId, CancellationToken cancellationToken = default) - { - if (keyId != this.KeyId.ToString()) - { - throw new ArgumentException("Mock key resolver cannot find this keyId."); - } - - ResolvedAsync++; - return Task.FromResult((IKeyEncryptionKey)this); - } - - #region Track 1 Impl - - string IKey.DefaultEncryptionAlgorithm => throw new NotImplementedException(); - - string IKey.DefaultKeyWrapAlgorithm => throw new NotImplementedException(); - - string IKey.DefaultSignatureAlgorithm => throw new NotImplementedException(); - - string IKey.Kid => KeyId; - - Task IKey.DecryptAsync(byte[] ciphertext, byte[] iv, byte[] authenticationData, byte[] authenticationTag, string algorithm, CancellationToken token) - { - throw new NotImplementedException(); - } - - Task> IKey.EncryptAsync(byte[] plaintext, byte[] iv, byte[] authenticationData, string algorithm, CancellationToken token) - { - throw new NotImplementedException(); - } - - async Task> IKey.WrapKeyAsync(byte[] key, string algorithm, CancellationToken token) - => new Tuple((await WrapKeyAsync(algorithm, key, token)), null); - - async Task IKey.UnwrapKeyAsync(byte[] encryptedKey, string algorithm, CancellationToken token) - => await UnwrapKeyAsync(algorithm, encryptedKey, token); - - Task> IKey.SignAsync(byte[] digest, string algorithm, CancellationToken token) - { - throw new NotImplementedException(); - } - - Task IKey.VerifyAsync(byte[] digest, byte[] signature, string algorithm, CancellationToken token) - { - throw new NotImplementedException(); - } - - void IDisposable.Dispose() - { - // no-op - } - - async Task IKeyResolver.ResolveKeyAsync(string kid, CancellationToken token) - => (MockKeyEncryptionKey)await ResolveAsync(kid, token); // we know we returned `this`; - #endregion - } -} diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index 6e53eec22cd34..ed381d3506e36 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -1,10 +1,5 @@ namespace Azure.Storage { - public partial class ClientSideEncryptionKeyNotFoundException : System.Exception - { - public ClientSideEncryptionKeyNotFoundException(string keyId) { } - public string KeyId { get { throw null; } } - } public partial class ClientSideEncryptionOptions { public ClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) { } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs index 5ae4c9d2dd806..6535e05065eb8 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -153,13 +153,15 @@ private static async Task> GetContentEncryptionKeyAsync( return async ? await key.UnwrapKeyAsync( encryptionData.WrappedContentKey.Algorithm, - encryptionData.WrappedContentKey.EncryptedKey).ConfigureAwait(false) + encryptionData.WrappedContentKey.EncryptedKey, + cancellationToken).ConfigureAwait(false) : key.UnwrapKey( encryptionData.WrappedContentKey.Algorithm, - encryptionData.WrappedContentKey.EncryptedKey); + encryptionData.WrappedContentKey.EncryptedKey, + cancellationToken); } -#pragma warning disable CS1587 // XML comment is not placed on a valid language element + /// /// Wraps a stream of ciphertext to stream plaintext. /// @@ -169,9 +171,12 @@ private static async Task> GetContentEncryptionKeyAsync( /// /// /// - private static Stream WrapStream(Stream contentStream, byte[] contentEncryptionKey, -#pragma warning restore CS1587 // XML comment is not placed on a valid language element - EncryptionData encryptionData, byte[] iv, bool noPadding) + private static Stream WrapStream( + Stream contentStream, + byte[] contentEncryptionKey, + EncryptionData encryptionData, + byte[] iv, + bool noPadding) { if (encryptionData.EncryptionAgent.EncryptionAlgorithm == ClientSideEncryptionAlgorithm.AesCbc256) { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs deleted file mode 100644 index e4258c647338b..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptedBlobRange.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using Azure.Storage.Cryptography; - -namespace Azure.Storage.Blobs.Specialized.Models -{ - /// - /// This is a representation of a range of bytes on an encrypted blob, which may be expanded from the requested - /// range to include extra data needed for decryption. It contains the original range as well as the calculated - /// expanded range. - /// - internal struct EncryptedBlobRange - { - /// - /// The original blob range requested by the user. - /// - public HttpRange OriginalRange { get; } - - /// - /// The blob range to actually request from the service that will allow - /// decryption of the original range. - /// - public HttpRange AdjustedRange { get; } - - public EncryptedBlobRange(HttpRange originalRange) - { - OriginalRange = originalRange; - - int offsetAdjustment = 0; - long? adjustedDownloadCount = originalRange.Length; - - // Calculate offsetAdjustment. - if (OriginalRange.Offset != 0) - { - // Align with encryption block boundary. - int diff; - if ((diff = (int)(OriginalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) - { - offsetAdjustment += diff; - if (adjustedDownloadCount != default) - { - adjustedDownloadCount += diff; - } - } - - // Account for IV. - if (OriginalRange.Offset >= EncryptionConstants.EncryptionBlockSize) - { - offsetAdjustment += EncryptionConstants.EncryptionBlockSize; - // Increment adjustedDownloadCount if necessary. - if (adjustedDownloadCount != default) - { - adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; - } - } - } - - // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible - // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. - if (adjustedDownloadCount != null) - { - adjustedDownloadCount += ( - EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount - % EncryptionConstants.EncryptionBlockSize) - ) % EncryptionConstants.EncryptionBlockSize; - } - - AdjustedRange = new HttpRange(OriginalRange.Offset - offsetAdjustment, adjustedDownloadCount); - } - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs index 9239e0510b9e7..ce84702e548e0 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs @@ -4,7 +4,7 @@ namespace Azure.Storage.Cryptography.Models { /// - /// Represents the envelope key details stored on the service. + /// Represents the envelope key details JSON schema stored on the service. /// In the envelope technique, a securely generated content encryption key (CEK) is generated /// for every encryption operation. It is then encrypted (wrapped) with the user-provided key /// encryption key (KEK), using a key-wrap algorithm. The wrapped CEK is stored with the diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index 2ef7af1d77bbc..b6b0797ab3687 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -317,18 +317,22 @@ internal UpdateReceipt() { } } namespace Azure.Storage.Queues.Specialized { - public partial class AdvancedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions + public partial class ExtendedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions { - public AdvancedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } + public ExtendedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } - public Azure.Storage.Queues.Specialized.IMissingClientSideEncryptionKeyListener OnMissingClientSideEncryptionKey { get { throw null; } set { } } + public Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener OnClientSideDecryptionFailure { get { throw null; } set { } } } - public partial interface IMissingClientSideEncryptionKeyListener + public partial interface IClientSideDecryptionFailureListener { - void OnMissingKey(Azure.Storage.Queues.Models.PeekedMessage message); - void OnMissingKey(Azure.Storage.Queues.Models.QueueMessage message); - System.Threading.Tasks.Task OnMissingKeyAsync(Azure.Storage.Queues.Models.PeekedMessage message); - System.Threading.Tasks.Task OnMissingKeyAsync(Azure.Storage.Queues.Models.QueueMessage message); + void OnFailure(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); + void OnFailure(Azure.Storage.Queues.Models.QueueMessage message, System.Exception exception); + System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); + System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.QueueMessage message, System.Exception exception); + } + public static partial class QueueClientExtensions + { + public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions, Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener listener = null) { throw null; } } } namespace Azure.Storage.Sas diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs similarity index 96% rename from sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs rename to sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs index e91707ada33ec..5c9ff5ac46512 100644 --- a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs @@ -12,7 +12,7 @@ namespace Azure.Storage.Queues.Specialized /// Storage. /// #pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. - public class AdvancedQueueClientOptions : QueueClientOptions + public class ExtendedQueueClientOptions : QueueClientOptions #pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion { /// @@ -23,7 +23,7 @@ public class AdvancedQueueClientOptions : QueueClientOptions /// The of the service API used when /// making requests. /// - public AdvancedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) + public ExtendedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) { } diff --git a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs similarity index 95% rename from sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs rename to sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs index 374b53420bc50..725dcd463b150 100644 --- a/sdk/storage/Azure.Storage.Queues/src/AdvancedQueueClientExtensions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs @@ -6,7 +6,7 @@ namespace Azure.Storage.Queues.Specialized /// /// Provides advanced extensions for queue service clients. /// - public static class AdvancedQueueClientExtensions + public static class QueueClientExtensions { /// /// Creates a new instance of the class, maintaining all the same diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index 7315891066f3b..ac8997c97c3e8 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -18,7 +18,6 @@ - \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index a7afc58355e93..1879817bb254b 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Security.Cryptography; using System.Text; +using System.Threading; using System.Threading.Tasks; using Azure.Core.Cryptography; using Azure.Core.TestFramework; @@ -14,16 +15,20 @@ using Azure.Storage.Blobs.Tests; using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Models; -using Azure.Storage.Queues.Specialized; using Azure.Storage.Queues.Specialized.Models; using Azure.Storage.Queues.Tests; -using Azure.Storage.Tests.Shared; +using Moq; using NUnit.Framework; +using static Moq.It; namespace Azure.Storage.Queues.Test { public class ClientSideEncryptionTests : QueueTestBase { + + private const string s_algorithmName = "some algorithm name"; + private static readonly CancellationToken s_cancellationToken = new CancellationTokenSource().Token; + private readonly string SampleUTF8String = Encoding.UTF8.GetString( new byte[] { 0xe1, 0x9a, 0xa0, 0xe1, 0x9b, 0x87, 0xe1, 0x9a, 0xbb, 0x0a }); // valid UTF-8 bytes @@ -94,6 +99,102 @@ public string GetRandomMessage(int size) return Encoding.ASCII.GetString(buf); } + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + if (IsAsync) + { + keyMock.Setup(k => k.WrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Task.FromResult(Xor(userKeyBytes, key.ToArray()))); + keyMock.Setup(k => k.UnwrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey.ToArray()))); + } + else + { + keyMock.Setup(k => k.WrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, wrappedKey.ToArray())); + } + + return keyMock; + } + + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + resolverMock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + } + else + { + resolverMock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? iKey : throw new Exception("Mock resolver couldn't resolve key id.")); + } + + return resolverMock; + } + + private Mock GetTrackOneIKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.Kid).Returns(keyId); + keyMock.SetupGet(k => k.DefaultKeyWrapAlgorithm).Returns(s_algorithmName); + // track one had async-only key wrapping + keyMock.Setup(k => k.WrapKeyAsync(IsNotNull(), IsAny(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + // track 1 doesn't pass in the algorithm name, it lets the implementation return the default algorithm it chose + .Returns((key, algorithm, cancellationToken) => Task.FromResult(Tuple.Create(Xor(userKeyBytes, key), s_algorithmName))); + keyMock.Setup(k => k.UnwrapKeyAsync(IsNotNull(), s_algorithmName, IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((wrappedKey, algorithm, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey))); + + return keyMock; + } + + private Mock GetTrackOneIKeyResolver(Microsoft.Azure.KeyVault.Core.IKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + resolverMock.Setup(r => r.ResolveKeyAsync(IsNotNull(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((keyId, cancellationToken) => iKey?.Kid == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + + return resolverMock; + } + + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } #endregion [TestCase(16, false)] // a single cipher block @@ -107,21 +208,20 @@ public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) var message = usePrebuiltMessage ? GetRandomMessage(messageSize) : SampleUTF8String; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyWrapAlgorithm = s_algorithmName })) { var queue = disposable.Queue; // upload with encryption - await queue.SendMessageAsync(message); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); // download without decrypting - var receivedMessages = (await new QueueClient(queue.Uri, GetNewSharedKeyCredentials()).ReceiveMessagesAsync()).Value; + var receivedMessages = (await InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials())).ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; Assert.AreEqual(1, receivedMessages.Length); var encryptedMessage = receivedMessages[0].MessageText; // json of message and metadata var parsedEncryptedMessage = EncryptedMessageSerializer.Deserialize(encryptedMessage); @@ -129,10 +229,12 @@ public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) // encrypt original data manually for comparison EncryptionData encryptionMetadata = parsedEncryptedMessage.EncryptionData; Assert.NotNull(encryptionMetadata, "Never encrypted data."); + var explicitlyUnwrappedKey = IsAsync + ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) + : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); string expectedEncryptedMessage = LocalManualEncryption( message, - (await mockKey.UnwrapKeyAsync(null, encryptionMetadata.WrappedContentKey.EncryptedKey) - .ConfigureAwait(false)).ToArray(), + explicitlyUnwrappedKey, encryptionMetadata.ContentEncryptionIV); // compare data @@ -151,21 +253,22 @@ public async Task RoundtripAsync(int messageSize, bool usePrebuiltMessage) var message = usePrebuiltMessage ? GetRandomMessage(messageSize) : SampleUTF8String; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { var queue = disposable.Queue; // upload with encryption - await queue.SendMessageAsync(message); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); // download with decryption - var receivedMessages = (await queue.ReceiveMessagesAsync()).Value; + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; Assert.AreEqual(1, receivedMessages.Length); var downloadedMessage = receivedMessages[0].MessageText; @@ -183,12 +286,18 @@ public async Task Track2DownloadTrack1Blob(int messageSize, bool usePrebuiltMess var message = usePrebuiltMessage ? GetRandomMessage(messageSize) : SampleUTF8String; - var mockKey = new MockKeyEncryptionKey(); + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object).Object; await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { var track2Queue = disposable.Queue; @@ -198,18 +307,21 @@ public async Task Track2DownloadTrack1Blob(int messageSize, bool usePrebuiltMess var track1Queue = new Microsoft.Azure.Storage.Queue.CloudQueue( track2Queue.Uri, new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); - await track1Queue.AddMessageAsync( - new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), - null, - null, - new Microsoft.Azure.Storage.Queue.QueueRequestOptions() - { - EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, mockKey) - }, - null); + var track1RequestOptions = new Microsoft.Azure.Storage.Queue.QueueRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, default) + }; + if (IsAsync) + { + await track1Queue.AddMessageAsync(new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), null, null, track1RequestOptions, null, s_cancellationToken); + } + else + { + track1Queue.AddMessage(new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), null, null, track1RequestOptions, null); + } // download with track 2 - var receivedMessages = (await track2Queue.ReceiveMessagesAsync()).Value; + var receivedMessages = (await track2Queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; Assert.AreEqual(1, receivedMessages.Length); var downloadedMessage = receivedMessages[0].MessageText; @@ -227,18 +339,24 @@ public async Task Track1DownloadTrack2Blob(int messageSize, bool usePrebuiltMess var message = usePrebuiltMessage ? GetRandomMessage(messageSize) : SampleUTF8String; - var mockKey = new MockKeyEncryptionKey(); + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetTrackOneIKeyResolver(GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object).Object; await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyWrapAlgorithm = s_algorithmName })) { var track2Queue = disposable.Queue; // upload with track 2 - await track2Queue.SendMessageAsync(message); + await track2Queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); // download with track 1 var creds = GetNewSharedKeyCredentials(); @@ -249,7 +367,7 @@ public async Task Track1DownloadTrack2Blob(int messageSize, bool usePrebuiltMess null, new Microsoft.Azure.Storage.Queue.QueueRequestOptions() { - EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, mockKey) + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(default, mockKeyResolver) }, null); @@ -272,9 +390,9 @@ public async Task RoundtripWithKeyvaultProvider() { var queue = disposable.Queue; - await queue.SendMessageAsync(message); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); - var receivedMessages = (await queue.ReceiveMessagesAsync()).Value; + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; Assert.AreEqual(1, receivedMessages.Length); var downloadedMessage = receivedMessages[0].MessageText; @@ -287,22 +405,23 @@ public async Task RoundtripWithKeyvaultProvider() public async Task ReadPlaintextMessage() { var message = "any old message"; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { var encryptedQueueClient = disposable.Queue; var plainQueueClient = new QueueClient(encryptedQueueClient.Uri, GetNewSharedKeyCredentials()); // upload with encryption - await plainQueueClient.SendMessageAsync(message); + await plainQueueClient.SendMessageAsync(message, cancellationToken: s_cancellationToken); // download with decryption - var receivedMessages = (await encryptedQueueClient.ReceiveMessagesAsync()).Value; + var receivedMessages = (await encryptedQueueClient.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; Assert.AreEqual(1, receivedMessages.Length); var downloadedMessage = receivedMessages[0].MessageText; @@ -316,20 +435,28 @@ public async Task ReadPlaintextMessage() public async Task OnlyOneKeyWrapCall() { var message = "any old message"; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey(); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey.Object); await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyEncryptionKey = mockKey.Object, + KeyResolver = mockKeyResolver.Object, + KeyWrapAlgorithm = s_algorithmName })) { var queue = disposable.Queue; - await queue.SendMessageAsync(message).ConfigureAwait(false); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); + + var wrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("WrapKey"); + var wrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("WrapKeyAsync"); - Assert.AreEqual(1, IsAsync ? mockKey.WrappedAsync : mockKey.WrappedSync); - Assert.AreEqual(0, IsAsync ? mockKey.WrappedSync : mockKey.WrappedAsync); + Assert.AreEqual(1, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == wrapAsyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == wrapSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == wrapSyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == wrapAsyncMethod)); } } @@ -338,37 +465,50 @@ public async Task OnlyOneKeyWrapCall() public async Task OnlyOneKeyResolveAndUnwrapCall() { var message = "any old message"; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey(); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey.Object); await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyEncryptionKey = mockKey.Object, + KeyResolver = mockKeyResolver.Object, + KeyWrapAlgorithm = s_algorithmName })) { var queue = disposable.Queue; - await queue.SendMessageAsync(message).ConfigureAwait(false); - mockKey.ResetCounters(); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); // replace with client that has only key resolver var options = GetOptions(); options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = default, // we want the key resolver to trigger; no cached key - KeyResolver = mockKey + KeyResolver = mockKeyResolver.Object }; queue = InstrumentClient(new QueueClient( queue.Uri, GetNewSharedKeyCredentials(), options)); - await queue.ReceiveMessagesAsync(); - - Assert.AreEqual(1, IsAsync ? mockKey.ResolvedAsync : mockKey.ResolvedSync); - Assert.AreEqual(0, IsAsync ? mockKey.ResolvedSync : mockKey.ResolvedAsync); - - Assert.AreEqual(1, IsAsync ? mockKey.UnwrappedAsync : mockKey.UnwrappedSync); - Assert.AreEqual(0, IsAsync ? mockKey.UnwrappedSync : mockKey.UnwrappedAsync); + await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken); + + var resolveSyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("Resolve"); + var resolveAsyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("ResolveAsync"); + var unwrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); + var unwrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); + + Assert.AreEqual(1, IsAsync + ? mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveAsyncMethod) + : mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveSyncMethod) + : mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveAsyncMethod)); + + Assert.AreEqual(1, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == unwrapAsyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == unwrapSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == unwrapSyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == unwrapAsyncMethod)); } } @@ -387,19 +527,20 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) const int numMessages = 5; var message = "any old message"; - var mockKey = new MockKeyEncryptionKey(); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; await using (var disposable = await GetTestEncryptedQueueAsync( new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = mockKey, - KeyResolver = mockKey, - KeyWrapAlgorithm = "mock" + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName })) { var queue = disposable.Queue; foreach (var _ in Enumerable.Range(0, numMessages)) { - await queue.SendMessageAsync(message).ConfigureAwait(false); + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); } bool threw = false; @@ -415,7 +556,7 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) KeyWrapAlgorithm = "test" }; options._onClientSideDecryptionFailure = listener; - result = await new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options).ReceiveMessagesAsync(numMessages); + result = await InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options)).ReceiveMessagesAsync(numMessages, cancellationToken: s_cancellationToken); } catch (Exception) { diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs index d9d4e5fabcc5d..99aa74b18bbee 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -5,15 +5,17 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; -using System.Threading.Tasks; +using System.Threading; +using Azure.Core.Cryptography; using Azure.Core.Pipeline; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Specialized.Models; -using Azure.Storage.Tests.Shared; -using Newtonsoft.Json; +using Moq; using NUnit.Framework; +using static Moq.It; namespace Azure.Storage.Queues.Test { @@ -22,12 +24,48 @@ public class EncryptedMessageSerializerTests // doesn't inherit our test base be private const string TestMessage = "This can technically be a valid encrypted message."; private const string KeyWrapAlgorithm = "my_key_wrap_algorithm"; + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + keyMock.Setup(k => k.WrapKey(KeyWrapAlgorithm, IsNotNull>(), IsAny())) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(KeyWrapAlgorithm, IsNotNull>(), IsAny())) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, userKeyBytes.ToArray())); + + return keyMock; + } + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + [Test] public void SerializeEncryptedMessage() { var result = ClientSideEncryptor.BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - new MockKeyEncryptionKey(), + GetIKeyEncryptionKey().Object, KeyWrapAlgorithm, async: false, default).EnsureCompleted(); @@ -46,11 +84,11 @@ public void SerializeEncryptedMessage() public void DeserializeEncryptedMessage() { var result = ClientSideEncryptor.BufferedEncryptInternal( - new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - new MockKeyEncryptionKey(), - KeyWrapAlgorithm, - async: false, - default).EnsureCompleted(); + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm, + async: false, + default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() { EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), @@ -67,11 +105,11 @@ public void DeserializeEncryptedMessage() public void TryDeserializeEncryptedMessage() { var result = ClientSideEncryptor.BufferedEncryptInternal( - new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - new MockKeyEncryptionKey(), - KeyWrapAlgorithm, - async: false, - default).EnsureCompleted(); + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm, + async: false, + default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() { EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), From 69be140add118f5beefac30a09c7f02c17af9129 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Mon, 1 Jun 2020 17:46:17 -0700 Subject: [PATCH 10/21] Refactors and bug fixes --- ...crosoft.Extensions.Azure.netstandard2.0.cs | 33 --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 163 +-------------- .../Azure.Storage.Blobs/src/BlobClient.cs | 57 +----- .../src/BlobClientSideDecryptor.cs | 188 ++++++++++++++++++ .../src/BlobClientSideEncryptor.cs | 78 ++++++++ .../src/Models/ContentRange.cs | 2 +- .../AlwaysFailsKeyEncryptionKeyResolver.cs | 38 ---- .../tests/ClientSideEncryptionTests.cs | 134 ++++++++++++- .../Models/EncryptionDataSerializer.cs | 31 +++ .../Azure.Storage.Queues/src/QueueClient.cs | 112 ++--------- .../src/QueueClientExtensions.cs | 19 +- .../src/QueueClientSideDecryptor.cs | 107 ++++++++++ .../src/QueueClientSideEncryptor.cs | 44 ++++ .../tests/Azure.Storage.Queues.Tests.csproj | 1 - .../tests/ClientSideEncryptionTests.cs | 188 +++++++++++++++--- ...kMissingClientSideEncryptionKeyListener.cs | 42 ---- 16 files changed, 773 insertions(+), 464 deletions(-) delete mode 100644 sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs create mode 100644 sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs delete mode 100644 sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs create mode 100644 sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs create mode 100644 sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs delete mode 100644 sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs diff --git a/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs b/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs deleted file mode 100644 index a9be4f097bab5..0000000000000 --- a/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace Microsoft.Extensions.Azure -{ - public static partial class AzureClientBuilderExtensions - { - public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, Microsoft.Extensions.Configuration.IConfiguration configuration) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Action configureOptions) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Action configureOptions) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder WithCredential(this Azure.Core.Extensions.IAzureClientBuilder builder, Azure.Core.TokenCredential credential) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder WithCredential(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Func credentialFactory) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder WithName(this Azure.Core.Extensions.IAzureClientBuilder builder, string name) where TOptions : class { throw null; } - public static Azure.Core.Extensions.IAzureClientBuilder WithVersion(this Azure.Core.Extensions.IAzureClientBuilder builder, TVersion version) where TOptions : class { throw null; } - } - public sealed partial class AzureClientFactoryBuilder : Azure.Core.Extensions.IAzureClientFactoryBuilder, Azure.Core.Extensions.IAzureClientFactoryBuilderWithConfiguration, Azure.Core.Extensions.IAzureClientFactoryBuilderWithCredential - { - internal AzureClientFactoryBuilder() { } - Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilder.RegisterClientFactory(System.Func clientFactory) { throw null; } - Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilderWithConfiguration.RegisterClientFactory(Microsoft.Extensions.Configuration.IConfiguration configuration) { throw null; } - Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilderWithCredential.RegisterClientFactory(System.Func clientFactory, bool requiresCredential) { throw null; } - public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(Microsoft.Extensions.Configuration.IConfiguration configuration) { throw null; } - public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(System.Action configureOptions) { throw null; } - public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(System.Action configureOptions) { throw null; } - public Microsoft.Extensions.Azure.AzureClientFactoryBuilder UseCredential(Azure.Core.TokenCredential tokenCredential) { throw null; } - public Microsoft.Extensions.Azure.AzureClientFactoryBuilder UseCredential(System.Func tokenCredentialFactory) { throw null; } - } - public static partial class AzureClientServiceCollectionExtensions - { - public static void AddAzureClients(this Microsoft.Extensions.DependencyInjection.IServiceCollection collection, System.Action configureClients) { } - } - public partial interface IAzureClientFactory - { - TClient CreateClient(string name); - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 6a1bca0c159cc..62265f8eff28c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -9,8 +9,6 @@ using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; using Azure.Storage.Cryptography; -using Azure.Storage.Cryptography.Models; -using Azure.Storage.Shared; using Metadata = System.Collections.Generic.IDictionary; #pragma warning disable SA1402 // File may only contain a single type @@ -665,7 +663,7 @@ private async Task> DownloadInternal( { if (UsingClientSideEncryption) { - range = GetEncryptedBlobRange(range); + range = BlobClientSideDecryptor.GetEncryptedBlobRange(range); } // Start downloading the blob @@ -715,7 +713,8 @@ private async Task> DownloadInternal( // we already return a nonseekable stream; returning a crypto stream is fine if (UsingClientSideEncryption) { - stream = await ClientSideDecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); + stream = await new BlobClientSideDecryptor(ClientSideEncryption) + .ClientSideDecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); } response.Value.Content = stream; @@ -2984,162 +2983,6 @@ private async Task SetAccessTierInternal( } } #endregion SetAccessTier - - private async Task ClientSideDecryptInternal( - Stream content, - Metadata metadata, - HttpRange originalRange, - string receivedContentRange, - bool async, - CancellationToken cancellationToken) - { - ContentRange? contentRange = string.IsNullOrWhiteSpace(receivedContentRange) - ? default - : ContentRange.Parse(receivedContentRange); - - EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata); - if (encryptionData == default) - { - return content; // TODO readjust range - } - - bool ivInStream = originalRange.Offset >= 16; - - // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. - var plaintext = await ClientSideDecryptor.DecryptInternal( - content, - encryptionData, - ivInStream, - ClientSideEncryption.KeyResolver, - ClientSideEncryption.KeyEncryptionKey, - CanIgnorePadding(contentRange), - async, - cancellationToken).ConfigureAwait(false); - - // retrim start of stream to original requested location - // keeping in mind whether we already pulled the IV out of the stream as well - int gap = (int)(originalRange.Offset - (contentRange?.Start ?? 0)) - - (ivInStream ? EncryptionConstants.EncryptionBlockSize : 0); - if (gap > 0) - { - // throw away initial bytes we want to trim off; stream cannot seek into future - if (async) - { - await plaintext.ReadAsync(new byte[gap], 0, gap, cancellationToken).ConfigureAwait(false); - } - else - { - plaintext.Read(new byte[gap], 0, gap); - } - } - - if (originalRange.Length.HasValue) - { - plaintext = new WindowStream(plaintext, originalRange.Length.Value); - } - - return plaintext; - } - - internal static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata metadata) - { - if (metadata == default) - { - return default; - } - if (!metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string encryptedDataString)) - { - return default; - } - - EncryptionData encryptionData = EncryptionDataSerializer.Deserialize(encryptedDataString); - - _ = encryptionData.ContentEncryptionIV ?? throw EncryptionErrors.MissingEncryptionMetadata( - nameof(EncryptionData.ContentEncryptionIV)); - _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw EncryptionErrors.MissingEncryptionMetadata( - nameof(EncryptionData.WrappedContentKey.EncryptedKey)); - - return encryptionData; - } - - /// - /// Gets whether to ignore padding options for decryption. - /// - /// Downloaded content range. - /// True if we should ignore padding. - /// - /// If the last cipher block of the blob was returned, we need the padding. Otherwise, we can ignore it. - /// - private static bool CanIgnorePadding(ContentRange? contentRange) - { - // if Content-Range not present, we requested the whole blob - if (!contentRange.HasValue) - { - return false; - } - - // if range is wildcard, we requested the whole blob - if (!contentRange.Value.End.HasValue) - { - return false; - } - - // blob storage will always return ContentRange.Size - // we don't have to worry about the impossible decision of what to do if it doesn't - - // did we request the last block? - // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block - if (contentRange.Value.Size - contentRange.Value.End == 1) - { - return false; - } - - return true; - } - - internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) - { - int offsetAdjustment = 0; - long? adjustedDownloadCount = originalRange.Length; - - // Calculate offsetAdjustment. - if (originalRange.Offset != 0) - { - // Align with encryption block boundary. - int diff; - if ((diff = (int)(originalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) - { - offsetAdjustment += diff; - if (adjustedDownloadCount != default) - { - adjustedDownloadCount += diff; - } - } - - // Account for IV. - if (originalRange.Offset >= EncryptionConstants.EncryptionBlockSize) - { - offsetAdjustment += EncryptionConstants.EncryptionBlockSize; - // Increment adjustedDownloadCount if necessary. - if (adjustedDownloadCount != default) - { - adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; - } - } - } - - // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible - // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. - if (adjustedDownloadCount != null) - { - adjustedDownloadCount += ( - EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount - % EncryptionConstants.EncryptionBlockSize) - ) % EncryptionConstants.EncryptionBlockSize; - } - - return new HttpRange(originalRange.Offset - offsetAdjustment, adjustedDownloadCount); - } } /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index d051b9a6ecdc4..36fce14af91fd 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -11,9 +10,6 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Metadata = System.Collections.Generic.IDictionary; -using Azure.Storage.Cryptography.Models; -using System.Collections.Generic; -using Azure.Storage.Cryptography; namespace Azure.Storage.Blobs { @@ -954,7 +950,8 @@ internal async Task> StagedUploadAsync( if (UsingClientSideEncryption) { // content is now unseekable, so PartitionedUploader will be forced to do a buffered multipart upload - (content, metadata) = await ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false); + (content, metadata) = await new BlobClientSideEncryptor(ClientSideEncryption) + .ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false); } var client = new BlockBlobClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); @@ -1067,55 +1064,5 @@ internal async Task> StagedUploadAsync( } } #endregion Upload - - /// - /// Applies client-side encryption to the data for upload. - /// - /// - /// Content to encrypt. - /// - /// - /// Metadata to add encryption metadata to. - /// - /// - /// Whether to perform this operation asynchronously. - /// - /// - /// Cancellation token. - /// - /// Transformed content stream and metadata. - private async Task<(Stream, Metadata)> ClientSideEncryptInternal( - Stream content, - Metadata metadata, - bool async, - CancellationToken cancellationToken) - { - if (ClientSideEncryption?.KeyEncryptionKey == default || ClientSideEncryption?.KeyWrapAlgorithm == default) - { - throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(ClientSideEncryption.KeyEncryptionKey), nameof(ClientSideEncryption.KeyWrapAlgorithm)); - } - - //long originalLength = content.Length; - - (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await ClientSideEncryptor.EncryptInternal( - content, - ClientSideEncryption.KeyEncryptionKey, - ClientSideEncryption.KeyWrapAlgorithm, - async, - cancellationToken).ConfigureAwait(false); - - //Stream seekableCiphertext = new RollingBufferStream( - // nonSeekableCiphertext, - // EncryptionConstants.DefaultRollingBufferSize, - // GetExpectedCryptoStreamLength(originalLength)); - - metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); - metadata.Add(EncryptionConstants.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); - - return (nonSeekableCiphertext, metadata); - } - - private static long GetExpectedCryptoStreamLength(long originalLength) - => originalLength + (EncryptionConstants.EncryptionBlockSize - originalLength % EncryptionConstants.EncryptionBlockSize); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs new file mode 100644 index 0000000000000..60e77aaa3b7e6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Shared; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Blobs +{ + internal class BlobClientSideDecryptor + { + private readonly IKeyEncryptionKeyResolver _resolver; + private readonly IKeyEncryptionKey _cachedIKey; + + public BlobClientSideDecryptor(ClientSideEncryptionOptions options) + { + _resolver = options.KeyResolver; + _cachedIKey = options.KeyEncryptionKey; + } + + public async Task ClientSideDecryptInternal( + Stream content, + Metadata metadata, + HttpRange originalRange, + string receivedContentRange, + bool async, + CancellationToken cancellationToken) + { + ContentRange? contentRange = string.IsNullOrWhiteSpace(receivedContentRange) + ? default + : ContentRange.Parse(receivedContentRange); + + EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata); + if (encryptionData == default) + { + return await TrimStreamInternal(content, originalRange, contentRange, pulledOutIv: false, async, cancellationToken).ConfigureAwait(false); + } + + bool ivInStream = originalRange.Offset >= EncryptionConstants.EncryptionBlockSize; + + // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. + var plaintext = await ClientSideDecryptor.DecryptInternal( + content, + encryptionData, + ivInStream, + _resolver, + _cachedIKey, + CanIgnorePadding(contentRange), + async, + cancellationToken).ConfigureAwait(false); + + return await TrimStreamInternal(plaintext, originalRange, contentRange, ivInStream, async, cancellationToken).ConfigureAwait(false); + } + + private static async Task TrimStreamInternal(Stream stream, HttpRange originalRange, ContentRange? receivedRange, bool pulledOutIv, bool async, CancellationToken cancellationToken) + { + // retrim start of stream to original requested location + // keeping in mind whether we already pulled the IV out of the stream as well + int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0)) + - (pulledOutIv ? EncryptionConstants.EncryptionBlockSize : 0); + if (gap > 0) + { + // throw away initial bytes we want to trim off; stream cannot seek into future + if (async) + { + await stream.ReadAsync(new byte[gap], 0, gap, cancellationToken).ConfigureAwait(false); + } + else + { + stream.Read(new byte[gap], 0, gap); + } + } + + if (originalRange.Length.HasValue) + { + stream = new WindowStream(stream, originalRange.Length.Value); + } + + return stream; + } + + private static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata metadata) + { + if (metadata == default) + { + return default; + } + if (!metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string encryptedDataString)) + { + return default; + } + + EncryptionData encryptionData = EncryptionDataSerializer.Deserialize(encryptedDataString); + + _ = encryptionData.ContentEncryptionIV ?? throw EncryptionErrors.MissingEncryptionMetadata( + nameof(EncryptionData.ContentEncryptionIV)); + _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw EncryptionErrors.MissingEncryptionMetadata( + nameof(EncryptionData.WrappedContentKey.EncryptedKey)); + + return encryptionData; + } + + /// + /// Gets whether to ignore padding options for decryption. + /// + /// Downloaded content range. + /// True if we should ignore padding. + /// + /// If the last cipher block of the blob was returned, we need the padding. Otherwise, we can ignore it. + /// + private static bool CanIgnorePadding(ContentRange? contentRange) + { + // if Content-Range not present, we requested the whole blob + if (!contentRange.HasValue) + { + return false; + } + + // if range is wildcard, we requested the whole blob + if (!contentRange.Value.End.HasValue) + { + return false; + } + + // blob storage will always return ContentRange.Size + // we don't have to worry about the impossible decision of what to do if it doesn't + + // did we request the last block? + // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block + if (contentRange.Value.Size - contentRange.Value.End == 1) + { + return false; + } + + return true; + } + + internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) + { + int offsetAdjustment = 0; + long? adjustedDownloadCount = originalRange.Length; + + // Calculate offsetAdjustment. + if (originalRange.Offset != 0) + { + // Align with encryption block boundary. + int diff; + if ((diff = (int)(originalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) + { + offsetAdjustment += diff; + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += diff; + } + } + + // Account for IV. + if (originalRange.Offset >= EncryptionConstants.EncryptionBlockSize) + { + offsetAdjustment += EncryptionConstants.EncryptionBlockSize; + // Increment adjustedDownloadCount if necessary. + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; + } + } + } + + // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible + // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. + if (adjustedDownloadCount != null) + { + adjustedDownloadCount += ( + EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount + % EncryptionConstants.EncryptionBlockSize) + ) % EncryptionConstants.EncryptionBlockSize; + } + + return new HttpRange(originalRange.Offset - offsetAdjustment, adjustedDownloadCount); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs new file mode 100644 index 0000000000000..f3c802f939645 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Blobs +{ + internal class BlobClientSideEncryptor + { + private readonly IKeyEncryptionKey _keyEncryptionKey; + private readonly string _keyWrapAlgorithm; + + public BlobClientSideEncryptor(ClientSideEncryptionOptions options) + { + _keyEncryptionKey = options.KeyEncryptionKey; + _keyWrapAlgorithm = options.KeyWrapAlgorithm; + } + + /// + /// Applies client-side encryption to the data for upload. + /// + /// + /// Content to encrypt. + /// + /// + /// Metadata to add encryption metadata to. + /// + /// + /// Whether to perform this operation asynchronously. + /// + /// + /// Cancellation token. + /// + /// Transformed content stream and metadata. + public async Task<(Stream, Metadata)> ClientSideEncryptInternal( + Stream content, + Metadata metadata, + bool async, + CancellationToken cancellationToken) + { + if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); + } + + //long originalLength = content.Length; + + (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await ClientSideEncryptor.EncryptInternal( + content, + _keyEncryptionKey, + _keyWrapAlgorithm, + async, + cancellationToken).ConfigureAwait(false); + + //Stream seekableCiphertext = new RollingBufferStream( + // nonSeekableCiphertext, + // EncryptionConstants.DefaultRollingBufferSize, + // GetExpectedCryptoStreamLength(originalLength)); + + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + metadata.Add(EncryptionConstants.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); + + return (nonSeekableCiphertext, metadata); + } + + private static long GetExpectedCryptoStreamLength(long originalLength) + => originalLength + (EncryptionConstants.EncryptionBlockSize - originalLength % EncryptionConstants.EncryptionBlockSize); + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs index da712dd19c871..fe5f6fb914c14 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs @@ -29,7 +29,7 @@ public RangeUnit(string value) } /// - /// AES-CBC using a 256 bit key. + /// Label for bytes as the measurement of content range. /// public static RangeUnit Bytes { get; } = new RangeUnit(BytesValue); diff --git a/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs b/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs deleted file mode 100644 index 0cb9666182d61..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/tests/AlwaysFailsKeyEncryptionKeyResolver.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Azure.Core.Cryptography; - -namespace Azure.Storage.Blobs.Tests -{ - internal class AlwaysFailsKeyEncryptionKeyResolver : IKeyEncryptionKeyResolver - { - /// - /// False means the resolver returns null. - /// True means the resolver throws. - /// - public bool ShouldThrow { get; set; } = false; - - public IKeyEncryptionKey Resolve(string keyId, CancellationToken cancellationToken = default) - { - if (ShouldThrow) - { - throw new Exception(); - } - return default; - } - - public Task ResolveAsync(string keyId, CancellationToken cancellationToken = default) - { - if (ShouldThrow) - { - throw new Exception(); - } - return Task.FromResult(default); - } - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 3fe25aeda8d6d..b3525ef844bf4 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.Linq; using System.Security.Cryptography; @@ -11,6 +12,8 @@ using Azure.Core.Cryptography; using Azure.Core.TestFramework; using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Azure.Storage.Blobs.Tests; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; @@ -31,10 +34,10 @@ public ClientSideEncryptionTests(bool async, BlobClientOptions.ServiceVersion se { } - - #region Utility - - private byte[] LocalManualEncryption(byte[] data, byte[] key, byte[] iv) + /// + /// Provides encryption functionality clone of client logic, letting us validate the client got it right end-to-end. + /// + private byte[] EncryptData(byte[] data, byte[] key, byte[] iv) { using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) using (var encryptor = aesProvider.CreateEncryptor()) @@ -103,6 +106,39 @@ private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = defau return keyMock; } + private Mock GetAlwaysFailsKeyResolver(bool throws) + { + var mock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + if (throws) + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns(Task.FromResult(null)); + } + } + else + { + if (throws) + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((IKeyEncryptionKey)null); + } + } + + return mock; + } + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) { var resolverMock = new Mock(MockBehavior.Strict); @@ -169,7 +205,38 @@ private static byte[] Xor(byte[] a, byte[] b) return result; } - #endregion + [Test] + [LiveOnly] + public void CanSwapKey() + { + var options1 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "foo" + }; + var options2 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "bar" + }; + + var client = new BlobClient(new Uri("http://someuri.com"), new ExtendedBlobClientOptions() + { + ClientSideEncryption = options1, + }); + + Assert.AreEqual(options1.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options1.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options1.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + + client = client.WithClientSideEncryptionOptions(options2); + + Assert.AreEqual(options2.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options2.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options2.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + } [TestCase(16)] // a single cipher block [TestCase(14)] // a single unalligned cipher block @@ -210,7 +277,7 @@ public async Task UploadAsync(long dataSize) var explicitlyUnwrappedKey = IsAsync // can't instrument this ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); - byte[] expectedEncryptedData = LocalManualEncryption( + byte[] expectedEncryptedData = EncryptData( data, explicitlyUnwrappedKey, encryptionMetadata.ContentEncryptionIV); @@ -484,18 +551,23 @@ public async Task CannotFindKeyAsync(bool resolverThrows) await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); bool threw = false; + var resolver = GetAlwaysFailsKeyResolver(resolverThrows); try { // download but can't find key var options = GetOptions(); options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { - KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ShouldThrow = resolverThrows }, + KeyResolver = resolver.Object, KeyWrapAlgorithm = "test" }; var encryptedDataStream = new MemoryStream(); await InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options)).DownloadToAsync(encryptedDataStream, cancellationToken: s_cancellationToken); } + catch (MockException e) + { + Assert.Fail(e.Message); + } catch (Exception) { threw = true; @@ -503,6 +575,50 @@ public async Task CannotFindKeyAsync(bool resolverThrows) finally { Assert.IsTrue(threw); + // we already asserted the correct method was called in `catch (MockException e)` + Assert.AreEqual(1, resolver.Invocations.Count); + } + } + } + + // using 5 to setup offsets to avoid any off-by-one confusion in debugging + [TestCase(0, null)] + [TestCase(0, 2 * EncryptionConstants.EncryptionBlockSize)] + [TestCase(0, 2 * EncryptionConstants.EncryptionBlockSize + 5)] + [TestCase(EncryptionConstants.EncryptionBlockSize, EncryptionConstants.EncryptionBlockSize)] + [TestCase(EncryptionConstants.EncryptionBlockSize, EncryptionConstants.EncryptionBlockSize + 5)] + [TestCase(EncryptionConstants.EncryptionBlockSize + 5, 2 * EncryptionConstants.EncryptionBlockSize)] + [LiveOnly] + public async Task AppropriateRangeDownloadOnPlaintext(int rangeOffset, int? rangeLength) + { + var data = GetRandomBuffer(rangeOffset + (rangeLength ?? Constants.KB) + EncryptionConstants.EncryptionBlockSize); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey().Object).Object; + await using (var disposable = await GetTestContainerAsync()) + { + // upload plaintext + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + await blob.UploadAsync(new MemoryStream(data)); + + // download plaintext range with encrypted client + var cryptoClient = InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), new ExtendedBlobClientOptions() + { + ClientSideEncryption = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver + } + })); + var desiredRange = new HttpRange(rangeOffset, rangeLength); + var response = await cryptoClient.DownloadAsync(desiredRange); + + // assert we recieved the data we requested + int expectedLength = rangeLength ?? data.Length - rangeOffset; + var memoryStream = new MemoryStream(); + await response.Value.Content.CopyToAsync(memoryStream); + var recievedData = memoryStream.ToArray(); + Assert.AreEqual(expectedLength, recievedData.Length); + for (int i = 0; i < recievedData.Length; i++) + { + Assert.AreEqual(data[i + rangeOffset], recievedData[i]); } } } @@ -512,7 +628,7 @@ public async Task CannotFindKeyAsync(bool resolverThrows) [Ignore("stress test")] public async Task StressAsync() { - static async Task RoundTripDataHelper(BlobClient client, byte[] data) + static async Task RoundTripData(BlobClient client, byte[] data) { using (var dataStream = new MemoryStream(data)) { @@ -542,7 +658,7 @@ static async Task RoundTripDataHelper(BlobClient client, byte[] data) { var blob = disposable.Container.GetBlobClient(GetNewBlobName()); - downloadTasks.Add(RoundTripDataHelper(blob, data)); + downloadTasks.Add(RoundTripData(blob, data)); } var downloads = await Task.WhenAll(downloadTasks); diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs index e9b8af4c1ceb1..7302bfa066d87 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -11,11 +11,22 @@ namespace Azure.Storage.Cryptography.Models internal static class EncryptionDataSerializer { #region Serialize + + /// + /// Serializes an EncryptionData instance into JSON. + /// + /// Data to serialize. + /// The JSON string. public static string Serialize(EncryptionData data) { return Encoding.UTF8.GetString(SerializeEncryptionData(data).ToArray()); } + /// + /// Serializes an EncryptionData instance into JSON. + /// + /// Data to serialize. + /// The JSON UTF8 bytes. public static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) { var writer = new Core.ArrayBufferWriter(); @@ -29,6 +40,11 @@ public static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) return writer.WrittenMemory; } + /// + /// Serializes an EncryptionData instance into JSON and writes it to the given JSON writer. + /// + /// The writer to write the serialization to. + /// Data to serialize. public static void WriteEncryptionData(Utf8JsonWriter json, EncryptionData data) { json.WriteString(nameof(data.EncryptionMode), data.EncryptionMode); @@ -71,12 +87,22 @@ private static void WriteDictionary(Utf8JsonWriter json, IDictionary + /// Deserializes an EncryptionData instance from JSON. + /// + /// The serialized data string. + /// The instance. public static EncryptionData Deserialize(string serializedData) { var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(serializedData)); return DeserializeEncryptionData(ref reader); } + /// + /// Reads an EncryptionData instance from a JSON reader. + /// + /// The reader to parse an EncryptionData isntance from. + /// The instance. public static EncryptionData DeserializeEncryptionData(ref Utf8JsonReader reader) { using JsonDocument json = JsonDocument.ParseValue(ref reader); @@ -84,6 +110,11 @@ public static EncryptionData DeserializeEncryptionData(ref Utf8JsonReader reader return ReadEncryptionData(root); } + /// + /// Reads an EncryptionData instance from a parsed JSON object. + /// + /// The JSON object model. + /// The instance. public static EncryptionData ReadEncryptionData(JsonElement root) { var data = new EncryptionData(); diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index 98d0ed60c54b6..c652df96e5c0c 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -89,7 +89,9 @@ public class QueueClient internal bool UsingClientSideEncryption => ClientSideEncryption != default; - private readonly IClientSideDecryptionFailureListener _missingClientSideEncryptionKeyListener; + private readonly IClientSideDecryptionFailureListener _onClientSideDecryptionFailure; + + internal virtual IClientSideDecryptionFailureListener OnClientSideDecryptionFailure => _onClientSideDecryptionFailure; /// /// QueueMaxMessagesPeek indicates the maximum number of messages @@ -198,7 +200,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; + _onClientSideDecryptionFailure = options._onClientSideDecryptionFailure; } /// @@ -291,7 +293,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; + _onClientSideDecryptionFailure = options._onClientSideDecryptionFailure; } /// @@ -333,7 +335,7 @@ internal QueueClient( _version = version; _clientDiagnostics = clientDiagnostics; _clientSideEncryption = encryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = listener; + _onClientSideDecryptionFailure = listener; } #endregion ctors @@ -1510,7 +1512,8 @@ private async Task> SendMessageInternal( try { messageText = UsingClientSideEncryption - ? await ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) + ? await new QueueClientSideEncryptor(ClientSideEncryption) + .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) : messageText; Response> messages = @@ -1707,7 +1710,8 @@ private async Task> ReceiveMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + await new QueueClientSideDecryptor(ClientSideEncryption, OnClientSideDecryptionFailure) + .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } else @@ -1825,7 +1829,8 @@ private async Task> PeekMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + await new QueueClientSideDecryptor(ClientSideEncryption, OnClientSideDecryptionFailure) + .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } else @@ -2106,98 +2111,5 @@ private async Task> UpdateMessageInternal( } } #endregion UpdateMessage - - private async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) - { - var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); - (byte[] ciphertext, EncryptionData encryptionData) = await ClientSideEncryptor.BufferedEncryptInternal( - new MemoryStream(bytesToEncrypt), - ClientSideEncryption.KeyEncryptionKey, - ClientSideEncryption.KeyWrapAlgorithm, - async, - cancellationToken).ConfigureAwait(false); - - return EncryptedMessageSerializer.Serialize(new EncryptedMessage - { - EncryptedMessageContents = Convert.ToBase64String(ciphertext), - EncryptionData = encryptionData - }); - } - - private async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) - { - var filteredMessages = new List(); - foreach (var message in messages) - { - try - { - message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); - filteredMessages.Add(message); - } - catch (Exception e) when (_missingClientSideEncryptionKeyListener != default) - { - if (async) - { - await _missingClientSideEncryptionKeyListener.OnFailureAsync(message, e).ConfigureAwait(false); - } - else - { - _missingClientSideEncryptionKeyListener.OnFailure(message, e); - } - } - } - return filteredMessages.ToArray(); - } - private async Task ClientSideDecryptMessagesInternal(PeekedMessage[] messages, bool async, CancellationToken cancellationToken) - { - var filteredMessages = new List(); - foreach (var message in messages) - { - try - { - message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); - filteredMessages.Add(message); - } - catch (Exception e) when (_missingClientSideEncryptionKeyListener != default) - { - if (async) - { - await _missingClientSideEncryptionKeyListener.OnFailureAsync(message, e).ConfigureAwait(false); - } - else - { - _missingClientSideEncryptionKeyListener.OnFailure(message, e); - } - } - } - return filteredMessages.ToArray(); - } - - private async Task ClientSideDecryptInternal(string downloadedMessage, bool async, CancellationToken cancellationToken) - { - if (!EncryptedMessageSerializer.TryDeserialize(downloadedMessage, out var encryptedMessage)) - { - return downloadedMessage; // not recognized as client-side encrypted message - } - - var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); - var decryptedMessageStream = await ClientSideDecryptor.DecryptInternal( - encryptedMessageStream, - encryptedMessage.EncryptionData, - ivInStream: false, - ClientSideEncryption.KeyResolver, - ClientSideEncryption.KeyEncryptionKey, - noPadding: false, - async: async, - cancellationToken).ConfigureAwait(false); - // if we got back the stream we put in, then we couldn't decrypt and are supposed to return the original - // message to the user - if (encryptedMessageStream == decryptedMessageStream) - { - return downloadedMessage; - } - - return new StreamReader(decryptedMessageStream, Encoding.UTF8).ReadToEnd(); - } } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs index 725dcd463b150..44646a24ddc05 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs @@ -14,15 +14,30 @@ public static class QueueClientExtensions /// /// Client to base off of. /// New encryption options. Setting this to default will clear client-side encryption. - /// /// New instance with provided options and same internals otherwise. - public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions, IClientSideDecryptionFailureListener listener = default) + public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) => new QueueClient( client.Uri, client.Pipeline, client.Version, client.ClientDiagnostics, clientSideEncryptionOptions, + client.OnClientSideDecryptionFailure); + + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// Listener for when decryption of a single message fails. + /// New instance with provided options and same internals otherwise. + public static QueueClient WithClientSideEncryptionFailureListener(this QueueClient client, IClientSideDecryptionFailureListener listener) + => new QueueClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + client.ClientSideEncryption, listener); } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs new file mode 100644 index 0000000000000..fcc8b04524fba --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography; +using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; +using Azure.Storage.Queues.Specialized.Models; + +namespace Azure.Storage.Queues +{ + internal class QueueClientSideDecryptor + { + private readonly IKeyEncryptionKeyResolver _resolver; + private readonly IKeyEncryptionKey _cachedIKey; + private readonly IClientSideDecryptionFailureListener _listener; + + public QueueClientSideDecryptor(ClientSideEncryptionOptions options, IClientSideDecryptionFailureListener listener) + { + _resolver = options.KeyResolver; + _cachedIKey = options.KeyEncryptionKey; + _listener = listener; + } + + public async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) + { + var filteredMessages = new List(); + foreach (var message in messages) + { + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (Exception e) when (_listener != default) + { + if (async) + { + await _listener.OnFailureAsync(message, e).ConfigureAwait(false); + } + else + { + _listener.OnFailure(message, e); + } + } + } + return filteredMessages.ToArray(); + } + public async Task ClientSideDecryptMessagesInternal(PeekedMessage[] messages, bool async, CancellationToken cancellationToken) + { + var filteredMessages = new List(); + foreach (var message in messages) + { + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (Exception e) when (_listener != default) + { + if (async) + { + await _listener.OnFailureAsync(message, e).ConfigureAwait(false); + } + else + { + _listener.OnFailure(message, e); + } + } + } + return filteredMessages.ToArray(); + } + + private async Task ClientSideDecryptInternal(string downloadedMessage, bool async, CancellationToken cancellationToken) + { + if (!EncryptedMessageSerializer.TryDeserialize(downloadedMessage, out var encryptedMessage)) + { + return downloadedMessage; // not recognized as client-side encrypted message + } + + var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); + var decryptedMessageStream = await ClientSideDecryptor.DecryptInternal( + encryptedMessageStream, + encryptedMessage.EncryptionData, + ivInStream: false, + _resolver, + _cachedIKey, + noPadding: false, + async: async, + cancellationToken).ConfigureAwait(false); + // if we got back the stream we put in, then we couldn't decrypt and are supposed to return the original + // message to the user + if (encryptedMessageStream == decryptedMessageStream) + { + return downloadedMessage; + } + + return new StreamReader(decryptedMessageStream, Encoding.UTF8).ReadToEnd(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs new file mode 100644 index 0000000000000..8b417ee962034 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Specialized.Models; + +namespace Azure.Storage.Queues +{ + internal class QueueClientSideEncryptor + { + private readonly IKeyEncryptionKey _keyEncryptionKey; + private readonly string _keyWrapAlgorithm; + + public QueueClientSideEncryptor(ClientSideEncryptionOptions options) + { + _keyEncryptionKey = options.KeyEncryptionKey; + _keyWrapAlgorithm = options.KeyWrapAlgorithm; + } + + public async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) + { + var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); + (byte[] ciphertext, EncryptionData encryptionData) = await ClientSideEncryptor.BufferedEncryptInternal( + new MemoryStream(bytesToEncrypt), + _keyEncryptionKey, + _keyWrapAlgorithm, + async, + cancellationToken).ConfigureAwait(false); + + return EncryptedMessageSerializer.Serialize(new EncryptedMessage + { + EncryptedMessageContents = Convert.ToBase64String(ciphertext), + EncryptionData = encryptionData + }); + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index ac8997c97c3e8..ce10e105c6a18 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -18,6 +18,5 @@ - \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index 1879817bb254b..dfeb6a3f21cff 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -12,9 +12,9 @@ using Azure.Core.Cryptography; using Azure.Core.TestFramework; using Azure.Security.KeyVault.Keys.Cryptography; -using Azure.Storage.Blobs.Tests; using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; using Azure.Storage.Queues.Specialized.Models; using Azure.Storage.Queues.Tests; using Moq; @@ -37,9 +37,10 @@ public ClientSideEncryptionTests(bool async) { } - #region Utility - - private string LocalManualEncryption(string message, byte[] key, byte[] iv) + /// + /// Provides encryption functionality clone of client logic, letting us validate the client got it right end-to-end. + /// + private string EncryptData(string message, byte[] key, byte[] iv) { using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) using (var encryptor = aesProvider.CreateEncryptor()) @@ -130,6 +131,39 @@ private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = defau return keyMock; } + private Mock GetAlwaysFailsKeyResolver(bool throws) + { + var mock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + if (throws) + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns(Task.FromResult(null)); + } + } + else + { + if (throws) + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((IKeyEncryptionKey)null); + } + } + + return mock; + } + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) { var resolverMock = new Mock(MockBehavior.Strict); @@ -195,7 +229,63 @@ private static byte[] Xor(byte[] a, byte[] b) return result; } - #endregion + + private Mock GetFailureListener() + { + var mock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + mock.Setup(l => l.OnFailureAsync(IsNotNull(), IsNotNull())) + .Returns(Task.CompletedTask); + mock.Setup(l => l.OnFailureAsync(IsNotNull(), IsNotNull())) + .Returns(Task.CompletedTask); + } + else + { + mock.Setup(l => l.OnFailure(IsNotNull(), IsNotNull())); + mock.Setup(l => l.OnFailure(IsNotNull(), IsNotNull())); + } + + return mock; + } + + [Test] + [LiveOnly] + public void CanSwapKey() + { + var options1 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "foo" + }; + var options2 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "bar" + }; + var listener1 = GetFailureListener().Object; + var listener2 = GetFailureListener().Object; + + var client = new QueueClient(new Uri("http://someuri.com"), new ExtendedQueueClientOptions() + { + ClientSideEncryption = options1, + OnClientSideDecryptionFailure = listener1 + }); + + Assert.AreEqual(options1.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options1.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options1.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + Assert.AreEqual(listener1, client.OnClientSideDecryptionFailure); + + client = client.WithClientSideEncryptionOptions(options2).WithClientSideEncryptionFailureListener(listener2); + + Assert.AreEqual(options2.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options2.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options2.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + Assert.AreEqual(listener2, client.OnClientSideDecryptionFailure); + } [TestCase(16, false)] // a single cipher block [TestCase(14, false)] // a single unalligned cipher block @@ -232,7 +322,7 @@ public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) var explicitlyUnwrappedKey = IsAsync ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); - string expectedEncryptedMessage = LocalManualEncryption( + string expectedEncryptedMessage = EncryptData( message, explicitlyUnwrappedKey, encryptionMetadata.ContentEncryptionIV); @@ -491,10 +581,10 @@ public async Task OnlyOneKeyResolveAndUnwrapCall() await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken); - var resolveSyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("Resolve"); - var resolveAsyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("ResolveAsync"); - var unwrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); - var unwrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); + System.Reflection.MethodInfo resolveSyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("Resolve"); + System.Reflection.MethodInfo resolveAsyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("ResolveAsync"); + System.Reflection.MethodInfo unwrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); + System.Reflection.MethodInfo unwrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); Assert.AreEqual(1, IsAsync ? mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveAsyncMethod) @@ -512,17 +602,21 @@ public async Task OnlyOneKeyResolveAndUnwrapCall() } } - [TestCase(true, false)] - [TestCase(false, false)] - [TestCase(true, true)] - [TestCase(false, true)] + [TestCase(true, false, false)] + [TestCase(false, false, false)] + [TestCase(true, true, false)] + [TestCase(false, true, false)] + [TestCase(true, false, true)] + [TestCase(false, false, true)] + [TestCase(true, true, true)] + [TestCase(false, true, true)] [LiveOnly] - public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) + public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows, bool peek) { - MockMissingClientSideEncryptionKeyListener listener = null; + Mock listener = null; if (useListener) { - listener = new MockMissingClientSideEncryptionKeyListener(); + listener = GetFailureListener(); } const int numMessages = 5; @@ -544,7 +638,8 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) } bool threw = false; - QueueMessage[] result = default; + var resolver = GetAlwaysFailsKeyResolver(resolverThrows); + int returnedMessages = int.MinValue; // obviously wrong value, but need to initialize to something before try block try { // download but can't find key @@ -552,11 +647,18 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { // note decryption will throw whether the resolver throws or just returns null - KeyResolver = new AlwaysFailsKeyEncryptionKeyResolver() { ShouldThrow = resolverThrows }, + KeyResolver = resolver.Object, KeyWrapAlgorithm = "test" }; - options._onClientSideDecryptionFailure = listener; - result = await InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options)).ReceiveMessagesAsync(numMessages, cancellationToken: s_cancellationToken); + options._onClientSideDecryptionFailure = listener.Object; + var badQueueClient = InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options)); + returnedMessages = peek + ? (await badQueueClient.PeekMessagesAsync(numMessages, cancellationToken: s_cancellationToken)).Value.Length + : (await badQueueClient.ReceiveMessagesAsync(numMessages, cancellationToken: s_cancellationToken)).Value.Length; + } + catch (MockException e) + { + Assert.Fail(e.Message); } catch (Exception) { @@ -568,8 +670,48 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows) if (useListener) { - Assert.AreEqual(numMessages, listener.TimesInvoked); - Assert.AreEqual(0, result.Length); // all messages should have been filtered out + // we already asserted the correct method was called in `catch (MockException e)` + Assert.AreEqual(numMessages, resolver.Invocations.Count); + + // 4 possible methods we could have been testing + System.Reflection.MethodInfo onReceiveFailAsync = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailureAsync", new Type[] { typeof(QueueMessage), typeof(Exception) }); + System.Reflection.MethodInfo onReceiveFail = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailure", new Type[] { typeof(QueueMessage), typeof(Exception) }); + System.Reflection.MethodInfo onPeekFailAsync = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailureAsync", new Type[] { typeof(PeekedMessage), typeof(Exception) }); + System.Reflection.MethodInfo onPeekFail = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailure", new Type[] { typeof(PeekedMessage), typeof(Exception) }); + + // determine what method we were testing and which we weren't + System.Reflection.MethodInfo targetMethod; + IEnumerable nonTargetMethods; + if (IsAsync && peek) + { + targetMethod = onPeekFailAsync; + nonTargetMethods = new List { onReceiveFail, onReceiveFailAsync, onPeekFail }; + } + else if (IsAsync) + { + targetMethod = onReceiveFailAsync; + nonTargetMethods = new List { onReceiveFail, onPeekFail, onPeekFailAsync }; + } + else if (peek) + { + targetMethod = onPeekFail; + nonTargetMethods = new List { onReceiveFail, onReceiveFailAsync, onPeekFailAsync }; + } + else + { + targetMethod = onReceiveFail; + nonTargetMethods = new List { onPeekFailAsync, onReceiveFailAsync, onPeekFail }; + } + + // assert target method was called the expected number of times and other methods weren't called at all + Assert.AreEqual(numMessages, listener.Invocations.Count(invocation => invocation.Method == targetMethod)); + foreach (var method in nonTargetMethods) + { + Assert.AreEqual(0, listener.Invocations.Count(invocation => invocation.Method == method)); + } + + // assert all messages were filtered out of formal response + Assert.AreEqual(0, returnedMessages); } } } diff --git a/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs b/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs deleted file mode 100644 index 5497c699cac85..0000000000000 --- a/sdk/storage/Azure.Storage.Queues/tests/MockMissingClientSideEncryptionKeyListener.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Threading.Tasks; -using Azure.Storage.Queues.Models; -using Azure.Storage.Queues.Specialized; -using NUnit.Framework; - -namespace Azure.Storage.Queues.Tests -{ - internal class MockMissingClientSideEncryptionKeyListener : IClientSideDecryptionFailureListener - { - public int TimesInvoked { get; private set; } = 0; - - public void OnFailure(QueueMessage message, Exception e) - { - Assert.IsNotNull(e); - TimesInvoked++; - } - - public void OnFailure(PeekedMessage message, Exception e) - { - Assert.IsNotNull(e); - TimesInvoked++; - } - - public Task OnFailureAsync(QueueMessage message, Exception e) - { - Assert.IsNotNull(e); - TimesInvoked++; - return Task.CompletedTask; - } - - public Task OnFailureAsync(PeekedMessage message, Exception e) - { - Assert.IsNotNull(e); - TimesInvoked++; - return Task.CompletedTask; - } - } -} From e0ae5209c218b555e5a79026108e93fe3a6afcb6 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 2 Jun 2020 10:01:19 -0700 Subject: [PATCH 11/21] Export-API --- .../api/Azure.Storage.Queues.netstandard2.0.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index b6b0797ab3687..6f0f477adf06e 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -332,7 +332,8 @@ public partial interface IClientSideDecryptionFailureListener } public static partial class QueueClientExtensions { - public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions, Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener listener = null) { throw null; } + public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionFailureListener(this Azure.Storage.Queues.QueueClient client, Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener listener) { throw null; } + public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } } } namespace Azure.Storage.Sas From 1633ce0c70b0c6d48b461249df7fecb69a2332ca Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 2 Jun 2020 12:12:42 -0700 Subject: [PATCH 12/21] Constant/error/assertion refactors --- .../src/AppendBlobClient.cs | 14 ++++- .../src/BlobClientSideDecryptor.cs | 24 ++++---- .../src/BlobClientSideEncryptor.cs | 4 +- .../src/BlockBlobClient.cs | 12 ++++ .../Azure.Storage.Blobs/src/PageBlobClient.cs | 15 ++++- .../tests/ClientSideEncryptionTests.cs | 14 ++--- .../ClientSideDecryptor.cs | 59 ++++++++++++++++++- .../ClientSideEncryptionVersionExtensions.cs | 4 +- .../ClientSideEncryptor.cs | 4 +- .../EncryptionConstants.cs | 28 --------- .../ClientsideEncryption/EncryptionErrors.cs | 27 --------- .../Models/EncryptionData.cs | 4 +- .../src/Shared/Constants.cs | 23 ++++++++ .../src/Shared/Errors.Clients.cs | 18 ++++++ .../Azure.Storage.Common/src/Shared/Errors.cs | 4 -- 15 files changed, 163 insertions(+), 91 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs delete mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 4f6e9e7b8b745..cfe4277de7f5f 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -98,6 +98,7 @@ public AppendBlobClient(string connectionString, string blobContainerName, strin public AppendBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -118,6 +119,7 @@ public AppendBlobClient(string connectionString, string blobContainerName, strin public AppendBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -141,6 +143,7 @@ public AppendBlobClient(Uri blobUri, BlobClientOptions options = default) public AppendBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -164,6 +167,7 @@ public AppendBlobClient(Uri blobUri, StorageSharedKeyCredential credential, Blob public AppendBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -202,6 +206,14 @@ internal AppendBlobClient( encryptionScope) { } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(AppendBlobClient)); + } + } #endregion ctors /// @@ -1051,7 +1063,7 @@ public static AppendBlobClient GetAppendBlobClient( { if (client.ClientSideEncryption != default) { - throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(AppendBlobClient)); } return new AppendBlobClient( client.Uri.AppendToPath(blobName), diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs index 60e77aaa3b7e6..2b9ae0f8300e9 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -42,7 +42,7 @@ public async Task ClientSideDecryptInternal( return await TrimStreamInternal(content, originalRange, contentRange, pulledOutIv: false, async, cancellationToken).ConfigureAwait(false); } - bool ivInStream = originalRange.Offset >= EncryptionConstants.EncryptionBlockSize; + bool ivInStream = originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize; // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. var plaintext = await ClientSideDecryptor.DecryptInternal( @@ -63,7 +63,7 @@ private static async Task TrimStreamInternal(Stream stream, HttpRange or // retrim start of stream to original requested location // keeping in mind whether we already pulled the IV out of the stream as well int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0)) - - (pulledOutIv ? EncryptionConstants.EncryptionBlockSize : 0); + - (pulledOutIv ? Constants.ClientSideEncryption.EncryptionBlockSize : 0); if (gap > 0) { // throw away initial bytes we want to trim off; stream cannot seek into future @@ -91,16 +91,16 @@ private static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata met { return default; } - if (!metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string encryptedDataString)) + if (!metadata.TryGetValue(Constants.ClientSideEncryption.EncryptionDataKey, out string encryptedDataString)) { return default; } EncryptionData encryptionData = EncryptionDataSerializer.Deserialize(encryptedDataString); - _ = encryptionData.ContentEncryptionIV ?? throw EncryptionErrors.MissingEncryptionMetadata( + _ = encryptionData.ContentEncryptionIV ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata( nameof(EncryptionData.ContentEncryptionIV)); - _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw EncryptionErrors.MissingEncryptionMetadata( + _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata( nameof(EncryptionData.WrappedContentKey.EncryptedKey)); return encryptionData; @@ -151,7 +151,7 @@ internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) { // Align with encryption block boundary. int diff; - if ((diff = (int)(originalRange.Offset % EncryptionConstants.EncryptionBlockSize)) != 0) + if ((diff = (int)(originalRange.Offset % Constants.ClientSideEncryption.EncryptionBlockSize)) != 0) { offsetAdjustment += diff; if (adjustedDownloadCount != default) @@ -161,13 +161,13 @@ internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) } // Account for IV. - if (originalRange.Offset >= EncryptionConstants.EncryptionBlockSize) + if (originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize) { - offsetAdjustment += EncryptionConstants.EncryptionBlockSize; + offsetAdjustment += Constants.ClientSideEncryption.EncryptionBlockSize; // Increment adjustedDownloadCount if necessary. if (adjustedDownloadCount != default) { - adjustedDownloadCount += EncryptionConstants.EncryptionBlockSize; + adjustedDownloadCount += Constants.ClientSideEncryption.EncryptionBlockSize; } } } @@ -177,9 +177,9 @@ internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) if (adjustedDownloadCount != null) { adjustedDownloadCount += ( - EncryptionConstants.EncryptionBlockSize - (int)(adjustedDownloadCount - % EncryptionConstants.EncryptionBlockSize) - ) % EncryptionConstants.EncryptionBlockSize; + Constants.ClientSideEncryption.EncryptionBlockSize - (int)(adjustedDownloadCount + % Constants.ClientSideEncryption.EncryptionBlockSize) + ) % Constants.ClientSideEncryption.EncryptionBlockSize; } return new HttpRange(originalRange.Offset - offsetAdjustment, adjustedDownloadCount); diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs index f3c802f939645..35fc640527323 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs @@ -67,12 +67,12 @@ public BlobClientSideEncryptor(ClientSideEncryptionOptions options) // GetExpectedCryptoStreamLength(originalLength)); metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); - metadata.Add(EncryptionConstants.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); + metadata.Add(Constants.ClientSideEncryption.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); return (nonSeekableCiphertext, metadata); } private static long GetExpectedCryptoStreamLength(long originalLength) - => originalLength + (EncryptionConstants.EncryptionBlockSize - originalLength % EncryptionConstants.EncryptionBlockSize); + => originalLength + (Constants.ClientSideEncryption.EncryptionBlockSize - originalLength % Constants.ClientSideEncryption.EncryptionBlockSize); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 796217ff44054..a6b9f4e4743f2 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -153,6 +153,7 @@ public BlockBlobClient(string connectionString, string containerName, string blo public BlockBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -172,6 +173,7 @@ public BlockBlobClient(string connectionString, string blobContainerName, string public BlockBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -194,6 +196,7 @@ public BlockBlobClient(Uri blobUri, BlobClientOptions options = default) public BlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -216,6 +219,7 @@ public BlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobC public BlockBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -278,6 +282,14 @@ protected static BlockBlobClient CreateClient(Uri blobUri, BlobClientOptions opt { return new BlockBlobClient(blobUri, pipeline, options.Version, new ClientDiagnostics(options), null, null); } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + } #endregion ctors /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index d23fe6e7aa7c9..1122a327b313e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -6,6 +6,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using System.Xml.Serialization; using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; @@ -98,6 +99,7 @@ public PageBlobClient(string connectionString, string blobContainerName, string public PageBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -117,6 +119,7 @@ public PageBlobClient(string connectionString, string blobContainerName, string public PageBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -139,6 +142,7 @@ public PageBlobClient(Uri blobUri, BlobClientOptions options = default) public PageBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -161,6 +165,7 @@ public PageBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobCl public PageBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -198,6 +203,14 @@ internal PageBlobClient( encryptionScope) { } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(PageBlobClient)); + } + } #endregion ctors /// @@ -2621,7 +2634,7 @@ public static PageBlobClient GetPageBlobClient( { if (client.ClientSideEncryption != default) { - throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(PageBlobClient)); } return new PageBlobClient( client.Uri.AppendToPath(blobName), diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index b3525ef844bf4..1ae5e16e304c1 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -267,7 +267,7 @@ public async Task UploadAsync(long dataSize) var encryptedData = encryptedDataStream.ToArray(); // encrypt original data manually for comparison - if (!(await blob.GetPropertiesAsync()).Value.Metadata.TryGetValue(EncryptionConstants.EncryptionDataKey, out string serialEncryptionData)) + if (!(await blob.GetPropertiesAsync()).Value.Metadata.TryGetValue(Constants.ClientSideEncryption.EncryptionDataKey, out string serialEncryptionData)) { Assert.Fail("No encryption metadata present."); } @@ -583,15 +583,15 @@ public async Task CannotFindKeyAsync(bool resolverThrows) // using 5 to setup offsets to avoid any off-by-one confusion in debugging [TestCase(0, null)] - [TestCase(0, 2 * EncryptionConstants.EncryptionBlockSize)] - [TestCase(0, 2 * EncryptionConstants.EncryptionBlockSize + 5)] - [TestCase(EncryptionConstants.EncryptionBlockSize, EncryptionConstants.EncryptionBlockSize)] - [TestCase(EncryptionConstants.EncryptionBlockSize, EncryptionConstants.EncryptionBlockSize + 5)] - [TestCase(EncryptionConstants.EncryptionBlockSize + 5, 2 * EncryptionConstants.EncryptionBlockSize)] + [TestCase(0, 2 * Constants.ClientSideEncryption.EncryptionBlockSize)] + [TestCase(0, 2 * Constants.ClientSideEncryption.EncryptionBlockSize + 5)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize, Constants.ClientSideEncryption.EncryptionBlockSize)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize, Constants.ClientSideEncryption.EncryptionBlockSize + 5)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize + 5, 2 * Constants.ClientSideEncryption.EncryptionBlockSize)] [LiveOnly] public async Task AppropriateRangeDownloadOnPlaintext(int rangeOffset, int? rangeLength) { - var data = GetRandomBuffer(rangeOffset + (rangeLength ?? Constants.KB) + EncryptionConstants.EncryptionBlockSize); + var data = GetRandomBuffer(rangeOffset + (rangeLength ?? Constants.KB) + Constants.ClientSideEncryption.EncryptionBlockSize); var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey().Object).Object; await using (var disposable = await GetTestContainerAsync()) { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs index 6535e05065eb8..21889d9c98d20 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -56,6 +56,59 @@ public static async Task DecryptInternal( bool noPadding, bool async, CancellationToken cancellationToken) + { + switch (encryptionData.EncryptionAgent.Protocol) + { + case ClientSideEncryptionVersion.V1_0: + return await DecryptInternalV1_0(ciphertext, encryptionData, ivInStream, keyResolver, potentialCachedKeyWrapper, noPadding, async, cancellationToken).ConfigureAwait(false); + default: + throw Errors.ClientSideEncryption.BadEncryptionAgent(encryptionData.EncryptionAgent.Protocol.ToString()); + } + } + + /// + /// Decrypts the given stream if decryption information is provided. + /// Does not shave off unwanted start/end bytes, but will shave off padding. + /// + /// Stream to decrypt. + /// + /// Encryption metadata and wrapped content encryption key. + /// + /// + /// Whether to use the first block of the stream for the IV instead of the value in + /// . Generally for partial blob downloads where the + /// previous block of the ciphertext is the IV for the next. + /// + /// + /// Resolver to fetch the key encryption key. + /// + /// + /// Clients that can upload data have a key encryption key stored on them. Checking if + /// a cached key exists and matches the saves a call + /// to the external key resolver implementation when available. + /// + /// + /// Whether to ignore padding. Generally for partial blob downloads where the end of + /// the blob (where the padding occurs) was not downloaded. + /// + /// Whether to perform this function asynchronously. + /// + /// + /// Decrypted plaintext. + /// + /// + /// Exceptions thrown based on implementations of and + /// . + /// + public static async Task DecryptInternalV1_0( + Stream ciphertext, + EncryptionData encryptionData, + bool ivInStream, + IKeyEncryptionKeyResolver keyResolver, + IKeyEncryptionKey potentialCachedKeyWrapper, + bool noPadding, + bool async, + CancellationToken cancellationToken) { var contentEncryptionKey = await GetContentEncryptionKeyAsync( encryptionData, @@ -75,7 +128,7 @@ public static async Task DecryptInternal( } else { - IV = new byte[EncryptionConstants.EncryptionBlockSize]; + IV = new byte[Constants.ClientSideEncryption.EncryptionBlockSize]; if (async) { await ciphertext.ReadAsync(IV, 0, IV.Length, cancellationToken).ConfigureAwait(false); @@ -147,7 +200,7 @@ private static async Task> GetContentEncryptionKeyAsync( // exception here instead of nullref. if (key == default) { - throw EncryptionErrors.KeyNotFound(encryptionData.WrappedContentKey.KeyId); + throw Errors.ClientSideEncryption.KeyNotFound(encryptionData.WrappedContentKey.KeyId); } return async @@ -194,7 +247,7 @@ private static Stream WrapStream( } } - throw EncryptionErrors.BadEncryptionAlgorithm(encryptionData.EncryptionAgent.EncryptionAlgorithm.ToString()); + throw Errors.ClientSideEncryption.BadEncryptionAlgorithm(encryptionData.EncryptionAgent.EncryptionAlgorithm.ToString()); } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs index d284a5dd4cb9a..ca5e03ac3ee11 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs @@ -18,7 +18,7 @@ public static string Serialize(this ClientSideEncryptionVersion version) return ClientSideEncryptionVersionString.V1_0; default: // sanity check; serialize is in this file to make it easy to add the serialization cases - throw Errors.ClientSideEncryptionVersionNotSupported(); + throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported(); } } @@ -30,7 +30,7 @@ public static ClientSideEncryptionVersion ToClientSideEncryptionVersion(this str return ClientSideEncryptionVersion.V1_0; default: // This library doesn't support the stated encryption version - throw Errors.ClientSideEncryptionVersionNotSupported(versionString); + throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported(versionString); } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs index 999b1a48433ee..1a85cbff72f7b 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs @@ -29,7 +29,7 @@ internal static class ClientSideEncryptor bool async, CancellationToken cancellationToken) { - var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); EncryptionData encryptionData = default; Stream ciphertext = default; @@ -69,7 +69,7 @@ internal static class ClientSideEncryptor bool async, CancellationToken cancellationToken) { - var generatedKey = CreateKey(EncryptionConstants.EncryptionKeySizeBits); + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); EncryptionData encryptionData = default; var ciphertext = new MemoryStream(); byte[] bufferedCiphertext = default; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs deleted file mode 100644 index 679a39e5b05d2..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionConstants.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -namespace Azure.Storage.Cryptography -{ - internal static class EncryptionConstants - { - public const ClientSideEncryptionVersion CurrentVersion = ClientSideEncryptionVersion.V1_0; - - public const string AgentMetadataKey = "EncryptionLibrary"; - - public const string AesCbcPkcs5Padding = "AES/CBC/PKCS5Padding"; - - public const string AesCbcNoPadding = "AES/CBC/NoPadding"; - - public const string Aes = "AES"; - - public const string EncryptionDataKey = "encryptiondata"; - - public const string EncryptionMode = "FullBlob"; - - public const int EncryptionBlockSize = 16; - - public const int EncryptionKeySizeBits = 256; - - public const string XMsRange = "x-ms-range"; - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs deleted file mode 100644 index 48303644b4fdb..0000000000000 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/EncryptionErrors.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; - -namespace Azure.Storage.Cryptography -{ - internal static class EncryptionErrors - { - public static ArgumentException KeyNotFound(string keyId) - => new ArgumentException($"Resolution of id {keyId} returned null."); - - public static ArgumentException BadEncryptionAgent(string agent) - => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + - $"the Encryption Agent protocol \"{agent}\" set on the blob."); - - public static ArgumentException BadEncryptionAlgorithm(string algorithm) - => new ArgumentException($"Invalid Encryption Algorithm \"{algorithm}\" found on the resource. This version of the client" + - "library does not support the given encryption algorithm."); - - public static ArgumentException NoKeyAccessor() - => new ArgumentException("No key to decrypt data with."); - - public static InvalidOperationException MissingEncryptionMetadata(string field) - => new InvalidOperationException($"Missing field \"{field}\" in encryption metadata"); - } -} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs index decbaa460bac9..c60cadf21d57a 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -53,7 +53,7 @@ internal static async Task CreateInternalV1_0( CancellationToken cancellationToken) => new EncryptionData() { - EncryptionMode = EncryptionConstants.EncryptionMode, + EncryptionMode = Constants.ClientSideEncryption.EncryptionMode, ContentEncryptionIV = contentEncryptionIv, EncryptionAgent = new EncryptionAgent() { @@ -62,7 +62,7 @@ internal static async Task CreateInternalV1_0( }, KeyWrappingMetadata = new Dictionary() { - { EncryptionConstants.AgentMetadataKey, AgentString } + { Constants.ClientSideEncryption.AgentMetadataKey, AgentString } }, WrappedContentKey = new KeyEnvelope() { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 7f7ba5fb7e65e..fef6d7a1de0b3 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -388,6 +388,29 @@ internal static class AccountResources } } + internal static class ClientSideEncryption + { + public const ClientSideEncryptionVersion CurrentVersion = ClientSideEncryptionVersion.V1_0; + + public const string AgentMetadataKey = "EncryptionLibrary"; + + public const string AesCbcPkcs5Padding = "AES/CBC/PKCS5Padding"; + + public const string AesCbcNoPadding = "AES/CBC/NoPadding"; + + public const string Aes = "AES"; + + public const string EncryptionDataKey = "encryptiondata"; + + public const string EncryptionMode = "FullBlob"; + + public const int EncryptionBlockSize = 16; + + public const int EncryptionKeySizeBits = 256; + + public const string XMsRange = "x-ms-range"; + } + /// /// XML Element Name constant values. /// diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index 81fd09680741a..7148d5626bbce 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -88,6 +88,10 @@ public static void VerifyHttpsTokenAuth(Uri uri) public static class ClientSideEncryption { + public static InvalidOperationException ClientSideEncryptionVersionNotSupported(string versionString = default) + => new InvalidOperationException("This library does not support the given version of client-side encryption." + + versionString == default ? "" : $" Version ID = {versionString}"); + public static InvalidOperationException TypeNotSupported(Type type) => new InvalidOperationException( $"Client-side encryption is not supported for type \"{type.FullName}\". " + @@ -95,6 +99,20 @@ public static InvalidOperationException TypeNotSupported(Type type) public static InvalidOperationException MissingRequiredEncryptionResources(params string[] resourceNames) => new InvalidOperationException("Cannot encrypt without specifying " + string.Join(",", resourceNames.AsEnumerable())); + + public static ArgumentException KeyNotFound(string keyId) + => new ArgumentException($"Resolution of id {keyId} returned null."); + + public static ArgumentException BadEncryptionAgent(string agent) + => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + + $"the Encryption Agent protocol \"{agent}\" set on the blob."); + + public static ArgumentException BadEncryptionAlgorithm(string algorithm) + => new ArgumentException($"Invalid Encryption Algorithm \"{algorithm}\" found on the resource. This version of the client" + + "library does not support the given encryption algorithm."); + + public static InvalidOperationException MissingEncryptionMetadata(string field) + => new InvalidOperationException($"Missing field \"{field}\" in encryption metadata"); } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index 454d68f1a1b31..0ee6912e60d3c 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -42,9 +42,5 @@ public static ArgumentOutOfRangeException InvalidSasProtocol(string protocol, st public static ArgumentException InvalidService(char s) => new ArgumentException($"Invalid service: '{s}'"); - - public static InvalidOperationException ClientSideEncryptionVersionNotSupported(string versionString = default) - => new InvalidOperationException("This library does not support the given version of client-side encryption." + - versionString == default ? "" : $" Version ID = {versionString}"); } } From e32eaa65bc7dba079911fd034f2c66cd4d443c80 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 2 Jun 2020 15:11:41 -0700 Subject: [PATCH 13/21] Refactors --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 2 +- .../Azure.Storage.Blobs/src/BlobClient.cs | 9 ++- .../src/BlobClientSideDecryptor.cs | 13 ++-- .../src/BlobClientSideEncryptor.cs | 29 ++------ .../ClientSideDecryptor.cs | 70 +++++++++---------- .../ClientSideEncryptor.cs | 45 +++++++----- .../Models/EncryptionDataSerializer.cs | 2 +- .../Azure.Storage.Queues/src/QueueClient.cs | 6 +- .../src/QueueClientSideDecryptor.cs | 13 ++-- .../src/QueueClientSideEncryptor.cs | 13 ++-- .../tests/EncryptedMessageSerializerTests.cs | 24 ++++--- 11 files changed, 104 insertions(+), 122 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 62265f8eff28c..8647cad8a6587 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -713,7 +713,7 @@ private async Task> DownloadInternal( // we already return a nonseekable stream; returning a crypto stream is fine if (UsingClientSideEncryption) { - stream = await new BlobClientSideDecryptor(ClientSideEncryption) + stream = await new BlobClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption)) .ClientSideDecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index 36fce14af91fd..ae5a8a2228db9 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -9,6 +9,7 @@ using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Blobs @@ -950,7 +951,7 @@ internal async Task> StagedUploadAsync( if (UsingClientSideEncryption) { // content is now unseekable, so PartitionedUploader will be forced to do a buffered multipart upload - (content, metadata) = await new BlobClientSideEncryptor(ClientSideEncryption) + (content, metadata) = await new BlobClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) .ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false); } @@ -1028,8 +1029,10 @@ internal async Task> StagedUploadAsync( bool async = true, CancellationToken cancellationToken = default) { - // TODO uncomment when upload from file gets its own implementation - //// if clientside encryption, upload from stream, where our crypto logic is + // TODO Upload from file will get it's own implementation in the future that opens more + // than one stream at once. This is incompatible with .NET's CryptoStream. We will + // need to uncomment the below code and revert to upload from stream if client-side + // encryption is enabled. //if (ClientSideEncryption != default) //{ // using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read)) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs index 2b9ae0f8300e9..03bfebcc98513 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -4,7 +4,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Azure.Core.Cryptography; using Azure.Storage.Blobs.Models; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; @@ -15,13 +14,11 @@ namespace Azure.Storage.Blobs { internal class BlobClientSideDecryptor { - private readonly IKeyEncryptionKeyResolver _resolver; - private readonly IKeyEncryptionKey _cachedIKey; + private readonly ClientSideDecryptor _decryptor; - public BlobClientSideDecryptor(ClientSideEncryptionOptions options) + public BlobClientSideDecryptor(ClientSideDecryptor decryptor) { - _resolver = options.KeyResolver; - _cachedIKey = options.KeyEncryptionKey; + _decryptor = decryptor; } public async Task ClientSideDecryptInternal( @@ -45,12 +42,10 @@ public async Task ClientSideDecryptInternal( bool ivInStream = originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize; // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. - var plaintext = await ClientSideDecryptor.DecryptInternal( + var plaintext = await _decryptor.DecryptInternal( content, encryptionData, ivInStream, - _resolver, - _cachedIKey, CanIgnorePadding(contentRange), async, cancellationToken).ConfigureAwait(false); diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs index 35fc640527323..e94c6e694c89d 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs @@ -4,10 +4,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Azure.Core.Cryptography; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Metadata = System.Collections.Generic.IDictionary; @@ -16,13 +14,11 @@ namespace Azure.Storage.Blobs { internal class BlobClientSideEncryptor { - private readonly IKeyEncryptionKey _keyEncryptionKey; - private readonly string _keyWrapAlgorithm; + private readonly ClientSideEncryptor _encryptor; - public BlobClientSideEncryptor(ClientSideEncryptionOptions options) + public BlobClientSideEncryptor(ClientSideEncryptor encryptor) { - _keyEncryptionKey = options.KeyEncryptionKey; - _keyWrapAlgorithm = options.KeyWrapAlgorithm; + _encryptor = encryptor; } /// @@ -47,32 +43,15 @@ public BlobClientSideEncryptor(ClientSideEncryptionOptions options) bool async, CancellationToken cancellationToken) { - if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) - { - throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); - } - - //long originalLength = content.Length; - - (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await ClientSideEncryptor.EncryptInternal( + (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await _encryptor.EncryptInternal( content, - _keyEncryptionKey, - _keyWrapAlgorithm, async, cancellationToken).ConfigureAwait(false); - //Stream seekableCiphertext = new RollingBufferStream( - // nonSeekableCiphertext, - // EncryptionConstants.DefaultRollingBufferSize, - // GetExpectedCryptoStreamLength(originalLength)); - metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); metadata.Add(Constants.ClientSideEncryption.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); return (nonSeekableCiphertext, metadata); } - - private static long GetExpectedCryptoStreamLength(long originalLength) - => originalLength + (Constants.ClientSideEncryption.EncryptionBlockSize - originalLength % Constants.ClientSideEncryption.EncryptionBlockSize); } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs index 21889d9c98d20..270d1d02d577a 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -11,8 +11,26 @@ namespace Azure.Storage.Cryptography { - internal static class ClientSideDecryptor + internal class ClientSideDecryptor { + /// + /// Clients that can upload data have a key encryption key stored on them. Checking if + /// a cached key exists and matches a given saves a call + /// to the external key resolver implementation when available. + /// + private readonly IKeyEncryptionKey _potentialCachedIKeyEncryptionKey; + + /// + /// Resolver to fetch the key encryption key. + /// + private readonly IKeyEncryptionKeyResolver _keyResolver; + + public ClientSideDecryptor(ClientSideEncryptionOptions options) + { + _potentialCachedIKeyEncryptionKey = options.KeyEncryptionKey; + _keyResolver = options.KeyResolver; + } + /// /// Decrypts the given stream if decryption information is provided. /// Does not shave off unwanted start/end bytes, but will shave off padding. @@ -26,14 +44,6 @@ internal static class ClientSideDecryptor /// . Generally for partial blob downloads where the /// previous block of the ciphertext is the IV for the next. /// - /// - /// Resolver to fetch the key encryption key. - /// - /// - /// Clients that can upload data have a key encryption key stored on them. Checking if - /// a cached key exists and matches the saves a call - /// to the external key resolver implementation when available. - /// /// /// Whether to ignore padding. Generally for partial blob downloads where the end of /// the blob (where the padding occurs) was not downloaded. @@ -47,12 +57,10 @@ internal static class ClientSideDecryptor /// Exceptions thrown based on implementations of and /// . /// - public static async Task DecryptInternal( + public async Task DecryptInternal( Stream ciphertext, EncryptionData encryptionData, bool ivInStream, - IKeyEncryptionKeyResolver keyResolver, - IKeyEncryptionKey potentialCachedKeyWrapper, bool noPadding, bool async, CancellationToken cancellationToken) @@ -60,7 +68,13 @@ public static async Task DecryptInternal( switch (encryptionData.EncryptionAgent.Protocol) { case ClientSideEncryptionVersion.V1_0: - return await DecryptInternalV1_0(ciphertext, encryptionData, ivInStream, keyResolver, potentialCachedKeyWrapper, noPadding, async, cancellationToken).ConfigureAwait(false); + return await DecryptInternalV1_0( + ciphertext, + encryptionData, + ivInStream, + noPadding, + async, + cancellationToken).ConfigureAwait(false); default: throw Errors.ClientSideEncryption.BadEncryptionAgent(encryptionData.EncryptionAgent.Protocol.ToString()); } @@ -79,14 +93,6 @@ public static async Task DecryptInternal( /// . Generally for partial blob downloads where the /// previous block of the ciphertext is the IV for the next. /// - /// - /// Resolver to fetch the key encryption key. - /// - /// - /// Clients that can upload data have a key encryption key stored on them. Checking if - /// a cached key exists and matches the saves a call - /// to the external key resolver implementation when available. - /// /// /// Whether to ignore padding. Generally for partial blob downloads where the end of /// the blob (where the padding occurs) was not downloaded. @@ -100,20 +106,16 @@ public static async Task DecryptInternal( /// Exceptions thrown based on implementations of and /// . /// - public static async Task DecryptInternalV1_0( + private async Task DecryptInternalV1_0( Stream ciphertext, EncryptionData encryptionData, bool ivInStream, - IKeyEncryptionKeyResolver keyResolver, - IKeyEncryptionKey potentialCachedKeyWrapper, bool noPadding, bool async, CancellationToken cancellationToken) { var contentEncryptionKey = await GetContentEncryptionKeyAsync( encryptionData, - keyResolver, - potentialCachedKeyWrapper, async, cancellationToken).ConfigureAwait(false); @@ -162,8 +164,6 @@ public static async Task DecryptInternalV1_0( /// correct key wrapper. /// /// The encryption data. - /// - /// /// Whether to perform asynchronously. /// /// @@ -173,27 +173,25 @@ public static async Task DecryptInternalV1_0( /// Exceptions thrown based on implementations of and /// . /// - private static async Task> GetContentEncryptionKeyAsync( + private async Task> GetContentEncryptionKeyAsync( #pragma warning restore CS1587 // XML comment is not placed on a valid language element EncryptionData encryptionData, - IKeyEncryptionKeyResolver keyResolver, - IKeyEncryptionKey potentiallyCachedKeyWrapper, bool async, CancellationToken cancellationToken) { IKeyEncryptionKey key = default; // If we already have a local key and it is the correct one, use that. - if (encryptionData.WrappedContentKey.KeyId == potentiallyCachedKeyWrapper?.KeyId) + if (encryptionData.WrappedContentKey.KeyId == _potentialCachedIKeyEncryptionKey?.KeyId) { - key = potentiallyCachedKeyWrapper; + key = _potentialCachedIKeyEncryptionKey; } // Otherwise, use the resolver. - else if (keyResolver != null) + else if (_keyResolver != null) { key = async - ? await keyResolver.ResolveAsync(encryptionData.WrappedContentKey.KeyId, cancellationToken).ConfigureAwait(false) - : keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); + ? await _keyResolver.ResolveAsync(encryptionData.WrappedContentKey.KeyId, cancellationToken).ConfigureAwait(false) + : _keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); } // We throw for every other reason that decryption couldn't happen. Throw a reasonable diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs index 1a85cbff72f7b..d4fbcfdecb5e4 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs @@ -10,25 +10,35 @@ namespace Azure.Storage.Cryptography { - internal static class ClientSideEncryptor + internal class ClientSideEncryptor { + private readonly IKeyEncryptionKey _keyEncryptionKey; + private readonly string _keyWrapAlgorithm; + + public ClientSideEncryptor(ClientSideEncryptionOptions options) + { + _keyEncryptionKey = options.KeyEncryptionKey; + _keyWrapAlgorithm = options.KeyWrapAlgorithm; + } + /// /// Wraps the given read-stream in a CryptoStream and provides the metadata used to create /// that stream. /// /// Stream to wrap. - /// Key encryption key (KEK). - /// Algorithm to encrypt the content encryption key (CEK) with. - /// Whether to wrap the CEK asynchronously. + /// Whether to wrap the content encryption key asynchronously. /// Cancellation token. /// The wrapped stream to read from and the encryption metadata for the wrapped stream. - public static async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( + public async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( Stream plaintext, - IKeyEncryptionKey keyWrapper, - string keyWrapAlgorithm, bool async, CancellationToken cancellationToken) { + if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); + } + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); EncryptionData encryptionData = default; Stream ciphertext = default; @@ -37,9 +47,9 @@ internal static class ClientSideEncryptor { encryptionData = await EncryptionData.CreateInternalV1_0( contentEncryptionIv: aesProvider.IV, - keyWrapAlgorithm: keyWrapAlgorithm, + keyWrapAlgorithm: _keyWrapAlgorithm, contentEncryptionKey: generatedKey, - keyEncryptionKey: keyWrapper, + keyEncryptionKey: _keyEncryptionKey, async: async, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -57,18 +67,19 @@ internal static class ClientSideEncryptor /// optimized for known-size data that will already be buffered in memory. /// /// Stream to encrypt. - /// Key encryption key (KEK). - /// Algorithm to encrypt the content encryption key (CEK) with. - /// Whether to wrap the CEK asynchronously. + /// Whether to wrap the content encryption key asynchronously. /// Cancellation token. /// The encrypted data and the encryption metadata for the wrapped stream. - public static async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( + public async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( Stream plaintext, - IKeyEncryptionKey keyWrapper, - string keyWrapAlgorithm, bool async, CancellationToken cancellationToken) { + if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); + } + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); EncryptionData encryptionData = default; var ciphertext = new MemoryStream(); @@ -78,9 +89,9 @@ internal static class ClientSideEncryptor { encryptionData = await EncryptionData.CreateInternalV1_0( contentEncryptionIv: aesProvider.IV, - keyWrapAlgorithm: keyWrapAlgorithm, + keyWrapAlgorithm: _keyWrapAlgorithm, contentEncryptionKey: generatedKey, - keyEncryptionKey: keyWrapper, + keyEncryptionKey: _keyEncryptionKey, async: async, cancellationToken: cancellationToken).ConfigureAwait(false); diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs index 7302bfa066d87..447aa5cd00c63 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -27,7 +27,7 @@ public static string Serialize(EncryptionData data) /// /// Data to serialize. /// The JSON UTF8 bytes. - public static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) + private static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) { var writer = new Core.ArrayBufferWriter(); using var json = new Utf8JsonWriter(writer); diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index c652df96e5c0c..26698bf8276e8 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -1512,7 +1512,7 @@ private async Task> SendMessageInternal( try { messageText = UsingClientSideEncryption - ? await new QueueClientSideEncryptor(ClientSideEncryption) + ? await new QueueClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) : messageText; @@ -1710,7 +1710,7 @@ private async Task> ReceiveMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await new QueueClientSideDecryptor(ClientSideEncryption, OnClientSideDecryptionFailure) + await new QueueClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption), OnClientSideDecryptionFailure) .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } @@ -1829,7 +1829,7 @@ private async Task> PeekMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await new QueueClientSideDecryptor(ClientSideEncryption, OnClientSideDecryptionFailure) + await new QueueClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption), OnClientSideDecryptionFailure) .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs index fcc8b04524fba..7c189f7488213 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs @@ -7,7 +7,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Azure.Core.Cryptography; using Azure.Storage.Cryptography; using Azure.Storage.Queues.Models; using Azure.Storage.Queues.Specialized; @@ -17,14 +16,12 @@ namespace Azure.Storage.Queues { internal class QueueClientSideDecryptor { - private readonly IKeyEncryptionKeyResolver _resolver; - private readonly IKeyEncryptionKey _cachedIKey; + private readonly ClientSideDecryptor _decryptor; private readonly IClientSideDecryptionFailureListener _listener; - public QueueClientSideDecryptor(ClientSideEncryptionOptions options, IClientSideDecryptionFailureListener listener) + public QueueClientSideDecryptor(ClientSideDecryptor decryptor, IClientSideDecryptionFailureListener listener) { - _resolver = options.KeyResolver; - _cachedIKey = options.KeyEncryptionKey; + _decryptor = decryptor; _listener = listener; } @@ -85,12 +82,10 @@ private async Task ClientSideDecryptInternal(string downloadedMessage, b } var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); - var decryptedMessageStream = await ClientSideDecryptor.DecryptInternal( + var decryptedMessageStream = await _decryptor.DecryptInternal( encryptedMessageStream, encryptedMessage.EncryptionData, ivInStream: false, - _resolver, - _cachedIKey, noPadding: false, async: async, cancellationToken).ConfigureAwait(false); diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs index 8b417ee962034..e79ae7a1fb49d 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Azure.Core.Cryptography; using Azure.Storage.Cryptography; using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Specialized.Models; @@ -15,22 +14,18 @@ namespace Azure.Storage.Queues { internal class QueueClientSideEncryptor { - private readonly IKeyEncryptionKey _keyEncryptionKey; - private readonly string _keyWrapAlgorithm; + private readonly ClientSideEncryptor _encryptor; - public QueueClientSideEncryptor(ClientSideEncryptionOptions options) + public QueueClientSideEncryptor(ClientSideEncryptor encryptor) { - _keyEncryptionKey = options.KeyEncryptionKey; - _keyWrapAlgorithm = options.KeyWrapAlgorithm; + _encryptor = encryptor; } public async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) { var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); - (byte[] ciphertext, EncryptionData encryptionData) = await ClientSideEncryptor.BufferedEncryptInternal( + (byte[] ciphertext, EncryptionData encryptionData) = await _encryptor.BufferedEncryptInternal( new MemoryStream(bytesToEncrypt), - _keyEncryptionKey, - _keyWrapAlgorithm, async, cancellationToken).ConfigureAwait(false); diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs index 99aa74b18bbee..228df281462d5 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -63,10 +63,12 @@ private static byte[] Xor(byte[] a, byte[] b) [Test] public void SerializeEncryptedMessage() { - var result = ClientSideEncryptor.BufferedEncryptInternal( + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - GetIKeyEncryptionKey().Object, - KeyWrapAlgorithm, async: false, default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() @@ -83,10 +85,12 @@ public void SerializeEncryptedMessage() [Test] public void DeserializeEncryptedMessage() { - var result = ClientSideEncryptor.BufferedEncryptInternal( + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - GetIKeyEncryptionKey().Object, - KeyWrapAlgorithm, async: false, default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() @@ -104,10 +108,12 @@ public void DeserializeEncryptedMessage() [Test] public void TryDeserializeEncryptedMessage() { - var result = ClientSideEncryptor.BufferedEncryptInternal( + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), - GetIKeyEncryptionKey().Object, - KeyWrapAlgorithm, async: false, default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() From 55ab4cc42f27519b0979f65c0713797ff0debb1a Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Tue, 2 Jun 2020 16:02:06 -0700 Subject: [PATCH 14/21] Revert deletion of an api.cs file --- ...crosoft.Extensions.Azure.netstandard2.0.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs diff --git a/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs b/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs new file mode 100644 index 0000000000000..a9be4f097bab5 --- /dev/null +++ b/sdk/core/Microsoft.Extensions.Azure/api/Microsoft.Extensions.Azure.netstandard2.0.cs @@ -0,0 +1,33 @@ +namespace Microsoft.Extensions.Azure +{ + public static partial class AzureClientBuilderExtensions + { + public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, Microsoft.Extensions.Configuration.IConfiguration configuration) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Action configureOptions) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder ConfigureOptions(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Action configureOptions) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder WithCredential(this Azure.Core.Extensions.IAzureClientBuilder builder, Azure.Core.TokenCredential credential) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder WithCredential(this Azure.Core.Extensions.IAzureClientBuilder builder, System.Func credentialFactory) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder WithName(this Azure.Core.Extensions.IAzureClientBuilder builder, string name) where TOptions : class { throw null; } + public static Azure.Core.Extensions.IAzureClientBuilder WithVersion(this Azure.Core.Extensions.IAzureClientBuilder builder, TVersion version) where TOptions : class { throw null; } + } + public sealed partial class AzureClientFactoryBuilder : Azure.Core.Extensions.IAzureClientFactoryBuilder, Azure.Core.Extensions.IAzureClientFactoryBuilderWithConfiguration, Azure.Core.Extensions.IAzureClientFactoryBuilderWithCredential + { + internal AzureClientFactoryBuilder() { } + Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilder.RegisterClientFactory(System.Func clientFactory) { throw null; } + Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilderWithConfiguration.RegisterClientFactory(Microsoft.Extensions.Configuration.IConfiguration configuration) { throw null; } + Azure.Core.Extensions.IAzureClientBuilder Azure.Core.Extensions.IAzureClientFactoryBuilderWithCredential.RegisterClientFactory(System.Func clientFactory, bool requiresCredential) { throw null; } + public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(Microsoft.Extensions.Configuration.IConfiguration configuration) { throw null; } + public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(System.Action configureOptions) { throw null; } + public Microsoft.Extensions.Azure.AzureClientFactoryBuilder ConfigureDefaults(System.Action configureOptions) { throw null; } + public Microsoft.Extensions.Azure.AzureClientFactoryBuilder UseCredential(Azure.Core.TokenCredential tokenCredential) { throw null; } + public Microsoft.Extensions.Azure.AzureClientFactoryBuilder UseCredential(System.Func tokenCredentialFactory) { throw null; } + } + public static partial class AzureClientServiceCollectionExtensions + { + public static void AddAzureClients(this Microsoft.Extensions.DependencyInjection.IServiceCollection collection, System.Action configureClients) { } + } + public partial interface IAzureClientFactory + { + TClient CreateClient(string name); + } +} From 9b8b321f2bc5833421f693513e9e514fe62f5507 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Thu, 4 Jun 2020 12:19:12 -0700 Subject: [PATCH 15/21] Minor Crypto API Adjustments --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 17 ++++++++ .../Azure.Storage.Blobs/src/BlobClient.cs | 2 + .../src/BlobClientExtensions.cs | 28 ------------ ...ons.cs => SpecializedBlobClientOptions.cs} | 4 +- .../tests/ClientSideEncryptionTests.cs | 4 +- .../Azure.Storage.Queues/src/QueueClient.cs | 43 +++++++++++++++++++ .../src/QueueClientExtensions.cs | 43 ------------------- ...ns.cs => SpecializedQueueClientOptions.cs} | 4 +- .../tests/ClientSideEncryptionTests.cs | 2 +- 9 files changed, 69 insertions(+), 78 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs rename sdk/storage/Azure.Storage.Blobs/src/{ExtendedBlobClientOptions.cs => SpecializedBlobClientOptions.cs} (89%) delete mode 100644 sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs rename sdk/storage/Azure.Storage.Queues/src/{ExtendedQueueClientOptions.cs => SpecializedQueueClientOptions.cs} (95%) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 8647cad8a6587..6e3b435139cdd 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -3012,5 +3012,22 @@ public static BlobBaseClient GetBlobBaseClient( client.CustomerProvidedKey, client.ClientSideEncryption, client.EncryptionScope); + + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// New instance with provided options and same internals otherwise. + public static BlobClient WithClientSideEncryptionOptions(this BlobClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) + => new BlobClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + client.CustomerProvidedKey, + clientSideEncryptionOptions, + client.EncryptionScope); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index ae5a8a2228db9..f3f09e6f8712a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -12,6 +12,8 @@ using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; +#pragma warning disable SA1402 // File may only contain a single type + namespace Azure.Storage.Blobs { /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs deleted file mode 100644 index a515bbea3f195..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -namespace Azure.Storage.Blobs.Specialized -{ - /// - /// Provides advanced extensions for blob service clients. - /// - public static class BlobClientExtensions - { - /// - /// Creates a new instance of the class, maintaining all the same - /// internals but specifying new . - /// - /// Client to base off of. - /// New encryption options. Setting this to default will clear client-side encryption. - /// New instance with provided options and same internals otherwise. - public static BlobClient WithClientSideEncryptionOptions(this BlobClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) - => new BlobClient( - client.Uri, - client.Pipeline, - client.Version, - client.ClientDiagnostics, - client.CustomerProvidedKey, - clientSideEncryptionOptions, - client.EncryptionScope); - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs similarity index 89% rename from sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs rename to sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs index 55f4f30fa916f..a52cc59740bc1 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/ExtendedBlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs @@ -8,7 +8,7 @@ namespace Azure.Storage.Blobs.Specialized /// Storage. /// #pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. - public class ExtendedBlobClientOptions : BlobClientOptions + public class SpecializedBlobClientOptions : BlobClientOptions #pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion { /// @@ -19,7 +19,7 @@ public class ExtendedBlobClientOptions : BlobClientOptions /// The of the service API used when /// making requests. /// - public ExtendedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) + public SpecializedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) { } diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs index 1ae5e16e304c1..d4943570dcb94 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -222,7 +222,7 @@ public void CanSwapKey() KeyWrapAlgorithm = "bar" }; - var client = new BlobClient(new Uri("http://someuri.com"), new ExtendedBlobClientOptions() + var client = new BlobClient(new Uri("http://someuri.com"), new SpecializedBlobClientOptions() { ClientSideEncryption = options1, }); @@ -600,7 +600,7 @@ public async Task AppropriateRangeDownloadOnPlaintext(int rangeOffset, int? rang await blob.UploadAsync(new MemoryStream(data)); // download plaintext range with encrypted client - var cryptoClient = InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), new ExtendedBlobClientOptions() + var cryptoClient = InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), new SpecializedBlobClientOptions() { ClientSideEncryption = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index 26698bf8276e8..e5fe4fc47527a 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -18,6 +18,8 @@ using Azure.Storage.Queues.Specialized.Models; using Metadata = System.Collections.Generic.IDictionary; +#pragma warning disable SA1402 // File may only contain a single type + namespace Azure.Storage.Queues { /// @@ -2113,3 +2115,44 @@ private async Task> UpdateMessageInternal( #endregion UpdateMessage } } + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Add methods to clients. + /// + public static partial class SpecializedQueueExtensions + { + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// New instance with provided options and same internals otherwise. + public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) + => new QueueClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + clientSideEncryptionOptions, + client.OnClientSideDecryptionFailure); + + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// Listener for when decryption of a single message fails. + /// New instance with provided options and same internals otherwise. + public static QueueClient WithClientSideEncryptionFailureListener(this QueueClient client, IClientSideDecryptionFailureListener listener) + => new QueueClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + client.ClientSideEncryption, + listener); + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs deleted file mode 100644 index 44646a24ddc05..0000000000000 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientExtensions.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -namespace Azure.Storage.Queues.Specialized -{ - /// - /// Provides advanced extensions for queue service clients. - /// - public static class QueueClientExtensions - { - /// - /// Creates a new instance of the class, maintaining all the same - /// internals but specifying new . - /// - /// Client to base off of. - /// New encryption options. Setting this to default will clear client-side encryption. - /// New instance with provided options and same internals otherwise. - public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) - => new QueueClient( - client.Uri, - client.Pipeline, - client.Version, - client.ClientDiagnostics, - clientSideEncryptionOptions, - client.OnClientSideDecryptionFailure); - - /// - /// Creates a new instance of the class, maintaining all the same - /// internals but specifying new . - /// - /// Client to base off of. - /// Listener for when decryption of a single message fails. - /// New instance with provided options and same internals otherwise. - public static QueueClient WithClientSideEncryptionFailureListener(this QueueClient client, IClientSideDecryptionFailureListener listener) - => new QueueClient( - client.Uri, - client.Pipeline, - client.Version, - client.ClientDiagnostics, - client.ClientSideEncryption, - listener); - } -} diff --git a/sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs similarity index 95% rename from sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs rename to sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs index 5c9ff5ac46512..38e89d3f196e2 100644 --- a/sdk/storage/Azure.Storage.Queues/src/ExtendedQueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs @@ -12,7 +12,7 @@ namespace Azure.Storage.Queues.Specialized /// Storage. /// #pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. - public class ExtendedQueueClientOptions : QueueClientOptions + public class SpecializedQueueClientOptions : QueueClientOptions #pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion { /// @@ -23,7 +23,7 @@ public class ExtendedQueueClientOptions : QueueClientOptions /// The of the service API used when /// making requests. /// - public ExtendedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) + public SpecializedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) { } diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index dfeb6a3f21cff..d002cf5154e90 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -268,7 +268,7 @@ public void CanSwapKey() var listener1 = GetFailureListener().Object; var listener2 = GetFailureListener().Object; - var client = new QueueClient(new Uri("http://someuri.com"), new ExtendedQueueClientOptions() + var client = new QueueClient(new Uri("http://someuri.com"), new SpecializedQueueClientOptions() { ClientSideEncryption = options1, OnClientSideDecryptionFailure = listener1 From 5d2220d9732cb200931bb883db20e751391f9077 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Thu, 4 Jun 2020 13:01:23 -0700 Subject: [PATCH 16/21] Export API --- .../api/Azure.Storage.Blobs.netstandard2.0.cs | 15 ++++++--------- .../api/Azure.Storage.Queues.netstandard2.0.cs | 14 +++++++------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 2ce20f1e07850..1bfb04906e8cf 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -989,10 +989,6 @@ public BlobBaseClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti public virtual Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshot(string snapshot) { throw null; } protected virtual Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } - public static partial class BlobClientExtensions - { - public static Azure.Storage.Blobs.BlobClient WithClientSideEncryptionOptions(this Azure.Storage.Blobs.BlobClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } - } public partial class BlobLeaseClient { public static readonly System.TimeSpan InfiniteLeaseDuration; @@ -1041,11 +1037,6 @@ public BlockBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredent public new Azure.Storage.Blobs.Specialized.BlockBlobClient WithSnapshot(string snapshot) { throw null; } protected sealed override Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } - public partial class ExtendedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions - { - public ExtendedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } - public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } - } public partial class PageBlobClient : Azure.Storage.Blobs.Specialized.BlobBaseClient { protected PageBlobClient() { } @@ -1081,6 +1072,11 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti public new Azure.Storage.Blobs.Specialized.PageBlobClient WithSnapshot(string snapshot) { throw null; } protected sealed override Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } + public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions + { + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } public static partial class SpecializedBlobExtensions { public static Azure.Storage.Blobs.Specialized.AppendBlobClient GetAppendBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } @@ -1089,6 +1085,7 @@ public static partial class SpecializedBlobExtensions public static Azure.Storage.Blobs.Specialized.BlobLeaseClient GetBlobLeaseClient(this Azure.Storage.Blobs.Specialized.BlobBaseClient client, string leaseId = null) { throw null; } public static Azure.Storage.Blobs.Specialized.BlockBlobClient GetBlockBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } public static Azure.Storage.Blobs.Specialized.PageBlobClient GetPageBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } + public static Azure.Storage.Blobs.BlobClient WithClientSideEncryptionOptions(this Azure.Storage.Blobs.BlobClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } } } namespace Azure.Storage.Sas diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index 6f0f477adf06e..c89ff4f806e55 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -317,12 +317,6 @@ internal UpdateReceipt() { } } namespace Azure.Storage.Queues.Specialized { - public partial class ExtendedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions - { - public ExtendedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } - public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } - public Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener OnClientSideDecryptionFailure { get { throw null; } set { } } - } public partial interface IClientSideDecryptionFailureListener { void OnFailure(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); @@ -330,7 +324,13 @@ public partial interface IClientSideDecryptionFailureListener System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.QueueMessage message, System.Exception exception); } - public static partial class QueueClientExtensions + public partial class SpecializedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions + { + public SpecializedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + public Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener OnClientSideDecryptionFailure { get { throw null; } set { } } + } + public static partial class SpecializedQueueExtensions { public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionFailureListener(this Azure.Storage.Queues.QueueClient client, Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener listener) { throw null; } public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } From b50607185a424d05cd28fe80279ca9b0b7bcb8e6 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 5 Jun 2020 13:45:10 -0700 Subject: [PATCH 17/21] Queue listener changed to events. Other minor PR feedback. --- .../src/AppendBlobClient.cs | 2 +- .../src/Azure.Storage.Blobs.csproj | 1 - .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 2 +- .../src/BlobClientSideDecryptor.cs | 25 ++-- .../src/ClientSideEncryptionVersion.cs | 2 +- .../src/ClientsideEncryptionOptions.cs | 7 +- .../ClientSideDecryptor.cs | 4 +- .../ClientSideEncryptionOptionsExtensions.cs | 36 +++++- .../Models/EncryptionAgent.cs | 2 +- .../Models/EncryptionData.cs | 2 +- .../Models/EncryptionDataSerializer.cs | 8 +- .../src/Models/EncryptedMessage.cs | 2 +- .../src/Models/EncryptedMessageSerializer.cs | 8 +- .../Azure.Storage.Queues/src/QueueClient.cs | 59 +++------- .../src/QueueClientOptions.cs | 1 - .../src/QueueClientSideDecryptor.cs | 32 ++--- .../src/QueueClientSideEncryptionOptions.cs | 104 ++++++++++++++++ .../src/QueueClientSideEncryptor.cs | 2 +- .../src/QueueServiceClient.cs | 10 +- .../src/SpecializedQueueClientOptions.cs | 46 -------- .../tests/ClientSideEncryptionTests.cs | 111 ++++++------------ .../tests/EncryptedMessageSerializerTests.cs | 10 +- 22 files changed, 246 insertions(+), 230 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index cfe4277de7f5f..49332c5bda855 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -209,7 +209,7 @@ internal AppendBlobClient( private static void AssertNoClientSideEncryption(BlobClientOptions options) { - if (options._clientSideEncryptionOptions != default) + if (options?._clientSideEncryptionOptions != default) { throw Errors.ClientSideEncryption.TypeNotSupported(typeof(AppendBlobClient)); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index fa73decae81f6..e92aad6a91721 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -40,7 +40,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 6e3b435139cdd..3569b8edd874a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -714,7 +714,7 @@ private async Task> DownloadInternal( if (UsingClientSideEncryption) { stream = await new BlobClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption)) - .ClientSideDecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); + .DecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); } response.Value.Content = stream; diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs index 03bfebcc98513..2fe25686d5ea5 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -21,7 +21,7 @@ public BlobClientSideDecryptor(ClientSideDecryptor decryptor) _decryptor = decryptor; } - public async Task ClientSideDecryptInternal( + public async Task DecryptInternal( Stream content, Metadata metadata, HttpRange originalRange, @@ -36,7 +36,7 @@ public async Task ClientSideDecryptInternal( EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata); if (encryptionData == default) { - return await TrimStreamInternal(content, originalRange, contentRange, pulledOutIv: false, async, cancellationToken).ConfigureAwait(false); + return await TrimStreamInternal(content, originalRange, contentRange, pulledOutIV: false, async, cancellationToken).ConfigureAwait(false); } bool ivInStream = originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize; @@ -53,22 +53,31 @@ public async Task ClientSideDecryptInternal( return await TrimStreamInternal(plaintext, originalRange, contentRange, ivInStream, async, cancellationToken).ConfigureAwait(false); } - private static async Task TrimStreamInternal(Stream stream, HttpRange originalRange, ContentRange? receivedRange, bool pulledOutIv, bool async, CancellationToken cancellationToken) + private static async Task TrimStreamInternal( + Stream stream, + HttpRange originalRange, + ContentRange? receivedRange, + bool pulledOutIV, + bool async, + CancellationToken cancellationToken) { // retrim start of stream to original requested location // keeping in mind whether we already pulled the IV out of the stream as well int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0)) - - (pulledOutIv ? Constants.ClientSideEncryption.EncryptionBlockSize : 0); - if (gap > 0) + - (pulledOutIV ? Constants.ClientSideEncryption.EncryptionBlockSize : 0); + + int read = 0; + while (gap > read) { + int toRead = gap - read; // throw away initial bytes we want to trim off; stream cannot seek into future if (async) { - await stream.ReadAsync(new byte[gap], 0, gap, cancellationToken).ConfigureAwait(false); + read += await stream.ReadAsync(new byte[toRead], 0, toRead, cancellationToken).ConfigureAwait(false); } else { - stream.Read(new byte[gap], 0, gap); + read += stream.Read(new byte[toRead], 0, toRead); } } @@ -112,7 +121,7 @@ private static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata met private static bool CanIgnorePadding(ContentRange? contentRange) { // if Content-Range not present, we requested the whole blob - if (!contentRange.HasValue) + if (contentRange == null) { return false; } diff --git a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs index b209b1ff28bde..f2d4725eb050d 100644 --- a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs +++ b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs @@ -12,7 +12,7 @@ public enum ClientSideEncryptionVersion /// /// 1.0 /// - V1_0 + V1_0 = 1 } #pragma warning restore CA1707 // Identifiers should not contain underscores } diff --git a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs index 6e50fd7a5d0ba..3d03a53d14927 100644 --- a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs +++ b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs @@ -11,10 +11,13 @@ namespace Azure.Storage /// public class ClientSideEncryptionOptions { + // NOTE there is a non-public Clone method for this class, compile-included into relevant packages from + // Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs + /// /// The version of clientside encryption to use. /// - public ClientSideEncryptionVersion Version { get; } + public ClientSideEncryptionVersion EncryptionVersion { get; } /// /// Required for upload operations. @@ -44,7 +47,7 @@ public class ClientSideEncryptionOptions /// The version of clientside encryption to use. public ClientSideEncryptionOptions(ClientSideEncryptionVersion version) { - Version = version; + EncryptionVersion = version; } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs index 270d1d02d577a..6c919630f84f5 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -65,7 +65,7 @@ public async Task DecryptInternal( bool async, CancellationToken cancellationToken) { - switch (encryptionData.EncryptionAgent.Protocol) + switch (encryptionData.EncryptionAgent.EncryptionVersion) { case ClientSideEncryptionVersion.V1_0: return await DecryptInternalV1_0( @@ -76,7 +76,7 @@ public async Task DecryptInternal( async, cancellationToken).ConfigureAwait(false); default: - throw Errors.ClientSideEncryption.BadEncryptionAgent(encryptionData.EncryptionAgent.Protocol.ToString()); + throw Errors.ClientSideEncryption.BadEncryptionAgent(encryptionData.EncryptionAgent.EncryptionVersion.ToString()); } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs index 7fe2b3bc5dfbd..ed09ed9c2da87 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs @@ -5,12 +5,36 @@ namespace Azure.Storage.Cryptography { internal static class ClientSideEncryptionOptionsExtensions { + /// + /// Extension method to clone an instance of . + /// + /// + /// public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions options) - => new ClientSideEncryptionOptions(options.Version) - { - KeyEncryptionKey = options.KeyEncryptionKey, - KeyResolver = options.KeyResolver, - KeyWrapAlgorithm = options.KeyWrapAlgorithm, - }; + { + var newOptions = new ClientSideEncryptionOptions(options.EncryptionVersion); + CopyOptions(options, newOptions); + return newOptions; + } + + /// + /// Copies all properties from one instance to another. It cannot copy + /// ; + /// that is the responsibility of the caller who made the instance. + /// + /// Object to copy from. + /// Object to copy to. + /// + /// This functionality has been pulled out to be accessible by other + /// clone methods available on subclasses. They need the ability to + /// instantiate the subclass destination first before copying over the + /// properties. + /// + internal static void CopyOptions(ClientSideEncryptionOptions source, ClientSideEncryptionOptions destination) + { + destination.KeyEncryptionKey = source.KeyEncryptionKey; + destination.KeyResolver = source.KeyResolver; + destination.KeyWrapAlgorithm = source.KeyWrapAlgorithm; + } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs index 347a3e8ff304a..c33ab8407c85c 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs @@ -11,7 +11,7 @@ internal class EncryptionAgent /// /// The protocol version used for encryption. /// - public ClientSideEncryptionVersion Protocol { get; set; } + public ClientSideEncryptionVersion EncryptionVersion { get; set; } /// /// The algorithm used for encryption. diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs index c60cadf21d57a..ead5f7c6d42c6 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -58,7 +58,7 @@ internal static async Task CreateInternalV1_0( EncryptionAgent = new EncryptionAgent() { EncryptionAlgorithm = ClientSideEncryptionAlgorithm.AesCbc256, - Protocol = ClientSideEncryptionVersion.V1_0 + EncryptionVersion = ClientSideEncryptionVersion.V1_0 }, KeyWrappingMetadata = new Dictionary() { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs index 447aa5cd00c63..96ed771efc32c 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -10,6 +10,8 @@ namespace Azure.Storage.Cryptography.Models { internal static class EncryptionDataSerializer { + private const string EncryptionAgent_EncryptionVersionName = "Protocol"; + #region Serialize /// @@ -73,7 +75,7 @@ private static void WriteWrappedKey(Utf8JsonWriter json, KeyEnvelope key) private static void WriteEncryptionAgent(Utf8JsonWriter json, EncryptionAgent encryptionAgent) { - json.WriteString(nameof(encryptionAgent.Protocol), encryptionAgent.Protocol.Serialize()); + json.WriteString(EncryptionAgent_EncryptionVersionName, encryptionAgent.EncryptionVersion.Serialize()); json.WriteString(nameof(encryptionAgent.EncryptionAlgorithm), encryptionAgent.EncryptionAlgorithm.ToString()); } @@ -186,9 +188,9 @@ private static void ReadPropertyValue(EncryptionAgent agent, JsonProperty proper { agent.EncryptionAlgorithm = new ClientSideEncryptionAlgorithm(property.Value.GetString()); } - else if (property.NameEquals(nameof(agent.Protocol))) + else if (property.NameEquals(EncryptionAgent_EncryptionVersionName)) { - agent.Protocol = property.Value.GetString().ToClientSideEncryptionVersion(); + agent.EncryptionVersion = property.Value.GetString().ToClientSideEncryptionVersion(); } } #endregion diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs index 79ed4f5dde82f..957943aecec84 100644 --- a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs @@ -7,7 +7,7 @@ namespace Azure.Storage.Queues.Specialized.Models { internal class EncryptedMessage { - public string EncryptedMessageContents { get; set; } + public string EncryptedMessageText { get; set; } public EncryptionData EncryptionData { get; set; } } diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs index 84a9eb6b2e948..46990350d57aa 100644 --- a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs @@ -10,6 +10,8 @@ namespace Azure.Storage.Queues.Specialized.Models { internal static class EncryptedMessageSerializer { + private const string EncryptedMessage_EncryptedMessageTextName = "EncryptedMessageContents"; + #region Serialize public static string Serialize(EncryptedMessage data) { @@ -31,7 +33,7 @@ public static ReadOnlyMemory SerializeEncryptedMessage(EncryptedMessage me public static void WriteEncryptedMessage(Utf8JsonWriter json, EncryptedMessage message) { - json.WriteString(nameof(message.EncryptedMessageContents), message.EncryptedMessageContents); + json.WriteString(EncryptedMessage_EncryptedMessageTextName, message.EncryptedMessageText); json.WriteStartObject(nameof(message.EncryptionData)); EncryptionDataSerializer.WriteEncryptionData(json, message.EncryptionData); @@ -81,9 +83,9 @@ private static EncryptedMessage ReadEncryptionData(JsonElement root) private static void ReadPropertyValue(EncryptedMessage data, JsonProperty property) { - if (property.NameEquals(nameof(data.EncryptedMessageContents))) + if (property.NameEquals(EncryptedMessage_EncryptedMessageTextName)) { - data.EncryptedMessageContents = property.Value.GetString(); + data.EncryptedMessageText = property.Value.GetString(); } else if (property.NameEquals(nameof(data.EncryptionData))) { diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index e5fe4fc47527a..fa9d96759ec05 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -4,18 +4,14 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Cryptography; -using Azure.Storage.Cryptography.Models; using Azure.Storage.Queues.Models; using Azure.Storage.Queues.Specialized; -using Azure.Storage.Queues.Specialized.Models; using Metadata = System.Collections.Generic.IDictionary; #pragma warning disable SA1402 // File may only contain a single type @@ -80,21 +76,17 @@ public class QueueClient internal virtual ClientDiagnostics ClientDiagnostics => _clientDiagnostics; /// - /// The to be used when sending/receiving requests. + /// The to be used when sending/receiving requests. /// - private readonly ClientSideEncryptionOptions _clientSideEncryption; + private readonly QueueClientSideEncryptionOptions _clientSideEncryption; /// - /// The to be used when sending/receiving requests. + /// The to be used when sending/receiving requests. /// - internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + internal virtual QueueClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; internal bool UsingClientSideEncryption => ClientSideEncryption != default; - private readonly IClientSideDecryptionFailureListener _onClientSideDecryptionFailure; - - internal virtual IClientSideDecryptionFailureListener OnClientSideDecryptionFailure => _onClientSideDecryptionFailure; - /// /// QueueMaxMessagesPeek indicates the maximum number of messages /// you can retrieve with each call to Peek. @@ -201,8 +193,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); - _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _onClientSideDecryptionFailure = options._onClientSideDecryptionFailure; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -294,8 +285,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); - _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _onClientSideDecryptionFailure = options._onClientSideDecryptionFailure; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -320,24 +310,19 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien /// /// Options for client-side encryption. /// - /// - /// Listener regarding partial decryption failures using clientside encryption. - /// internal QueueClient( Uri queueUri, HttpPipeline pipeline, QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, - ClientSideEncryptionOptions encryptionOptions, - IClientSideDecryptionFailureListener listener) + ClientSideEncryptionOptions encryptionOptions) { _uri = queueUri; _messagesUri = queueUri.AppendToPath(Constants.Queue.MessagesUri); _pipeline = pipeline; _version = version; _clientDiagnostics = clientDiagnostics; - _clientSideEncryption = encryptionOptions?.Clone(); - _onClientSideDecryptionFailure = listener; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(encryptionOptions); } #endregion ctors @@ -1712,7 +1697,7 @@ private async Task> ReceiveMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await new QueueClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption), OnClientSideDecryptionFailure) + await new QueueClientSideDecryptor(ClientSideEncryption) .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } @@ -1831,7 +1816,7 @@ private async Task> PeekMessagesInternal( else if (UsingClientSideEncryption) { return Response.FromValue( - await new QueueClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption), OnClientSideDecryptionFailure) + await new QueueClientSideDecryptor(ClientSideEncryption) .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), response.GetRawResponse()); } @@ -2088,6 +2073,11 @@ private async Task> UpdateMessageInternal( $"{nameof(visibilityTimeout)}: {visibilityTimeout}"); try { + messageText = UsingClientSideEncryption + ? await new QueueClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) + .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) + : messageText; + return await QueueRestClient.MessageId.UpdateAsync( ClientDiagnostics, Pipeline, @@ -2136,23 +2126,6 @@ public static QueueClient WithClientSideEncryptionOptions(this QueueClient clien client.Pipeline, client.Version, client.ClientDiagnostics, - clientSideEncryptionOptions, - client.OnClientSideDecryptionFailure); - - /// - /// Creates a new instance of the class, maintaining all the same - /// internals but specifying new . - /// - /// Client to base off of. - /// Listener for when decryption of a single message fails. - /// New instance with provided options and same internals otherwise. - public static QueueClient WithClientSideEncryptionFailureListener(this QueueClient client, IClientSideDecryptionFailureListener listener) - => new QueueClient( - client.Uri, - client.Pipeline, - client.Version, - client.ClientDiagnostics, - client.ClientSideEncryption, - listener); + clientSideEncryptionOptions); } } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index 2923b3fc94e38..ab3cdf9194163 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -85,7 +85,6 @@ public QueueClientOptions(ServiceVersion version = LatestVersion) #region Advanced Options internal ClientSideEncryptionOptions _clientSideEncryptionOptions; - internal IClientSideDecryptionFailureListener _onClientSideDecryptionFailure; #endregion /// diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs index 7c189f7488213..1408109e80f26 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs @@ -17,12 +17,12 @@ namespace Azure.Storage.Queues internal class QueueClientSideDecryptor { private readonly ClientSideDecryptor _decryptor; - private readonly IClientSideDecryptionFailureListener _listener; + public QueueClientSideEncryptionOptions Options { get; } - public QueueClientSideDecryptor(ClientSideDecryptor decryptor, IClientSideDecryptionFailureListener listener) + public QueueClientSideDecryptor(QueueClientSideEncryptionOptions options) { - _decryptor = decryptor; - _listener = listener; + _decryptor = new ClientSideDecryptor(options); + Options = options; } public async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) @@ -35,16 +35,9 @@ public async Task ClientSideDecryptMessagesInternal(QueueMessage message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); filteredMessages.Add(message); } - catch (Exception e) when (_listener != default) + catch (Exception e) when (Options.UsingDecryptionFailureHandler) { - if (async) - { - await _listener.OnFailureAsync(message, e).ConfigureAwait(false); - } - else - { - _listener.OnFailure(message, e); - } + Options.OnDecryptionFailed(message, e); } } return filteredMessages.ToArray(); @@ -59,16 +52,9 @@ public async Task ClientSideDecryptMessagesInternal(PeekedMessa message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); filteredMessages.Add(message); } - catch (Exception e) when (_listener != default) + catch (Exception e) when (Options.UsingDecryptionFailureHandler) { - if (async) - { - await _listener.OnFailureAsync(message, e).ConfigureAwait(false); - } - else - { - _listener.OnFailure(message, e); - } + Options.OnDecryptionFailed(message, e); } } return filteredMessages.ToArray(); @@ -81,7 +67,7 @@ private async Task ClientSideDecryptInternal(string downloadedMessage, b return downloadedMessage; // not recognized as client-side encrypted message } - var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageContents)); + var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageText)); var decryptedMessageStream = await _decryptor.DecryptInternal( encryptedMessageStream, encryptedMessage.EncryptionData, diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs new file mode 100644 index 0000000000000..27cbb2cf21f97 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Storage.Cryptography; + +#pragma warning disable SA1402 // File may only contain a single type + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Contains Queues-specific options for client-side encryption. + /// + public class QueueClientSideEncryptionOptions : ClientSideEncryptionOptions + { + /// + /// Initializes a new instance of the class. + /// + /// The version of clientside encryption to use. + public QueueClientSideEncryptionOptions(ClientSideEncryptionVersion version) : base(version) + { + } + + /// + /// Event when failure to decrypt a message occurs. + /// + public event EventHandler DecryptionFailed; + + internal bool UsingDecryptionFailureHandler => DecryptionFailed.GetInvocationList().Length > 0; + + internal void OnDecryptionFailed(object message, Exception e) + { + DecryptionFailed?.Invoke(this, new ClientSideDecryptionFailureEventArgs(message, e)); + } + + /// + /// Clones this class as this subclass instead of the base class. + /// + /// + /// Compiler restriction: can only copy an event in an instance method on the class containing the event. + /// This class exists to allow us to copy the event. + /// + private QueueClientSideEncryptionOptions CloneAsQueueClientSideEncryptionOptions() + { + // clone base class but as instance of this class + var newOptions = new QueueClientSideEncryptionOptions(EncryptionVersion); + ClientSideEncryptionOptionsExtensions.CopyOptions(this, newOptions); + + // clone data specific to this subclass + newOptions.DecryptionFailed = DecryptionFailed; + + return newOptions; + } + + /// + /// Clones the given as an instance of + /// . If the given instance is also a + /// , this clones it's specialty data as well. + /// + /// + internal static QueueClientSideEncryptionOptions CloneFrom(ClientSideEncryptionOptions options) + { + if (options == default) + { + return default; + } + else if (options is QueueClientSideEncryptionOptions) + { + return ((QueueClientSideEncryptionOptions)options).CloneAsQueueClientSideEncryptionOptions(); + } + else + { + // clone base class but as instance of this class + var newOptions = new QueueClientSideEncryptionOptions(options.EncryptionVersion); + ClientSideEncryptionOptionsExtensions.CopyOptions(options, newOptions); + + return newOptions; + } + } + } + + /// + /// Event args for when a queue message decryption fails. + /// + public class ClientSideDecryptionFailureEventArgs + { + /// + /// The exception thrown. + /// + public Exception Exception { get; } + + /// + /// Message the failure occured with. Can be an instance of either + /// or . + /// + public object Message { get; } + + internal ClientSideDecryptionFailureEventArgs(object message, Exception exception) + { + Message = message; + Exception = exception; + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs index e79ae7a1fb49d..1c5917fd8bab6 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs @@ -31,7 +31,7 @@ public async Task ClientSideEncryptInternal(string messageToUpload, bool return EncryptedMessageSerializer.Serialize(new EncryptedMessage { - EncryptedMessageContents = Convert.ToBase64String(ciphertext), + EncryptedMessageText = Convert.ToBase64String(ciphertext), EncryptionData = encryptionData }); } diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs index 107aecc1c4e7d..cf0526a0fc748 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs @@ -71,8 +71,6 @@ public class QueueServiceClient /// internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; - private readonly IClientSideDecryptionFailureListener _missingClientSideEncryptionKeyListener; - /// /// The Storage account name corresponding to the service client. /// @@ -142,8 +140,7 @@ public QueueServiceClient(string connectionString, QueueClientOptions options) _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); - _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -230,8 +227,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); - _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); - _missingClientSideEncryptionKeyListener = options._onClientSideDecryptionFailure; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } #endregion ctors @@ -248,7 +244,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q /// A for the desired queue. /// public virtual QueueClient GetQueueClient(string queueName) - => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption, _missingClientSideEncryptionKeyListener); + => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption); #region GetQueues /// diff --git a/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs index 38e89d3f196e2..a8daa566e1b38 100644 --- a/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs @@ -38,51 +38,5 @@ public ClientSideEncryptionOptions ClientSideEncryption get => _clientSideEncryptionOptions; set => _clientSideEncryptionOptions = value; } - - /// - /// Behavior when receiving a queue message that cannot be decrypted due to lack of key access. - /// Messages in the list of results that cannot be decrypted will be filtered out of the list and - /// sent to this listener. Default behavior, when no listener is provided, is for the overall message - /// fetch to throw. - /// - public IClientSideDecryptionFailureListener OnClientSideDecryptionFailure - { - get => _onClientSideDecryptionFailure; - set => _onClientSideDecryptionFailure = value; - } - } - - /// - /// Describes a listener to handle queue messages who's client-side encryption keys cannot be resolved. - /// - public interface IClientSideDecryptionFailureListener - { - /// - /// Handle a decryption failure in a call. - /// - /// Message that couldn't be decrypted. - /// Exception of the failure. - void OnFailure(QueueMessage message, Exception exception); - - /// - /// Handle a decryption failure in a call. - /// - /// Message that couldn't be decrypted. - /// Exception of the failure. - Task OnFailureAsync(QueueMessage message, Exception exception); - - /// - /// Handle a decryption failure in a call. - /// - /// Message that couldn't be decrypted. - /// Exception of the failure. - void OnFailure(PeekedMessage message, Exception exception); - - /// - /// Handle a decryption failure in a call. - /// - /// Message that couldn't be decrypted. - /// Exception of the failure. - Task OnFailureAsync(PeekedMessage message, Exception exception); } } diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index d002cf5154e90..9a07660289deb 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -230,61 +230,61 @@ private static byte[] Xor(byte[] a, byte[] b) return result; } - private Mock GetFailureListener() + [Test] + [LiveOnly] + public void CanSwapKey() { - var mock = new Mock(MockBehavior.Strict); - if (IsAsync) + int options1EventCalled = 0; + int options2EventCalled = 0; + void Options1_DecryptionFailed(object sender, ClientSideDecryptionFailureEventArgs e) { - mock.Setup(l => l.OnFailureAsync(IsNotNull(), IsNotNull())) - .Returns(Task.CompletedTask); - mock.Setup(l => l.OnFailureAsync(IsNotNull(), IsNotNull())) - .Returns(Task.CompletedTask); + options1EventCalled++; } - else + void Options2_DecryptionFailed(object sender, ClientSideDecryptionFailureEventArgs e) { - mock.Setup(l => l.OnFailure(IsNotNull(), IsNotNull())); - mock.Setup(l => l.OnFailure(IsNotNull(), IsNotNull())); + options2EventCalled++; } - - return mock; - } - - [Test] - [LiveOnly] - public void CanSwapKey() - { - var options1 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + var options1 = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = GetIKeyEncryptionKey().Object, KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, KeyWrapAlgorithm = "foo" }; - var options2 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + options1.DecryptionFailed += Options1_DecryptionFailed; + var options2 = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { KeyEncryptionKey = GetIKeyEncryptionKey().Object, KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, KeyWrapAlgorithm = "bar" }; - var listener1 = GetFailureListener().Object; - var listener2 = GetFailureListener().Object; + options2.DecryptionFailed += Options2_DecryptionFailed; var client = new QueueClient(new Uri("http://someuri.com"), new SpecializedQueueClientOptions() { - ClientSideEncryption = options1, - OnClientSideDecryptionFailure = listener1 + ClientSideEncryption = options1 }); Assert.AreEqual(options1.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); Assert.AreEqual(options1.KeyResolver, client.ClientSideEncryption.KeyResolver); Assert.AreEqual(options1.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); - Assert.AreEqual(listener1, client.OnClientSideDecryptionFailure); - client = client.WithClientSideEncryptionOptions(options2).WithClientSideEncryptionFailureListener(listener2); + Assert.AreEqual(0, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + client.ClientSideEncryption.OnDecryptionFailed(default, default); + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + + client = client.WithClientSideEncryptionOptions(options2); Assert.AreEqual(options2.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); Assert.AreEqual(options2.KeyResolver, client.ClientSideEncryption.KeyResolver); Assert.AreEqual(options2.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); - Assert.AreEqual(listener2, client.OnClientSideDecryptionFailure); + + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + client.ClientSideEncryption.OnDecryptionFailed(default, default); + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(1, options2EventCalled); } [TestCase(16, false)] // a single cipher block @@ -328,7 +328,7 @@ public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) encryptionMetadata.ContentEncryptionIV); // compare data - Assert.AreEqual(expectedEncryptedMessage, parsedEncryptedMessage.EncryptedMessageContents); + Assert.AreEqual(expectedEncryptedMessage, parsedEncryptedMessage.EncryptedMessageText); } } @@ -613,12 +613,6 @@ public async Task OnlyOneKeyResolveAndUnwrapCall() [LiveOnly] public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows, bool peek) { - Mock listener = null; - if (useListener) - { - listener = GetFailureListener(); - } - const int numMessages = 5; var message = "any old message"; var mockKey = GetIKeyEncryptionKey().Object; @@ -640,17 +634,22 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows, bool bool threw = false; var resolver = GetAlwaysFailsKeyResolver(resolverThrows); int returnedMessages = int.MinValue; // obviously wrong value, but need to initialize to something before try block + int failureEventCalled = 0; try { // download but can't find key - var options = GetOptions(); - options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + var encryptionOptions = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) { // note decryption will throw whether the resolver throws or just returns null KeyResolver = resolver.Object, KeyWrapAlgorithm = "test" }; - options._onClientSideDecryptionFailure = listener.Object; + if (useListener) + { + encryptionOptions.DecryptionFailed += (source, args) => failureEventCalled++; + } + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; var badQueueClient = InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options)); returnedMessages = peek ? (await badQueueClient.PeekMessagesAsync(numMessages, cancellationToken: s_cancellationToken)).Value.Length @@ -673,42 +672,8 @@ public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows, bool // we already asserted the correct method was called in `catch (MockException e)` Assert.AreEqual(numMessages, resolver.Invocations.Count); - // 4 possible methods we could have been testing - System.Reflection.MethodInfo onReceiveFailAsync = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailureAsync", new Type[] { typeof(QueueMessage), typeof(Exception) }); - System.Reflection.MethodInfo onReceiveFail = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailure", new Type[] { typeof(QueueMessage), typeof(Exception) }); - System.Reflection.MethodInfo onPeekFailAsync = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailureAsync", new Type[] { typeof(PeekedMessage), typeof(Exception) }); - System.Reflection.MethodInfo onPeekFail = typeof(IClientSideDecryptionFailureListener).GetMethod("OnFailure", new Type[] { typeof(PeekedMessage), typeof(Exception) }); - - // determine what method we were testing and which we weren't - System.Reflection.MethodInfo targetMethod; - IEnumerable nonTargetMethods; - if (IsAsync && peek) - { - targetMethod = onPeekFailAsync; - nonTargetMethods = new List { onReceiveFail, onReceiveFailAsync, onPeekFail }; - } - else if (IsAsync) - { - targetMethod = onReceiveFailAsync; - nonTargetMethods = new List { onReceiveFail, onPeekFail, onPeekFailAsync }; - } - else if (peek) - { - targetMethod = onPeekFail; - nonTargetMethods = new List { onReceiveFail, onReceiveFailAsync, onPeekFailAsync }; - } - else - { - targetMethod = onReceiveFail; - nonTargetMethods = new List { onPeekFailAsync, onReceiveFailAsync, onPeekFail }; - } - - // assert target method was called the expected number of times and other methods weren't called at all - Assert.AreEqual(numMessages, listener.Invocations.Count(invocation => invocation.Method == targetMethod)); - foreach (var method in nonTargetMethods) - { - Assert.AreEqual(0, listener.Invocations.Count(invocation => invocation.Method == method)); - } + // assert event was called for each message + Assert.AreEqual(numMessages, failureEventCalled); // assert all messages were filtered out of formal response Assert.AreEqual(0, returnedMessages); diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs index 228df281462d5..9d3bf67ca8e71 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -73,7 +73,7 @@ public void SerializeEncryptedMessage() default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() { - EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), EncryptionData = result.encryptionData }; @@ -95,7 +95,7 @@ public void DeserializeEncryptedMessage() default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() { - EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), EncryptionData = result.encryptionData }; var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); @@ -118,7 +118,7 @@ public void TryDeserializeEncryptedMessage() default).EnsureCompleted(); var encryptedMessage = new EncryptedMessage() { - EncryptedMessageContents = Convert.ToBase64String(result.ciphertext), + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), EncryptionData = result.encryptionData }; var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); @@ -143,7 +143,7 @@ public void TryDeserializeGracefulOnBadInput(string input) #region ModelComparison private static bool AreEqual(EncryptedMessage left, EncryptedMessage right) - => left.EncryptedMessageContents.Equals(right.EncryptedMessageContents, StringComparison.InvariantCulture) + => left.EncryptedMessageText.Equals(right.EncryptedMessageText, StringComparison.InvariantCulture) && AreEqual(left.EncryptionData, right.EncryptionData); private static bool AreEqual(EncryptionData left, EncryptionData right) @@ -160,7 +160,7 @@ private static bool AreEqual(KeyEnvelope left, KeyEnvelope right) private static bool AreEqual(EncryptionAgent left, EncryptionAgent right) => left.EncryptionAlgorithm.Equals(right.EncryptionAlgorithm) - && left.Protocol.Equals(right.Protocol); + && left.EncryptionVersion.Equals(right.EncryptionVersion); private static bool AreEqual(IDictionary left, IDictionary right) { From 77c38c31d2594f07126601663ef6d13595a98de1 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 5 Jun 2020 13:49:17 -0700 Subject: [PATCH 18/21] Export api --- .../api/Azure.Storage.Common.netstandard2.0.cs | 4 ++-- .../api/Azure.Storage.Queues.netstandard2.0.cs | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index ed381d3506e36..9902d1ac7bc2e 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -3,14 +3,14 @@ namespace Azure.Storage public partial class ClientSideEncryptionOptions { public ClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) { } + public Azure.Storage.ClientSideEncryptionVersion EncryptionVersion { get { throw null; } } public Azure.Core.Cryptography.IKeyEncryptionKey KeyEncryptionKey { get { throw null; } set { } } public Azure.Core.Cryptography.IKeyEncryptionKeyResolver KeyResolver { get { throw null; } set { } } public string KeyWrapAlgorithm { get { throw null; } set { } } - public Azure.Storage.ClientSideEncryptionVersion Version { get { throw null; } } } public enum ClientSideEncryptionVersion { - V1_0 = 0, + V1_0 = 1, } public partial class StorageSharedKeyCredential { diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index c89ff4f806e55..d8a95259324a2 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -317,22 +317,24 @@ internal UpdateReceipt() { } } namespace Azure.Storage.Queues.Specialized { - public partial interface IClientSideDecryptionFailureListener + public partial class ClientSideDecryptionFailureEventArgs { - void OnFailure(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); - void OnFailure(Azure.Storage.Queues.Models.QueueMessage message, System.Exception exception); - System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.PeekedMessage message, System.Exception exception); - System.Threading.Tasks.Task OnFailureAsync(Azure.Storage.Queues.Models.QueueMessage message, System.Exception exception); + internal ClientSideDecryptionFailureEventArgs() { } + public System.Exception Exception { get { throw null; } } + public object Message { get { throw null; } } + } + public partial class QueueClientSideEncryptionOptions : Azure.Storage.ClientSideEncryptionOptions + { + public QueueClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) : base (default(Azure.Storage.ClientSideEncryptionVersion)) { } + public event System.EventHandler DecryptionFailed { add { } remove { } } } public partial class SpecializedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions { public SpecializedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } - public Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener OnClientSideDecryptionFailure { get { throw null; } set { } } } public static partial class SpecializedQueueExtensions { - public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionFailureListener(this Azure.Storage.Queues.QueueClient client, Azure.Storage.Queues.Specialized.IClientSideDecryptionFailureListener listener) { throw null; } public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } } } From b3f5bc0361d2e6e9e6714f28a4854b72ee178539 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 5 Jun 2020 14:41:28 -0700 Subject: [PATCH 19/21] Testing for update message encryption --- .../Azure.Storage.Queues/src/QueueClient.cs | 2 +- .../tests/ClientSideEncryptionTests.cs | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index fa9d96759ec05..46b354f411615 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -2073,7 +2073,7 @@ private async Task> UpdateMessageInternal( $"{nameof(visibilityTimeout)}: {visibilityTimeout}"); try { - messageText = UsingClientSideEncryption + messageText = UsingClientSideEncryption && messageText != default ? await new QueueClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) : messageText; diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs index 9a07660289deb..f6b2bb965cbd6 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Threading; @@ -550,6 +551,54 @@ public async Task OnlyOneKeyWrapCall() } } + [Test] + [LiveOnly] + public async Task UpdateEncryptedMessage() + { + var message1 = GetRandomMessage(Constants.KB); + var message2 = GetRandomMessage(Constants.KB); + var mockKey = GetIKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey.Object, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message1, cancellationToken: s_cancellationToken); // invokes key wrap first time + + // download and update message + var messageToUpdate = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value[0]; + await queue.UpdateMessageAsync(messageToUpdate.MessageId, messageToUpdate.PopReceipt, message2, cancellationToken: s_cancellationToken); // invokes key unwrap first time and key wrap second time + + // download with decryption + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; // invokes key unwrap second time + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message2, downloadedMessage); + + // assert key wrap and unwrap were each invoked twice + MethodInfo keyWrap; + MethodInfo keyUnwrap; + if (IsAsync) + { + keyWrap = typeof(IKeyEncryptionKey).GetMethod("WrapKeyAsync"); + keyUnwrap = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); + } + else + { + keyWrap = typeof(IKeyEncryptionKey).GetMethod("WrapKey"); + keyUnwrap = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); + } + Assert.AreEqual(2, mockKey.Invocations.Count(invocation => invocation.Method == keyWrap)); + Assert.AreEqual(2, mockKey.Invocations.Count(invocation => invocation.Method == keyUnwrap)); + } + } + [Test] [LiveOnly] // cannot seed content encryption key public async Task OnlyOneKeyResolveAndUnwrapCall() From 46eeb8bd2a9ccc3bc0cea60b59e238e0afe77baf Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 5 Jun 2020 14:51:47 -0700 Subject: [PATCH 20/21] Minor event API adjustments --- .../src/QueueClientSideEncryptionOptions.cs | 45 +++---------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs index 27cbb2cf21f97..c28c8a951c3ae 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs @@ -30,26 +30,7 @@ public QueueClientSideEncryptionOptions(ClientSideEncryptionVersion version) : b internal void OnDecryptionFailed(object message, Exception e) { - DecryptionFailed?.Invoke(this, new ClientSideDecryptionFailureEventArgs(message, e)); - } - - /// - /// Clones this class as this subclass instead of the base class. - /// - /// - /// Compiler restriction: can only copy an event in an instance method on the class containing the event. - /// This class exists to allow us to copy the event. - /// - private QueueClientSideEncryptionOptions CloneAsQueueClientSideEncryptionOptions() - { - // clone base class but as instance of this class - var newOptions = new QueueClientSideEncryptionOptions(EncryptionVersion); - ClientSideEncryptionOptionsExtensions.CopyOptions(this, newOptions); - - // clone data specific to this subclass - newOptions.DecryptionFailed = DecryptionFailed; - - return newOptions; + DecryptionFailed?.Invoke(message, new ClientSideDecryptionFailureEventArgs(e)); } /// @@ -64,18 +45,13 @@ internal static QueueClientSideEncryptionOptions CloneFrom(ClientSideEncryptionO { return default; } - else if (options is QueueClientSideEncryptionOptions) + var newOptions = new QueueClientSideEncryptionOptions(options.EncryptionVersion); + ClientSideEncryptionOptionsExtensions.CopyOptions(options, newOptions); + if (options is QueueClientSideEncryptionOptions queueOptions) { - return ((QueueClientSideEncryptionOptions)options).CloneAsQueueClientSideEncryptionOptions(); - } - else - { - // clone base class but as instance of this class - var newOptions = new QueueClientSideEncryptionOptions(options.EncryptionVersion); - ClientSideEncryptionOptionsExtensions.CopyOptions(options, newOptions); - - return newOptions; + newOptions.DecryptionFailed = queueOptions.DecryptionFailed; } + return newOptions; } } @@ -89,15 +65,8 @@ public class ClientSideDecryptionFailureEventArgs /// public Exception Exception { get; } - /// - /// Message the failure occured with. Can be an instance of either - /// or . - /// - public object Message { get; } - - internal ClientSideDecryptionFailureEventArgs(object message, Exception exception) + internal ClientSideDecryptionFailureEventArgs(Exception exception) { - Message = message; Exception = exception; } } From 7d3ce1b6dcf37e7f4344f4f54bc5fecc85e32ac8 Mon Sep 17 00:00:00 2001 From: James Schreppler Date: Fri, 5 Jun 2020 14:57:02 -0700 Subject: [PATCH 21/21] export api --- .../api/Azure.Storage.Queues.netstandard2.0.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index d8a95259324a2..da3eaacfca334 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -321,7 +321,6 @@ public partial class ClientSideDecryptionFailureEventArgs { internal ClientSideDecryptionFailureEventArgs() { } public System.Exception Exception { get { throw null; } } - public object Message { get { throw null; } } } public partial class QueueClientSideEncryptionOptions : Azure.Storage.ClientSideEncryptionOptions {