diff --git a/eng/Packages.Data.props b/eng/Packages.Data.props index c2af61dd2e916..dbf0b46a62574 100644 --- a/eng/Packages.Data.props +++ b/eng/Packages.Data.props @@ -47,6 +47,7 @@ + diff --git a/sdk/core/Azure.Core/Azure.Core.All.sln b/sdk/core/Azure.Core/Azure.Core.All.sln index 8eb1f9e45d5be..fbe582008f679 100644 --- a/sdk/core/Azure.Core/Azure.Core.All.sln +++ b/sdk/core/Azure.Core/Azure.Core.All.sln @@ -85,8 +85,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.S EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.Tests", "..\..\storage\Azure.Storage.Blobs.Batch\tests\Azure.Storage.Blobs.Batch.Tests.csproj", "{EA651D86-70EE-4F32-AF1D-E5EB85ACF79F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Cryptography", "..\..\storage\Azure.Storage.Blobs.Cryptography\src\Azure.Storage.Blobs.Cryptography.csproj", "{908FBA6A-12B2-44DA-BCFB-11750B613FFA}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Samples.Tests", "..\..\storage\Azure.Storage.Blobs\samples\Azure.Storage.Blobs.Samples.Tests.csproj", "{83DC827C-9AD6-4C76-883F-53AB4A027E82}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Tests", "..\..\storage\Azure.Storage.Blobs\tests\Azure.Storage.Blobs.Tests.csproj", "{02708856-2D84-47E2-80DD-65A3A31E01C8}" diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj index c4fa15a3c5a23..739f325f6efb0 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/samples/Azure.Storage.Blobs.Batch.Samples.Tests.csproj @@ -9,7 +9,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj index cb4a73b1257ef..82561843c497d 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt b/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt deleted file mode 100644 index 54c406eff995d..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt +++ /dev/null @@ -1,6 +0,0 @@ -Breaking Changes -================ - -Next Preview --------------------------- -- New Azure.Storage.Blobs.Cryptography client library. diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md b/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md deleted file mode 100644 index 341f38fc0934e..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release History - -## 12.0.0-preview.5 - -- This preview is the first release supporting client-side encryption for Azure - Storage blobs. diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md b/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md deleted file mode 100644 index 1a6d44429b0f3..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md +++ /dev/null @@ -1,109 +0,0 @@ -# Azure Storage Blobs Cryptography client library for .NET - -> Server Version: 2019-02-02 - -Azure Blob storage is Microsoft's object storage solution for the cloud. Blob -storage is optimized for storing massive amounts of unstructured data. -Unstructured data is data that does not adhere to a particular data model or -definition, such as text or binary data. - -[Source code][source] | [Package (NuGet)][package] | [API reference documentation][docs] | [REST API documentation][rest_docs] | [Product documentation][product_docs] - -## Getting started - -### Install the package - -Install the Azure Storage Blobs Cryptography client library for .NET with [NuGet][nuget]: - -```Powershell -dotnet add package Azure.Storage.Blobs.Cryptography --version 12.0.0-preview.4 -``` - -### Prerequisites - -You need an [Azure subscription][azure_sub] and a -[Storage Account][storage_account_docs] to use this package. - -To create a new Storage Account, you can use the [Azure Portal][storage_account_create_portal], -[Azure PowerShell][storage_account_create_ps], or the [Azure CLI][storage_account_create_cli]. -Here's an example using the Azure CLI: - -```Powershell -az storage account create --name MyStorageAccount --resource-group MyResourceGroup --location westus --sku Standard_LRS -``` - -## Key concepts - -TODO: Add Key Concepts - -## Examples - -TODO: Add Examples - -## Troubleshooting - -All Blob service operations will throw a -[RequestFailedException][RequestFailedException] on failure with -helpful [`ErrorCode`s][error_codes]. Many of these errors are recoverable. - -TODO: Update sample - -```c# -string connectionString = ""; - -// Try to create a container named "sample-container" and avoid any potential race -// conditions that might arise by checking if the container exists before creating -BlobContainerClient container = new BlobContainerClient(connectionString, "sample-container"); -try -{ - container.Create(); -} -catch (RequestFailedException ex) - when (ex.ErrorCode == BlobErrorCode.ContainerAlreadyExists) -{ - // Ignore any errors if the container already exists -} -``` - -## Next steps - -TODO: Link Samples - -## Contributing - -See the [Storage CONTRIBUTING.md][storage_contrib] for details on building, -testing, and contributing to this library. - -This project welcomes contributions and suggestions. Most contributions require -you to agree to a Contributor License Agreement (CLA) declaring that you have -the right to, and actually do, grant us the rights to use your contribution. For -details, visit [cla.microsoft.com][cla]. - -This project has adopted the [Microsoft Open Source Code of Conduct][coc]. -For more information see the [Code of Conduct FAQ][coc_faq] -or contact [opencode@microsoft.com][coc_contact] with any -additional questions or comments. - -![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-net%2Fsdk%2Fstorage%2FAzure.Storage.Blobs.Cryptography%2FREADME.png) - - -[source]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/storage/Azure.Storage.Blobs.Cryptography/src -[package]: https://www.nuget.org/packages/Azure.Storage.Blobs.Cryptography/ -[docs]: https://azure.github.io/azure-sdk-for-net/storage.html -[rest_docs]: https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api -[product_docs]: https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview -[nuget]: https://www.nuget.org/ -[storage_account_docs]: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview -[storage_account_create_ps]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-powershell -[storage_account_create_cli]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-cli -[storage_account_create_portal]: https://docs.microsoft.com/en-us/azure/storage/common/storage-quickstart-create-account?tabs=azure-portal -[azure_cli]: https://docs.microsoft.com/cli/azure -[azure_sub]: https://azure.microsoft.com/free/ -[identity]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/identity/Azure.Identity/README.md -[RequestFailedException]: https://github.com/Azure/azure-sdk-for-net/tree/master/sdk/core/Azure.Core/src/RequestFailedException.cs -[error_codes]: https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes -[storage_contrib]: ../CONTRIBUTING.md -[cla]: https://cla.microsoft.com -[coc]: https://opensource.microsoft.com/codeofconduct/ -[coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ -[coc_contact]: mailto:opencode@microsoft.com diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs deleted file mode 100644 index a50a41a71bd94..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/api/Azure.Storage.Blobs.Cryptography.netstandard2.0.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Azure.Storage.Blobs.Specialized -{ - public partial class EncryptedBlockBlobClient : Azure.Storage.Blobs.BlobClient - { - protected EncryptedBlockBlobClient() { } - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName) { } - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName, Azure.Storage.Blobs.BlobClientOptions options) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Core.TokenCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Storage.Blobs.BlobClientOptions options = null) { } - public EncryptedBlockBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredential credential, Azure.Storage.Blobs.BlobClientOptions options = null) { } - } -} diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs deleted file mode 100644 index 813f19b67cc4e..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/AssemblyInfo.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("Azure.Storage.Blobs.Tests, PublicKey=" + - "0024000004800000940000000602000000240000525341310004000001000100d15ddcb2968829" + - "5338af4b7686603fe614abd555e09efba8fb88ee09e1f7b1ccaeed2e8f823fa9eef3fdd60217fc" + - "012ea67d2479751a0b8c087a4185541b851bd8b16f8d91b840e51b1cb0ba6fe647997e57429265" + - "e85ef62d565db50a69ae1647d54d7bd855e4db3d8a91510e5bcbd0edfbbecaa20a7bd9ae74593d" + - "aa7b11b4")] diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj deleted file mode 100644 index 3edd0054cacdd..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/Azure.Storage.Blobs.Cryptography.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - $(RequiredTargetFrameworks) - - - Microsoft Azure.Storage.Blobs.Cryptography client library - 12.0.0-preview.5 - BlobSDK;$(DefineConstants) - Microsoft Azure Storage Blobs Cryptography;Encrypted blob;client-side encryption;client side ecnryption;Microsoft;Azure;Blobs;Blob;Storage;StorageScalable;$(PackageCommonTags) - - This client library supports client-side encryption for the Microsoft Azure Storage Blob service. - For this release see notes - https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/README.md and https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/CHANGELOG.md - in addition to the breaking changes https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/storage/Azure.Storage.Blobs.Cryptography/BreakingChanges.txt - Microsoft Azure Storage quickstarts and tutorials - https://docs.microsoft.com/en-us/azure/storage/ - Microsoft Azure Storage REST API Reference - https://docs.microsoft.com/en-us/rest/api/storageservices/ - REST API Reference for Blob Service - https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api - - false - - - - - - diff --git a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs deleted file mode 100644 index 618931a4f77d2..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs.Cryptography/src/EncryptedBlockBlobClient.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using Azure.Core; -using Azure.Core.Pipeline; -using Azure.Storage.Blobs.Models; - -#pragma warning disable SA1402 // File may only contain a single type - -namespace Azure.Storage.Blobs.Specialized -{ - /// - /// The allows you to manipulate - /// Azure Storage block blobs with client-side encryption. - /// - public class EncryptedBlockBlobClient : BlobClient - { - #region ctors - /// - /// Initializes a new instance of the - /// class for mocking. - /// - protected EncryptedBlockBlobClient() - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A connection string includes the authentication information - /// required for your application to access data in an Azure Storage - /// account at runtime. - /// - /// For more information, . - /// - /// - /// The name of the container containing this encrypted block blob. - /// - /// - /// The name of this encrypted block blob. - /// - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName) - : base(connectionString, containerName, blobName) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A connection string includes the authentication information - /// required for your application to access data in an Azure Storage - /// account at runtime. - /// - /// For more information, . - /// - /// - /// The name of the container containing this encrypted block blob. - /// - /// - /// The name of this encrypted block blob. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(string connectionString, string containerName, string blobName, BlobClientOptions options) - : base(connectionString, containerName, blobName, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the encrypted block blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, BlobClientOptions options = default) - : base(blobUri, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// The shared key credential used to sign requests. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) - : base(blobUri, credential, options) - { - } - - /// - /// Initializes a new instance of the - /// class. - /// - /// - /// A referencing the blob that includes the - /// name of the account, the name of the container, and the name of - /// the blob. - /// - /// - /// The token credential used to sign requests. - /// - /// - /// Optional client options that define the transport pipeline - /// policies for authentication, retries, etc., that are applied to - /// every request. - /// - public EncryptedBlockBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) - : base(blobUri, credential, options) - { - } - - #endregion ctors - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 31a6d1b16ef05..1bfb04906e8cf 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -1072,6 +1072,11 @@ public PageBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredenti public new Azure.Storage.Blobs.Specialized.PageBlobClient WithSnapshot(string snapshot) { throw null; } protected sealed override Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; } } + public partial class SpecializedBlobClientOptions : Azure.Storage.Blobs.BlobClientOptions + { + public SpecializedBlobClientOptions(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion version = Azure.Storage.Blobs.BlobClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Blobs.BlobClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } public static partial class SpecializedBlobExtensions { public static Azure.Storage.Blobs.Specialized.AppendBlobClient GetAppendBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } @@ -1080,6 +1085,7 @@ public static partial class SpecializedBlobExtensions public static Azure.Storage.Blobs.Specialized.BlobLeaseClient GetBlobLeaseClient(this Azure.Storage.Blobs.Specialized.BlobBaseClient client, string leaseId = null) { throw null; } public static Azure.Storage.Blobs.Specialized.BlockBlobClient GetBlockBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } public static Azure.Storage.Blobs.Specialized.PageBlobClient GetPageBlobClient(this Azure.Storage.Blobs.BlobContainerClient client, string blobName) { throw null; } + public static Azure.Storage.Blobs.BlobClient WithClientSideEncryptionOptions(this Azure.Storage.Blobs.BlobClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } } } namespace Azure.Storage.Sas diff --git a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj index 62f992b5a1221..27e450932b380 100644 --- a/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/samples/Azure.Storage.Blobs.Samples.Tests.csproj @@ -9,7 +9,6 @@ - diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 57233499a3456..49332c5bda855 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -98,6 +98,7 @@ public AppendBlobClient(string connectionString, string blobContainerName, strin public AppendBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -118,6 +119,7 @@ public AppendBlobClient(string connectionString, string blobContainerName, strin public AppendBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -141,6 +143,7 @@ public AppendBlobClient(Uri blobUri, BlobClientOptions options = default) public AppendBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -164,6 +167,7 @@ public AppendBlobClient(Uri blobUri, StorageSharedKeyCredential credential, Blob public AppendBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -192,9 +196,24 @@ internal AppendBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options?._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(AppendBlobClient)); + } + } #endregion ctors /// @@ -1040,13 +1059,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static AppendBlobClient GetAppendBlobClient( this BlobContainerClient client, - string blobName) => - new AppendBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(AppendBlobClient)); + } + return new AppendBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index e78d8243720d1..317c746eecc17 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) @@ -16,6 +16,9 @@ REST API Reference for Blob Service - https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api + + + @@ -35,6 +38,8 @@ + + @@ -61,5 +66,6 @@ + \ No newline at end of file diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index f8716af71e2cb..2d821f5377b31 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -2,13 +2,13 @@ // Licensed under the MIT License. using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; #pragma warning disable SA1402 // File may only contain a single type @@ -75,6 +75,18 @@ public class BlobBaseClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + + internal bool UsingClientSideEncryption => ClientSideEncryption != default; + /// /// The name of the Encryption Scope to be used when sending requests. /// @@ -204,6 +216,7 @@ public BlobBaseClient(string connectionString, string blobContainerName, string _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -302,6 +315,7 @@ internal BlobBaseClient(Uri blobUri, HttpPipelinePolicy authentication, BlobClie _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -325,6 +339,7 @@ internal BlobBaseClient(Uri blobUri, HttpPipelinePolicy authentication, BlobClie /// /// Client diagnostics. /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. internal BlobBaseClient( Uri blobUri, @@ -332,6 +347,7 @@ internal BlobBaseClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) { _uri = blobUri; @@ -339,6 +355,7 @@ internal BlobBaseClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -370,7 +387,7 @@ internal BlobBaseClient( protected virtual BlobBaseClient WithSnapshotCore(string snapshot) { var builder = new BlobUriBuilder(Uri) { Snapshot = snapshot }; - return new BlobBaseClient(builder.ToUri(), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + return new BlobBaseClient(builder.ToUri(), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); } /// @@ -638,11 +655,17 @@ private async Task> DownloadInternal( bool async, CancellationToken cancellationToken) { + HttpRange requestedRange = range; using (Pipeline.BeginLoggingScope(nameof(BlobBaseClient))) { Pipeline.LogMethodEnter(nameof(BlobBaseClient), message: $"{nameof(Uri)}: {Uri}"); try { + if (UsingClientSideEncryption) + { + range = BlobClientSideDecryptor.GetEncryptedBlobRange(range); + } + // Start downloading the blob (Response response, Stream stream) = await StartDownloadAsync( range, @@ -661,7 +684,7 @@ private async Task> DownloadInternal( // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - response.Value.Content = RetriableStream.Create( + stream = RetriableStream.Create( stream, startOffset => StartDownloadAsync( @@ -686,6 +709,16 @@ private async Task> DownloadInternal( Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); + // if using clientside encryption, wrap the auto-retry stream in a decryptor + // we already return a nonseekable stream; returning a crypto stream is fine + if (UsingClientSideEncryption) + { + stream = await new BlobClientSideDecryptor(new ClientSideDecryptor(ClientSideEncryption)) + .DecryptInternal(stream, response.Value.Metadata, requestedRange, response.Value.ContentRange, async, cancellationToken).ConfigureAwait(false); + } + + response.Value.Content = stream; + // Wrap the FlattenedDownloadProperties into a BlobDownloadOperation // to make the Content easier to find return Response.FromValue(new BlobDownloadInfo(response.Value), response.GetRawResponse()); @@ -1201,7 +1234,7 @@ internal async Task StagedDownloadAsync( bool async = true, CancellationToken cancellationToken = default) { - var client = new BlobBaseClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + var client = new BlobBaseClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); PartitionedDownloader downloader = new PartitionedDownloader(client, transferOptions); @@ -2977,6 +3010,24 @@ public static BlobBaseClient GetBlobBaseClient( client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, + client.ClientSideEncryption, + client.EncryptionScope); + + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// New instance with provided options and same internals otherwise. + public static BlobClient WithClientSideEncryptionOptions(this BlobClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) + => new BlobClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + client.CustomerProvidedKey, + clientSideEncryptionOptions, client.EncryptionScope); } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs index 21994d631e8f2..f3f09e6f8712a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClient.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -10,8 +9,11 @@ using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; +#pragma warning disable SA1402 // File may only contain a single type + namespace Azure.Storage.Blobs { /// @@ -162,6 +164,7 @@ public BlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions opt /// /// Client diagnostics. /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. internal BlobClient( Uri blobUri, @@ -169,8 +172,9 @@ internal BlobClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, clientSideEncryption, encryptionScope) { } #endregion ctors @@ -946,6 +950,13 @@ internal async Task> StagedUploadAsync( bool async = true, CancellationToken cancellationToken = default) { + if (UsingClientSideEncryption) + { + // content is now unseekable, so PartitionedUploader will be forced to do a buffered multipart upload + (content, metadata) = await new BlobClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) + .ClientSideEncryptInternal(content, metadata, async, cancellationToken).ConfigureAwait(false); + } + var client = new BlockBlobClient(Uri, Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); PartitionedUploader uploader = new PartitionedUploader( @@ -1020,6 +1031,28 @@ internal async Task> StagedUploadAsync( bool async = true, CancellationToken cancellationToken = default) { + // TODO Upload from file will get it's own implementation in the future that opens more + // than one stream at once. This is incompatible with .NET's CryptoStream. We will + // need to uncomment the below code and revert to upload from stream if client-side + // encryption is enabled. + //if (ClientSideEncryption != default) + //{ + // using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read)) + // { + // return await StagedUploadAsync( + // stream, + // blobHttpHeaders, + // metadata, + // conditions, + // progressHandler, + // accessTier, + // transferOptions: transferOptions, + // async: async, + // cancellationToken: cancellationToken) + // .ConfigureAwait(false); + // } + //} + using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read)) { return await StagedUploadAsync( @@ -1036,21 +1069,5 @@ internal async Task> StagedUploadAsync( } } #endregion Upload - - // NOTE: TransformContent is no longer called by the new implementation - // of parallel upload. Leaving the virtual stub in for now to avoid - // any confusion. Will need to be added back for encryption work per - // #7127. - - /// - /// Performs a transform on the data for uploads. It is a no-op by default. - /// - /// Content to transform. - /// Content metadata to transform. - /// Transformed content stream and metadata. - internal virtual (Stream, Metadata) TransformContent(Stream content, Metadata metadata) - { - return (content, metadata); // no-op - } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index 26eb63f5cf05d..e6f39e9204a63 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -69,6 +69,10 @@ public enum ServiceVersion /// public Uri GeoRedundantSecondaryUri { get; set; } + #region Advanced Options + internal ClientSideEncryptionOptions _clientSideEncryptionOptions; + #endregion + /// /// Initializes a new instance of the /// class. diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs new file mode 100644 index 0000000000000..2fe25686d5ea5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Shared; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Blobs +{ + internal class BlobClientSideDecryptor + { + private readonly ClientSideDecryptor _decryptor; + + public BlobClientSideDecryptor(ClientSideDecryptor decryptor) + { + _decryptor = decryptor; + } + + public async Task DecryptInternal( + Stream content, + Metadata metadata, + HttpRange originalRange, + string receivedContentRange, + bool async, + CancellationToken cancellationToken) + { + ContentRange? contentRange = string.IsNullOrWhiteSpace(receivedContentRange) + ? default + : ContentRange.Parse(receivedContentRange); + + EncryptionData encryptionData = GetAndValidateEncryptionDataOrDefault(metadata); + if (encryptionData == default) + { + return await TrimStreamInternal(content, originalRange, contentRange, pulledOutIV: false, async, cancellationToken).ConfigureAwait(false); + } + + bool ivInStream = originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize; + + // this method throws when key cannot be resolved. Blobs is intended to throw on this failure. + var plaintext = await _decryptor.DecryptInternal( + content, + encryptionData, + ivInStream, + CanIgnorePadding(contentRange), + async, + cancellationToken).ConfigureAwait(false); + + return await TrimStreamInternal(plaintext, originalRange, contentRange, ivInStream, async, cancellationToken).ConfigureAwait(false); + } + + private static async Task TrimStreamInternal( + Stream stream, + HttpRange originalRange, + ContentRange? receivedRange, + bool pulledOutIV, + bool async, + CancellationToken cancellationToken) + { + // retrim start of stream to original requested location + // keeping in mind whether we already pulled the IV out of the stream as well + int gap = (int)(originalRange.Offset - (receivedRange?.Start ?? 0)) + - (pulledOutIV ? Constants.ClientSideEncryption.EncryptionBlockSize : 0); + + int read = 0; + while (gap > read) + { + int toRead = gap - read; + // throw away initial bytes we want to trim off; stream cannot seek into future + if (async) + { + read += await stream.ReadAsync(new byte[toRead], 0, toRead, cancellationToken).ConfigureAwait(false); + } + else + { + read += stream.Read(new byte[toRead], 0, toRead); + } + } + + if (originalRange.Length.HasValue) + { + stream = new WindowStream(stream, originalRange.Length.Value); + } + + return stream; + } + + private static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata metadata) + { + if (metadata == default) + { + return default; + } + if (!metadata.TryGetValue(Constants.ClientSideEncryption.EncryptionDataKey, out string encryptedDataString)) + { + return default; + } + + EncryptionData encryptionData = EncryptionDataSerializer.Deserialize(encryptedDataString); + + _ = encryptionData.ContentEncryptionIV ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata( + nameof(EncryptionData.ContentEncryptionIV)); + _ = encryptionData.WrappedContentKey.EncryptedKey ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata( + nameof(EncryptionData.WrappedContentKey.EncryptedKey)); + + return encryptionData; + } + + /// + /// Gets whether to ignore padding options for decryption. + /// + /// Downloaded content range. + /// True if we should ignore padding. + /// + /// If the last cipher block of the blob was returned, we need the padding. Otherwise, we can ignore it. + /// + private static bool CanIgnorePadding(ContentRange? contentRange) + { + // if Content-Range not present, we requested the whole blob + if (contentRange == null) + { + return false; + } + + // if range is wildcard, we requested the whole blob + if (!contentRange.Value.End.HasValue) + { + return false; + } + + // blob storage will always return ContentRange.Size + // we don't have to worry about the impossible decision of what to do if it doesn't + + // did we request the last block? + // end is inclusive/0-index, so end = n and size = n+1 means we requested the last block + if (contentRange.Value.Size - contentRange.Value.End == 1) + { + return false; + } + + return true; + } + + internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange) + { + int offsetAdjustment = 0; + long? adjustedDownloadCount = originalRange.Length; + + // Calculate offsetAdjustment. + if (originalRange.Offset != 0) + { + // Align with encryption block boundary. + int diff; + if ((diff = (int)(originalRange.Offset % Constants.ClientSideEncryption.EncryptionBlockSize)) != 0) + { + offsetAdjustment += diff; + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += diff; + } + } + + // Account for IV. + if (originalRange.Offset >= Constants.ClientSideEncryption.EncryptionBlockSize) + { + offsetAdjustment += Constants.ClientSideEncryption.EncryptionBlockSize; + // Increment adjustedDownloadCount if necessary. + if (adjustedDownloadCount != default) + { + adjustedDownloadCount += Constants.ClientSideEncryption.EncryptionBlockSize; + } + } + } + + // Align adjustedDownloadCount with encryption block boundary at the end of the range. Note that it is impossible + // to adjust past the end of the blob as an encrypted blob was padded to align to an encryption block boundary. + if (adjustedDownloadCount != null) + { + adjustedDownloadCount += ( + Constants.ClientSideEncryption.EncryptionBlockSize - (int)(adjustedDownloadCount + % Constants.ClientSideEncryption.EncryptionBlockSize) + ) % Constants.ClientSideEncryption.EncryptionBlockSize; + } + + return new HttpRange(originalRange.Offset - offsetAdjustment, adjustedDownloadCount); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs new file mode 100644 index 0000000000000..e94c6e694c89d --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientSideEncryptor.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Blobs +{ + internal class BlobClientSideEncryptor + { + private readonly ClientSideEncryptor _encryptor; + + public BlobClientSideEncryptor(ClientSideEncryptor encryptor) + { + _encryptor = encryptor; + } + + /// + /// Applies client-side encryption to the data for upload. + /// + /// + /// Content to encrypt. + /// + /// + /// Metadata to add encryption metadata to. + /// + /// + /// Whether to perform this operation asynchronously. + /// + /// + /// Cancellation token. + /// + /// Transformed content stream and metadata. + public async Task<(Stream, Metadata)> ClientSideEncryptInternal( + Stream content, + Metadata metadata, + bool async, + CancellationToken cancellationToken) + { + (Stream nonSeekableCiphertext, EncryptionData encryptionData) = await _encryptor.EncryptInternal( + content, + async, + cancellationToken).ConfigureAwait(false); + + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + metadata.Add(Constants.ClientSideEncryption.EncryptionDataKey, EncryptionDataSerializer.Serialize(encryptionData)); + + return (nonSeekableCiphertext, metadata); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs index 17003636389f5..d24c5dedf3044 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobContainerClient.cs @@ -11,6 +11,7 @@ using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Blobs @@ -92,6 +93,16 @@ public class BlobContainerClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The to be used when sending requests. /// @@ -287,6 +298,7 @@ internal BlobContainerClient(Uri blobContainerUri, HttpPipelinePolicy authentica _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -309,6 +321,7 @@ internal BlobContainerClient(Uri blobContainerUri, HttpPipelinePolicy authentica /// /// /// Customer provided key. + /// /// Encryption scope. internal BlobContainerClient( Uri containerUri, @@ -316,6 +329,7 @@ internal BlobContainerClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope) { _uri = containerUri; @@ -323,6 +337,7 @@ internal BlobContainerClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -350,7 +365,14 @@ internal BlobContainerClient( /// protected static BlobContainerClient CreateClient(Uri containerUri, BlobClientOptions options, HttpPipeline pipeline) { - return new BlobContainerClient(containerUri, pipeline, options.Version, new ClientDiagnostics(options), null, null); + return new BlobContainerClient( + containerUri, + pipeline, + options.Version, + new ClientDiagnostics(options), + customerProvidedKey: null, + clientSideEncryption: null, + encryptionScope: null); } #endregion ctor @@ -363,7 +385,7 @@ protected static BlobContainerClient CreateClient(Uri containerUri, BlobClientOp /// The name of the blob. /// A new instance. public virtual BlobClient GetBlobClient(string blobName) => - new BlobClient(Uri.AppendToPath(blobName), _pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + new BlobClient(Uri.AppendToPath(blobName), _pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); /// /// Sets the various name fields if they are currently null. diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs index 59ea8716fec12..43dcb30b0149e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobServiceClient.cs @@ -8,6 +8,7 @@ using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; +using Azure.Storage.Cryptography; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Blobs @@ -82,6 +83,16 @@ public class BlobServiceClient /// internal virtual CustomerProvidedKey? CustomerProvidedKey => _customerProvidedKey; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The name of the Encryption Scope to be used when sending request. /// @@ -163,6 +174,7 @@ public BlobServiceClient(string connectionString, BlobClientOptions options) _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); _customerProvidedKey = options.CustomerProvidedKey; + _clientSideEncryption = options._clientSideEncryptionOptions?.Clone(); _encryptionScope = options.EncryptionScope; BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); @@ -252,6 +264,7 @@ internal BlobServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Bl options?.Version ?? BlobClientOptions.LatestVersion, new ClientDiagnostics(options), options?.CustomerProvidedKey, + options?._clientSideEncryptionOptions?.Clone(), options?.EncryptionScope, options.Build(authentication)) { @@ -277,6 +290,7 @@ internal BlobServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Bl /// diagnostic scopes every request. /// /// Customer provided key. + /// Client-side encryption options. /// Encryption scope. /// /// The transport pipeline used to send every request. @@ -287,6 +301,7 @@ internal BlobServiceClient( BlobClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, + ClientSideEncryptionOptions clientSideEncryption, string encryptionScope, HttpPipeline pipeline) { @@ -296,12 +311,15 @@ internal BlobServiceClient( _version = version; _clientDiagnostics = clientDiagnostics; _customerProvidedKey = customerProvidedKey; + _clientSideEncryption = clientSideEncryption?.Clone(); _encryptionScope = encryptionScope; BlobErrors.VerifyCpkAndEncryptionScopeNotBothSet(_customerProvidedKey, _encryptionScope); BlobErrors.VerifyHttpsCustomerProvidedKey(_uri, _customerProvidedKey); } /// + /// Intended for DataLake to create a backing blob client. + /// /// Initializes a new instance of the /// class. /// @@ -330,7 +348,15 @@ protected static BlobServiceClient CreateClient( HttpPipelinePolicy authentication, HttpPipeline pipeline) { - return new BlobServiceClient(serviceUri, authentication, options.Version, new ClientDiagnostics(options), null, null, pipeline); + return new BlobServiceClient( + serviceUri, + authentication, + options.Version, + new ClientDiagnostics(options), + customerProvidedKey: null, + clientSideEncryption: null, + encryptionScope: null, + pipeline); } #endregion ctors @@ -347,7 +373,7 @@ protected static BlobServiceClient CreateClient( /// A for the desired container. /// public virtual BlobContainerClient GetBlobContainerClient(string blobContainerName) => - new BlobContainerClient(Uri.AppendToPath(blobContainerName), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, EncryptionScope); + new BlobContainerClient(Uri.AppendToPath(blobContainerName), Pipeline, Version, ClientDiagnostics, CustomerProvidedKey, ClientSideEncryption, EncryptionScope); #region protected static accessors for Azure.Storage.Blobs.Batch /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 1e2969d21d360..a6b9f4e4743f2 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -153,6 +153,7 @@ public BlockBlobClient(string connectionString, string containerName, string blo public BlockBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -172,6 +173,7 @@ public BlockBlobClient(string connectionString, string blobContainerName, string public BlockBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -194,6 +196,7 @@ public BlockBlobClient(Uri blobUri, BlobClientOptions options = default) public BlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -216,6 +219,7 @@ public BlockBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobC public BlockBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -243,7 +247,14 @@ internal BlockBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } @@ -271,6 +282,14 @@ protected static BlockBlobClient CreateClient(Uri blobUri, BlobClientOptions opt { return new BlockBlobClient(blobUri, pipeline, options.Version, new ClientDiagnostics(options), null, null); } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + } #endregion ctors /// @@ -1542,13 +1561,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static BlockBlobClient GetBlockBlobClient( this BlobContainerClient client, - string blobName) => - new BlockBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(BlockBlobClient)); + } + return new BlockBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs new file mode 100644 index 0000000000000..fe5f6fb914c14 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/ContentRange.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Text; + +namespace Azure.Storage.Blobs.Models +{ + internal struct ContentRange + { + private const string WildcardMarker = "*"; + + public struct RangeUnit + { + internal const string BytesValue = "bytes"; + + private readonly string _value; + + /// + /// Initializes a new instance of the structure. + /// + /// The string value of the instance. + public RangeUnit(string value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// Label for bytes as the measurement of content range. + /// + public static RangeUnit Bytes { get; } = new RangeUnit(BytesValue); + + /// + /// Determines if two values are the same. + /// + /// The first to compare. + /// The second to compare. + /// True if and are the same; otherwise, false. + public static bool operator ==(RangeUnit left, RangeUnit right) => left.Equals(right); + + /// + /// Determines if two values are different. + /// + /// The first to compare. + /// The second to compare. + /// True if and are different; otherwise, false. + public static bool operator !=(RangeUnit left, RangeUnit right) => !left.Equals(right); + + /// + /// Converts a string to a . + /// + /// The string value to convert. + public static implicit operator RangeUnit(string value) => new RangeUnit(value); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => obj is RangeUnit other && Equals(other); + + /// + public bool Equals(RangeUnit other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => _value?.GetHashCode() ?? 0; + + /// + public override string ToString() => _value; + } + + /// + /// Inclusive index where the range starts, measured in this instance's . + /// + public long? Start { get; } + + /// + /// Inclusive index where the range ends, measured in this instance's . + /// + public long? End { get; } + + /// + /// Size of this range, measured in this instance's . + /// + public long? Size { get; } + + /// + /// Unit this range is measured in. Generally "bytes". + /// + public RangeUnit Unit { get; } + + public ContentRange(RangeUnit unit, long? start, long? end, long? size) + { + Start = start; + End = end; + Size = size; + Unit = unit; + } + + public static ContentRange Parse(string headerValue) + { + /* Parse header value (e.g. " -/") + * Either side of the "/" can be an asterisk, so possible results include: + * [, , , ] + * [, "*", ] + * [, , , "*"] + * [, "*", "*"] (unsure if possible but not hard to support) + * "End" is the inclusive last byte; e.g. header "bytes 0-7/8" is the entire 8-byte blob + */ + var tokens = headerValue.Split(new char[] { ' ', '-', '/' }); // ["bytes", "", "", ""] + string unit = default; + long? start = default; + long? end = default; + long? size = default; + + try + { + // unit always first and always present + unit = tokens[0]; + + int blobSizeIndex; + if (tokens[1] == WildcardMarker) // [, "*", ( | "*")] + { + blobSizeIndex = 2; + } + else // [, , , ( | "*")] + { + blobSizeIndex = 3; + + start = int.Parse(tokens[1], CultureInfo.InvariantCulture); + end = int.Parse(tokens[2], CultureInfo.InvariantCulture); + } + + var rawSize = tokens[blobSizeIndex]; + if (rawSize != WildcardMarker) + { + size = int.Parse(rawSize, CultureInfo.InvariantCulture); + } + + return new ContentRange(unit, start, end, size); + } + catch (IndexOutOfRangeException) + { + throw Errors.ParsingHttpRangeFailed(); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index f324d547918c5..1122a327b313e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -6,6 +6,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using System.Xml.Serialization; using Azure.Core; using Azure.Core.Pipeline; using Azure.Storage.Blobs.Models; @@ -98,6 +99,7 @@ public PageBlobClient(string connectionString, string blobContainerName, string public PageBlobClient(string connectionString, string blobContainerName, string blobName, BlobClientOptions options) : base(connectionString, blobContainerName, blobName, options) { + AssertNoClientSideEncryption(options); } /// @@ -117,6 +119,7 @@ public PageBlobClient(string connectionString, string blobContainerName, string public PageBlobClient(Uri blobUri, BlobClientOptions options = default) : base(blobUri, options) { + AssertNoClientSideEncryption(options); } /// @@ -139,6 +142,7 @@ public PageBlobClient(Uri blobUri, BlobClientOptions options = default) public PageBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -161,6 +165,7 @@ public PageBlobClient(Uri blobUri, StorageSharedKeyCredential credential, BlobCl public PageBlobClient(Uri blobUri, TokenCredential credential, BlobClientOptions options = default) : base(blobUri, credential, options) { + AssertNoClientSideEncryption(options); } /// @@ -188,9 +193,24 @@ internal PageBlobClient( ClientDiagnostics clientDiagnostics, CustomerProvidedKey? customerProvidedKey, string encryptionScope) - : base(blobUri, pipeline, version, clientDiagnostics, customerProvidedKey, encryptionScope) + : base( + blobUri, + pipeline, + version, + clientDiagnostics, + customerProvidedKey, + clientSideEncryption: default, + encryptionScope) { } + + private static void AssertNoClientSideEncryption(BlobClientOptions options) + { + if (options._clientSideEncryptionOptions != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(PageBlobClient)); + } + } #endregion ctors /// @@ -2610,13 +2630,19 @@ public static partial class SpecializedBlobExtensions /// A new instance. public static PageBlobClient GetPageBlobClient( this BlobContainerClient client, - string blobName) => - new PageBlobClient( + string blobName) + { + if (client.ClientSideEncryption != default) + { + throw Errors.ClientSideEncryption.TypeNotSupported(typeof(PageBlobClient)); + } + return new PageBlobClient( client.Uri.AppendToPath(blobName), client.Pipeline, client.Version, client.ClientDiagnostics, client.CustomerProvidedKey, client.EncryptionScope); + } } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs new file mode 100644 index 0000000000000..a52cc59740bc1 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/src/SpecializedBlobClientOptions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Blobs.Specialized +{ + /// + /// Provides advanced client configuration options for connecting to Azure Blob + /// Storage. + /// +#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. + public class SpecializedBlobClientOptions : BlobClientOptions +#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion + { + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// The of the service API used when + /// making requests. + /// + public SpecializedBlobClientOptions(ServiceVersion version = LatestVersion) : base(version) + { + } + + /// + /// Settings for data encryption when uploading and downloading with a . + /// Client-side encryption adds metadata to your blob which is necessary for decryption. + /// + /// For more information, see . + /// + public ClientSideEncryptionOptions ClientSideEncryption + { + get => _clientSideEncryptionOptions; + set => _clientSideEncryptionOptions = value; + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj index 535e1e5a6f15d..e7a8aedd3d7e7 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj @@ -8,9 +8,13 @@ + + + + + - diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs index a743d937f7fbc..b58f81d629d5f 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobTestBase.cs @@ -226,6 +226,23 @@ public BlobServiceClient GetServiceClient_BlobServiceSas_Snapshot( new Uri($"{TestConfigDefault.BlobServiceEndpoint}?{sasCredentials ?? GetNewBlobServiceSasCredentialsSnapshot(containerName: containerName, blobName: blobName, snapshot: snapshot, sharedKeyCredentials: sharedKeyCredentials ?? GetNewSharedKeyCredentials())}"), GetOptions())); + public Security.KeyVault.Keys.KeyClient GetKeyClient_TargetKeyClient() + => GetKeyClient(TestConfigurations.DefaultTargetKeyVault); + + public TokenCredential GetTokenCredential_TargetKeyClient() + => GetKeyClientTokenCredential(TestConfigurations.DefaultTargetKeyVault); + + private static Security.KeyVault.Keys.KeyClient GetKeyClient(KeyVaultConfiguration config) + => new Security.KeyVault.Keys.KeyClient( + new Uri(config.VaultEndpoint), + GetKeyClientTokenCredential(config)); + + private static TokenCredential GetKeyClientTokenCredential(KeyVaultConfiguration config) + => new Identity.ClientSecretCredential( + config.ActiveDirectoryTenantId, + config.ActiveDirectoryApplicationId, + config.ActiveDirectoryApplicationSecret); + public async Task GetTestContainerAsync( BlobServiceClient service = default, string containerName = default, diff --git a/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs new file mode 100644 index 0000000000000..d4943570dcb94 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs @@ -0,0 +1,673 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Core.TestFramework; +using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; +using Azure.Storage.Blobs.Tests; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Test.Shared; +using Moq; +using NUnit.Framework; +using static Moq.It; + +namespace Azure.Storage.Blobs.Test +{ + public class ClientSideEncryptionTests : BlobTestBase + { + private const string s_algorithmName = "some algorithm name"; + private static readonly CancellationToken s_cancellationToken = new CancellationTokenSource().Token; + + public ClientSideEncryptionTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) + : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) + { + } + + /// + /// Provides encryption functionality clone of client logic, letting us validate the client got it right end-to-end. + /// + private byte[] EncryptData(byte[] data, byte[] key, byte[] iv) + { + using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) + using (var encryptor = aesProvider.CreateEncryptor()) + using (var memStream = new MemoryStream()) + using (var cryptoStream = new CryptoStream(memStream, encryptor, CryptoStreamMode.Write)) + { + cryptoStream.Write(data, 0, data.Length); + cryptoStream.FlushFinalBlock(); + return memStream.ToArray(); + } + } + + private async Task GetKeyvaultIKeyEncryptionKey() + { + var keyClient = GetKeyClient_TargetKeyClient(); + Security.KeyVault.Keys.KeyVaultKey key = await keyClient.CreateRsaKeyAsync( + new Security.KeyVault.Keys.CreateRsaKeyOptions($"CloudRsaKey-{Guid.NewGuid()}", false)); + return new CryptographyClient(key.Id, GetTokenCredential_TargetKeyClient()); + } + + private async Task GetTestContainerEncryptionAsync( + ClientSideEncryptionOptions encryptionOptions, + string containerName = default, + IDictionary metadata = default) + { + // normally set through property on subclass; this is easier to hook up in current test infra with internals access + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; + + containerName ??= GetNewContainerName(); + var service = GetServiceClient_SharedKey(options); + + BlobContainerClient container = InstrumentClient(service.GetBlobContainerClient(containerName)); + await container.CreateAsync(metadata: metadata); + return new DisposingContainer(container); + } + + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + if (IsAsync) + { + keyMock.Setup(k => k.WrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Task.FromResult(Xor(userKeyBytes, key.ToArray()))); + keyMock.Setup(k => k.UnwrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey.ToArray()))); + } + else + { + keyMock.Setup(k => k.WrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, wrappedKey.ToArray())); + } + + return keyMock; + } + + private Mock GetAlwaysFailsKeyResolver(bool throws) + { + var mock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + if (throws) + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns(Task.FromResult(null)); + } + } + else + { + if (throws) + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((IKeyEncryptionKey)null); + } + } + + return mock; + } + + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + resolverMock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + } + else + { + resolverMock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? iKey : throw new Exception("Mock resolver couldn't resolve key id.")); + } + + return resolverMock; + } + + private Mock GetTrackOneIKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.Kid).Returns(keyId); + keyMock.SetupGet(k => k.DefaultKeyWrapAlgorithm).Returns(s_algorithmName); + // track one had async-only key wrapping + keyMock.Setup(k => k.WrapKeyAsync(IsNotNull(), IsAny(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + // track 1 doesn't pass in the algorithm name, it lets the implementation return the default algorithm it chose + .Returns((key, algorithm, cancellationToken) => Task.FromResult(Tuple.Create(Xor(userKeyBytes, key), s_algorithmName))); + keyMock.Setup(k => k.UnwrapKeyAsync(IsNotNull(), s_algorithmName, IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((wrappedKey, algorithm, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey))); + + return keyMock; + } + + private Mock GetTrackOneIKeyResolver(Microsoft.Azure.KeyVault.Core.IKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + resolverMock.Setup(r => r.ResolveKeyAsync(IsNotNull(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((keyId, cancellationToken) => iKey?.Kid == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + + return resolverMock; + } + + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + + [Test] + [LiveOnly] + public void CanSwapKey() + { + var options1 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "foo" + }; + var options2 = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "bar" + }; + + var client = new BlobClient(new Uri("http://someuri.com"), new SpecializedBlobClientOptions() + { + ClientSideEncryption = options1, + }); + + Assert.AreEqual(options1.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options1.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options1.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + + client = client.WithClientSideEncryptionOptions(options2); + + Assert.AreEqual(options2.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options2.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options2.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + } + + [TestCase(16)] // a single cipher block + [TestCase(14)] // a single unalligned cipher block + [TestCase(Constants.KB)] // multiple blocks + [TestCase(Constants.KB - 4)] // multiple unalligned blocks + [TestCase(Constants.MB)] // larger test, increasing likelihood to trigger async extension usage bugs + [LiveOnly] // cannot seed content encryption key + public async Task UploadAsync(long dataSize) + { + var data = GetRandomBuffer(dataSize); + var mockKey = GetIKeyEncryptionKey().Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + var blobName = GetNewBlobName(); + var blob = InstrumentClient(disposable.Container.GetBlobClient(blobName)); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + // download without decrypting + var encryptedDataStream = new MemoryStream(); + await InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials())).DownloadToAsync(encryptedDataStream, cancellationToken: s_cancellationToken); + var encryptedData = encryptedDataStream.ToArray(); + + // encrypt original data manually for comparison + if (!(await blob.GetPropertiesAsync()).Value.Metadata.TryGetValue(Constants.ClientSideEncryption.EncryptionDataKey, out string serialEncryptionData)) + { + Assert.Fail("No encryption metadata present."); + } + EncryptionData encryptionMetadata = EncryptionDataSerializer.Deserialize(serialEncryptionData); + Assert.NotNull(encryptionMetadata, "Never encrypted data."); + + var explicitlyUnwrappedKey = IsAsync // can't instrument this + ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) + : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); + byte[] expectedEncryptedData = EncryptData( + data, + explicitlyUnwrappedKey, + encryptionMetadata.ContentEncryptionIV); + + // compare data + Assert.AreEqual(expectedEncryptedData, encryptedData); + } + } + + [TestCase(16)] // a single cipher block + [TestCase(14)] // a single unalligned cipher block + [TestCase(Constants.KB)] // multiple blocks + [TestCase(Constants.KB - 4)] // multiple unalligned blocks + [LiveOnly] // cannot seed content encryption key + public async Task RoundtripAsync(long dataSize) + { + var data = GetRandomBuffer(dataSize); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + // download with decryption + byte[] downloadData; + using (var stream = new MemoryStream()) + { + await blob.DownloadToAsync(stream, cancellationToken: s_cancellationToken); + downloadData = stream.ToArray(); + } + + // compare data + Assert.AreEqual(data, downloadData); + } + } + + [Test] // multiple unalligned blocks + [LiveOnly] // cannot seed content encryption key + public async Task KeyResolverKicksIn() + { + var data = GetRandomBuffer(Constants.KB); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + string blobName = GetNewBlobName(); + // upload with encryption + await InstrumentClient(disposable.Container.GetBlobClient(blobName)).UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + // download with decryption and no cached key + byte[] downloadData; + using (var stream = new MemoryStream()) + { + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver + }; + await InstrumentClient(new BlobContainerClient(disposable.Container.Uri, GetNewSharedKeyCredentials(), options).GetBlobClient(blobName)).DownloadToAsync(stream, cancellationToken: s_cancellationToken); + downloadData = stream.ToArray(); + } + + // compare data + Assert.AreEqual(data, downloadData); + } + } + + [TestCase(0, 16)] // first block + [TestCase(16, 16)] // not first block + [TestCase(32, 32)] // multiple blocks; IV not at blob start + [TestCase(16, 17)] // overlap end of block + [TestCase(32, 17)] // overlap end of block; IV not at blob start + [TestCase(15, 17)] // overlap beginning of block + [TestCase(31, 17)] // overlap beginning of block; IV not at blob start + [TestCase(15, 18)] // overlap both sides + [TestCase(31, 18)] // overlap both sides; IV not at blob start + [TestCase(16, null)] + [LiveOnly] // cannot seed content encryption key + public async Task PartialDownloadAsync(int offset, int? count) + { + var data = GetRandomBuffer(offset + (count ?? 16) + 32); // ensure we have enough room in original data + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + + // upload with encryption + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + // download range with decryption + byte[] downloadData; // no overload that takes Stream and HttpRange; we must buffer read + Stream downloadStream = (await blob.DownloadAsync(new HttpRange(offset, count), cancellationToken: s_cancellationToken)).Value.Content; + byte[] buffer = new byte[Constants.KB]; + using (MemoryStream stream = new MemoryStream()) + { + int read; + while ((read = downloadStream.Read(buffer, 0, buffer.Length)) > 0) + { + stream.Write(buffer, 0, read); + } + downloadData = stream.ToArray(); + } + + // compare range of original data to downloaded data + var slice = data.Skip(offset); + slice = count.HasValue + ? slice.Take(count.Value) + : slice; + var sliceArray = slice.ToArray(); + Assert.AreEqual(sliceArray, downloadData); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task Track2DownloadTrack1Blob() + { + var data = GetRandomBuffer(Constants.KB); + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object).Object; + await using (var disposable = await GetTestContainerEncryptionAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var track2Blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + + // upload with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Blob = new Microsoft.Azure.Storage.Blob.CloudBlockBlob( + track2Blob.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var track1RequestOptions = new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(mockKey, default) + }; + if (IsAsync) // can't instrument track 1 + { + await track1Blob.UploadFromByteArrayAsync(data, 0, data.Length, default, track1RequestOptions, default, s_cancellationToken); + } + else + { + track1Blob.UploadFromByteArray(data, 0, data.Length, default, track1RequestOptions, default); + } + + // download with track 2 + var downloadStream = new MemoryStream(); + await track2Blob.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); + + // compare original data to downloaded data + Assert.AreEqual(data, downloadStream.ToArray()); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task Track1DownloadTrack2Blob() + { + var data = GetRandomBuffer(Constants.KB); // ensure we have enough room in original data + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetTrackOneIKeyResolver(GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object).Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + var track2Blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + + // upload with track 2 + await track2Blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + // download with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Blob = new Microsoft.Azure.Storage.Blob.CloudBlockBlob( + track2Blob.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var downloadData = new byte[data.Length]; + var track1RequestOptions = new Microsoft.Azure.Storage.Blob.BlobRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Blob.BlobEncryptionPolicy(default, mockKeyResolver) + }; + if (IsAsync) // can't instrument track 1 + { + await track1Blob.DownloadToByteArrayAsync(downloadData, 0, default, track1RequestOptions, default, s_cancellationToken); + } + else + { + track1Blob.DownloadToByteArray(downloadData, 0, default, track1RequestOptions, default); + } + + // compare original data to downloaded data + Assert.AreEqual(data, downloadData); + } + } + + [Test] + [LiveOnly] // need access to keyvault service && cannot seed content encryption key + public async Task RoundtripWithKeyvaultProvider() + { + var data = GetRandomBuffer(Constants.KB); + IKeyEncryptionKey key = await GetKeyvaultIKeyEncryptionKey(); + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = key, + KeyWrapAlgorithm = "RSA-OAEP-256" + })) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + var downloadStream = new MemoryStream(); + await blob.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); + + Assert.AreEqual(data, downloadStream.ToArray()); + } + } + + [TestCase(true)] + [TestCase(false)] + [LiveOnly] + public async Task CannotFindKeyAsync(bool resolverThrows) + { + var data = GetRandomBuffer(Constants.KB); + var mockKey = GetIKeyEncryptionKey().Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + var blob = InstrumentClient(disposable.Container.GetBlobClient(GetNewBlobName())); + await blob.UploadAsync(new MemoryStream(data), cancellationToken: s_cancellationToken); + + bool threw = false; + var resolver = GetAlwaysFailsKeyResolver(resolverThrows); + try + { + // download but can't find key + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = resolver.Object, + KeyWrapAlgorithm = "test" + }; + var encryptedDataStream = new MemoryStream(); + await InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), options)).DownloadToAsync(encryptedDataStream, cancellationToken: s_cancellationToken); + } + catch (MockException e) + { + Assert.Fail(e.Message); + } + catch (Exception) + { + threw = true; + } + finally + { + Assert.IsTrue(threw); + // we already asserted the correct method was called in `catch (MockException e)` + Assert.AreEqual(1, resolver.Invocations.Count); + } + } + } + + // using 5 to setup offsets to avoid any off-by-one confusion in debugging + [TestCase(0, null)] + [TestCase(0, 2 * Constants.ClientSideEncryption.EncryptionBlockSize)] + [TestCase(0, 2 * Constants.ClientSideEncryption.EncryptionBlockSize + 5)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize, Constants.ClientSideEncryption.EncryptionBlockSize)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize, Constants.ClientSideEncryption.EncryptionBlockSize + 5)] + [TestCase(Constants.ClientSideEncryption.EncryptionBlockSize + 5, 2 * Constants.ClientSideEncryption.EncryptionBlockSize)] + [LiveOnly] + public async Task AppropriateRangeDownloadOnPlaintext(int rangeOffset, int? rangeLength) + { + var data = GetRandomBuffer(rangeOffset + (rangeLength ?? Constants.KB) + Constants.ClientSideEncryption.EncryptionBlockSize); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey().Object).Object; + await using (var disposable = await GetTestContainerAsync()) + { + // upload plaintext + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + await blob.UploadAsync(new MemoryStream(data)); + + // download plaintext range with encrypted client + var cryptoClient = InstrumentClient(new BlobClient(blob.Uri, GetNewSharedKeyCredentials(), new SpecializedBlobClientOptions() + { + ClientSideEncryption = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver + } + })); + var desiredRange = new HttpRange(rangeOffset, rangeLength); + var response = await cryptoClient.DownloadAsync(desiredRange); + + // assert we recieved the data we requested + int expectedLength = rangeLength ?? data.Length - rangeOffset; + var memoryStream = new MemoryStream(); + await response.Value.Content.CopyToAsync(memoryStream); + var recievedData = memoryStream.ToArray(); + Assert.AreEqual(expectedLength, recievedData.Length); + for (int i = 0; i < recievedData.Length; i++) + { + Assert.AreEqual(data[i + rangeOffset], recievedData[i]); + } + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + [Ignore("stress test")] + public async Task StressAsync() + { + static async Task RoundTripData(BlobClient client, byte[] data) + { + using (var dataStream = new MemoryStream(data)) + { + await client.UploadAsync(dataStream, cancellationToken: s_cancellationToken); + } + + using (var downloadStream = new MemoryStream()) + { + await client.DownloadToAsync(downloadStream, cancellationToken: s_cancellationToken); + return downloadStream.ToArray(); + } + } + + var data = GetRandomBuffer(10 * Constants.MB); + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestContainerEncryptionAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var downloadTasks = new List>(); + foreach (var _ in Enumerable.Range(0, 10)) + { + var blob = disposable.Container.GetBlobClient(GetNewBlobName()); + + downloadTasks.Add(RoundTripData(blob, data)); + } + + var downloads = await Task.WhenAll(downloadTasks); + + foreach (byte[] downloadData in downloads) + { + Assert.AreEqual(data, downloadData); + } + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs deleted file mode 100644 index f0b1d054a9005..0000000000000 --- a/sdk/storage/Azure.Storage.Blobs/tests/EncryptedBlockBlobClientTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.IO; -using System.Threading.Tasks; -using Azure.Core.TestFramework; -using Azure.Storage.Blobs.Specialized; -using Azure.Storage.Test.Shared; -using NUnit.Framework; - -namespace Azure.Storage.Blobs.Test -{ - public class EncryptedBlockBlobClientTests : BlobTestBase - { - public EncryptedBlockBlobClientTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) - : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) - { - } - - // Placeholder test to verify things work end-to-end - [Test] - public async Task DeleteAsync() - { - await using DisposingContainer test = await GetTestContainerAsync(); - - // First upload a regular block blob - var blobName = GetNewBlobName(); - var blob = InstrumentClient(test.Container.GetBlockBlobClient(blobName)); - var data = GetRandomBuffer(Constants.KB); - using var stream = new MemoryStream(data); - await blob.UploadAsync(stream); - - // Create an EncryptedBlockBlobClient pointing at the same blob - var encryptedBlob = InstrumentClient( - new EncryptedBlockBlobClient( - blob.Uri, - new StorageSharedKeyCredential( - TestConfigDefault.AccountName, - TestConfigDefault.AccountKey), - GetOptions())); - - // Delete the blob - var response = await encryptedBlob.DeleteAsync(); - Assert.IsNotNull(response.Headers.RequestId); - } - } -} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs b/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs new file mode 100644 index 0000000000000..534feb1877b92 --- /dev/null +++ b/sdk/storage/Azure.Storage.Blobs/tests/MockStream.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; + +namespace Azure.Storage.Blobs.Cryptography.Tests +{ + /// + /// Read-only, unseekable Stream where the 0-indexed nth byte read will have the value n. + /// + internal class MockStream : Stream + { + private byte _next = 0; + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => _next; + + public override long Position { get => _next; set => throw new NotSupportedException(); } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) + { + for (int i = offset; i < offset + count; i++) + { + buffer[i] = _next++; + } + return count; + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs index 1b8cf3faa6ddb..2585c1acda246 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/SasQueryParametersTests.cs @@ -3,7 +3,6 @@ using System; using Azure.Storage.Sas; -using Azure.Storage.Test.Shared; using NUnit.Framework; namespace Azure.Storage.Blobs.Test diff --git a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs index 9768ccbc8f975..9902d1ac7bc2e 100644 --- a/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Common/api/Azure.Storage.Common.netstandard2.0.cs @@ -1,5 +1,17 @@ namespace Azure.Storage { + public partial class ClientSideEncryptionOptions + { + public ClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) { } + public Azure.Storage.ClientSideEncryptionVersion EncryptionVersion { get { throw null; } } + public Azure.Core.Cryptography.IKeyEncryptionKey KeyEncryptionKey { get { throw null; } set { } } + public Azure.Core.Cryptography.IKeyEncryptionKeyResolver KeyResolver { get { throw null; } set { } } + public string KeyWrapAlgorithm { get { throw null; } set { } } + } + public enum ClientSideEncryptionVersion + { + V1_0 = 1, + } public partial class StorageSharedKeyCredential { public StorageSharedKeyCredential(string accountName, string accountKey) { } diff --git a/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj b/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj index 57daa4b8b3046..c0cf0883c9248 100644 --- a/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj +++ b/sdk/storage/Azure.Storage.Common/src/Azure.Storage.Common.csproj @@ -21,6 +21,7 @@ + diff --git a/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs new file mode 100644 index 0000000000000..f2d4725eb050d --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/ClientSideEncryptionVersion.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage +{ +#pragma warning disable CA1707 // Identifiers should not contain underscores + /// + /// The version of clientside encryption to use. + /// + public enum ClientSideEncryptionVersion + { + /// + /// 1.0 + /// + V1_0 = 1 + } +#pragma warning restore CA1707 // Identifiers should not contain underscores +} diff --git a/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs new file mode 100644 index 0000000000000..3d03a53d14927 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/ClientsideEncryptionOptions.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Core.Cryptography; + +namespace Azure.Storage +{ + /// + /// Provides the client configuration options for connecting to Azure Blob using clientside encryption. + /// + public class ClientSideEncryptionOptions + { + // NOTE there is a non-public Clone method for this class, compile-included into relevant packages from + // Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs + + /// + /// The version of clientside encryption to use. + /// + public ClientSideEncryptionVersion EncryptionVersion { get; } + + /// + /// Required for upload operations. + /// The key used to wrap the generated content encryption key. + /// For more information, see https://docs.microsoft.com/en-us/azure/storage/common/storage-client-side-encryption. + /// + public IKeyEncryptionKey KeyEncryptionKey { get; set; } + + /// + /// Required for download operations. + /// Fetches the correct key encryption key to unwrap the downloaded content encryption key. + /// For more information, see https://docs.microsoft.com/en-us/azure/storage/common/storage-client-side-encryption. + /// + public IKeyEncryptionKeyResolver KeyResolver { get; set; } + + /// + /// Required for upload operations. + /// The algorithm identifier to use when wrapping the content encryption key. This is passed into + /// + /// and its async counterpart. + /// + public string KeyWrapAlgorithm { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The version of clientside encryption to use. + public ClientSideEncryptionOptions(ClientSideEncryptionVersion version) + { + EncryptionVersion = version; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs new file mode 100644 index 0000000000000..6c919630f84f5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Cryptography +{ + internal class ClientSideDecryptor + { + /// + /// Clients that can upload data have a key encryption key stored on them. Checking if + /// a cached key exists and matches a given saves a call + /// to the external key resolver implementation when available. + /// + private readonly IKeyEncryptionKey _potentialCachedIKeyEncryptionKey; + + /// + /// Resolver to fetch the key encryption key. + /// + private readonly IKeyEncryptionKeyResolver _keyResolver; + + public ClientSideDecryptor(ClientSideEncryptionOptions options) + { + _potentialCachedIKeyEncryptionKey = options.KeyEncryptionKey; + _keyResolver = options.KeyResolver; + } + + /// + /// Decrypts the given stream if decryption information is provided. + /// Does not shave off unwanted start/end bytes, but will shave off padding. + /// + /// Stream to decrypt. + /// + /// Encryption metadata and wrapped content encryption key. + /// + /// + /// Whether to use the first block of the stream for the IV instead of the value in + /// . Generally for partial blob downloads where the + /// previous block of the ciphertext is the IV for the next. + /// + /// + /// Whether to ignore padding. Generally for partial blob downloads where the end of + /// the blob (where the padding occurs) was not downloaded. + /// + /// Whether to perform this function asynchronously. + /// + /// + /// Decrypted plaintext. + /// + /// + /// Exceptions thrown based on implementations of and + /// . + /// + public async Task DecryptInternal( + Stream ciphertext, + EncryptionData encryptionData, + bool ivInStream, + bool noPadding, + bool async, + CancellationToken cancellationToken) + { + switch (encryptionData.EncryptionAgent.EncryptionVersion) + { + case ClientSideEncryptionVersion.V1_0: + return await DecryptInternalV1_0( + ciphertext, + encryptionData, + ivInStream, + noPadding, + async, + cancellationToken).ConfigureAwait(false); + default: + throw Errors.ClientSideEncryption.BadEncryptionAgent(encryptionData.EncryptionAgent.EncryptionVersion.ToString()); + } + } + + /// + /// Decrypts the given stream if decryption information is provided. + /// Does not shave off unwanted start/end bytes, but will shave off padding. + /// + /// Stream to decrypt. + /// + /// Encryption metadata and wrapped content encryption key. + /// + /// + /// Whether to use the first block of the stream for the IV instead of the value in + /// . Generally for partial blob downloads where the + /// previous block of the ciphertext is the IV for the next. + /// + /// + /// Whether to ignore padding. Generally for partial blob downloads where the end of + /// the blob (where the padding occurs) was not downloaded. + /// + /// Whether to perform this function asynchronously. + /// + /// + /// Decrypted plaintext. + /// + /// + /// Exceptions thrown based on implementations of and + /// . + /// + private async Task DecryptInternalV1_0( + Stream ciphertext, + EncryptionData encryptionData, + bool ivInStream, + bool noPadding, + bool async, + CancellationToken cancellationToken) + { + var contentEncryptionKey = await GetContentEncryptionKeyAsync( + encryptionData, + async, + cancellationToken).ConfigureAwait(false); + + Stream plaintext; + //int read = 0; + if (encryptionData != default) + { + byte[] IV; + if (!ivInStream) + { + IV = encryptionData.ContentEncryptionIV; + } + else + { + IV = new byte[Constants.ClientSideEncryption.EncryptionBlockSize]; + if (async) + { + await ciphertext.ReadAsync(IV, 0, IV.Length, cancellationToken).ConfigureAwait(false); + } + else + { + ciphertext.Read(IV, 0, IV.Length); + } + //read = IV.Length; + } + + plaintext = WrapStream( + ciphertext, + contentEncryptionKey.ToArray(), + encryptionData, + IV, + noPadding); + } + else + { + plaintext = ciphertext; + } + + return plaintext; + } + +#pragma warning disable CS1587 // XML comment is not placed on a valid language element + /// + /// Returns the content encryption key for blob. First tries to get the key encryption key from KeyResolver, + /// then falls back to IKey stored on this EncryptionPolicy. Unwraps the content encryption key with the + /// correct key wrapper. + /// + /// The encryption data. + /// Whether to perform asynchronously. + /// + /// + /// Encryption key as a byte array. + /// + /// + /// Exceptions thrown based on implementations of and + /// . + /// + private async Task> GetContentEncryptionKeyAsync( +#pragma warning restore CS1587 // XML comment is not placed on a valid language element + EncryptionData encryptionData, + bool async, + CancellationToken cancellationToken) + { + IKeyEncryptionKey key = default; + + // If we already have a local key and it is the correct one, use that. + if (encryptionData.WrappedContentKey.KeyId == _potentialCachedIKeyEncryptionKey?.KeyId) + { + key = _potentialCachedIKeyEncryptionKey; + } + // Otherwise, use the resolver. + else if (_keyResolver != null) + { + key = async + ? await _keyResolver.ResolveAsync(encryptionData.WrappedContentKey.KeyId, cancellationToken).ConfigureAwait(false) + : _keyResolver.Resolve(encryptionData.WrappedContentKey.KeyId, cancellationToken); + } + + // We throw for every other reason that decryption couldn't happen. Throw a reasonable + // exception here instead of nullref. + if (key == default) + { + throw Errors.ClientSideEncryption.KeyNotFound(encryptionData.WrappedContentKey.KeyId); + } + + return async + ? await key.UnwrapKeyAsync( + encryptionData.WrappedContentKey.Algorithm, + encryptionData.WrappedContentKey.EncryptedKey, + cancellationToken).ConfigureAwait(false) + : key.UnwrapKey( + encryptionData.WrappedContentKey.Algorithm, + encryptionData.WrappedContentKey.EncryptedKey, + cancellationToken); + } + + + /// + /// Wraps a stream of ciphertext to stream plaintext. + /// + /// + /// + /// + /// + /// + /// + private static Stream WrapStream( + Stream contentStream, + byte[] contentEncryptionKey, + EncryptionData encryptionData, + byte[] iv, + bool noPadding) + { + if (encryptionData.EncryptionAgent.EncryptionAlgorithm == ClientSideEncryptionAlgorithm.AesCbc256) + { + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider()) + { + aesProvider.IV = iv ?? encryptionData.ContentEncryptionIV; + aesProvider.Key = contentEncryptionKey; + + if (noPadding) + { + aesProvider.Padding = PaddingMode.None; + } + + return new CryptoStream(contentStream, aesProvider.CreateDecryptor(), CryptoStreamMode.Read); + } + } + + throw Errors.ClientSideEncryption.BadEncryptionAlgorithm(encryptionData.EncryptionAgent.EncryptionAlgorithm.ToString()); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs new file mode 100644 index 0000000000000..ed09ed9c2da87 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionOptionsExtensions.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography +{ + internal static class ClientSideEncryptionOptionsExtensions + { + /// + /// Extension method to clone an instance of . + /// + /// + /// + public static ClientSideEncryptionOptions Clone(this ClientSideEncryptionOptions options) + { + var newOptions = new ClientSideEncryptionOptions(options.EncryptionVersion); + CopyOptions(options, newOptions); + return newOptions; + } + + /// + /// Copies all properties from one instance to another. It cannot copy + /// ; + /// that is the responsibility of the caller who made the instance. + /// + /// Object to copy from. + /// Object to copy to. + /// + /// This functionality has been pulled out to be accessible by other + /// clone methods available on subclasses. They need the ability to + /// instantiate the subclass destination first before copying over the + /// properties. + /// + internal static void CopyOptions(ClientSideEncryptionOptions source, ClientSideEncryptionOptions destination) + { + destination.KeyEncryptionKey = source.KeyEncryptionKey; + destination.KeyResolver = source.KeyResolver; + destination.KeyWrapAlgorithm = source.KeyWrapAlgorithm; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs new file mode 100644 index 0000000000000..ca5e03ac3ee11 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + internal static class ClientSideEncryptionVersionExtensions + { + public static class ClientSideEncryptionVersionString + { + public const string V1_0 = "1.0"; + } + + public static string Serialize(this ClientSideEncryptionVersion version) + { + switch (version) + { + case ClientSideEncryptionVersion.V1_0: + return ClientSideEncryptionVersionString.V1_0; + default: + // sanity check; serialize is in this file to make it easy to add the serialization cases + throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported(); + } + } + + public static ClientSideEncryptionVersion ToClientSideEncryptionVersion(this string versionString) + { + switch (versionString) + { + case ClientSideEncryptionVersionString.V1_0: + return ClientSideEncryptionVersion.V1_0; + default: + // This library doesn't support the stated encryption version + throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported(versionString); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs new file mode 100644 index 0000000000000..d4fbcfdecb5e4 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptor.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Cryptography +{ + internal class ClientSideEncryptor + { + private readonly IKeyEncryptionKey _keyEncryptionKey; + private readonly string _keyWrapAlgorithm; + + public ClientSideEncryptor(ClientSideEncryptionOptions options) + { + _keyEncryptionKey = options.KeyEncryptionKey; + _keyWrapAlgorithm = options.KeyWrapAlgorithm; + } + + /// + /// Wraps the given read-stream in a CryptoStream and provides the metadata used to create + /// that stream. + /// + /// Stream to wrap. + /// Whether to wrap the content encryption key asynchronously. + /// Cancellation token. + /// The wrapped stream to read from and the encryption metadata for the wrapped stream. + public async Task<(Stream ciphertext, EncryptionData encryptionData)> EncryptInternal( + Stream plaintext, + bool async, + CancellationToken cancellationToken) + { + if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); + } + + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + Stream ciphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: _keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: _keyEncryptionKey, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + ciphertext = new CryptoStream( + plaintext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Read); + } + + return (ciphertext, encryptionData); + } + + /// + /// Encrypts the given stream and provides the metadata used to encrypt. This method writes to a memory stream, + /// optimized for known-size data that will already be buffered in memory. + /// + /// Stream to encrypt. + /// Whether to wrap the content encryption key asynchronously. + /// Cancellation token. + /// The encrypted data and the encryption metadata for the wrapped stream. + public async Task<(byte[] ciphertext, EncryptionData encryptionData)> BufferedEncryptInternal( + Stream plaintext, + bool async, + CancellationToken cancellationToken) + { + if (_keyEncryptionKey == default || _keyWrapAlgorithm == default) + { + throw Errors.ClientSideEncryption.MissingRequiredEncryptionResources(nameof(_keyEncryptionKey), nameof(_keyWrapAlgorithm)); + } + + var generatedKey = CreateKey(Constants.ClientSideEncryption.EncryptionKeySizeBits); + EncryptionData encryptionData = default; + var ciphertext = new MemoryStream(); + byte[] bufferedCiphertext = default; + + using (AesCryptoServiceProvider aesProvider = new AesCryptoServiceProvider() { Key = generatedKey }) + { + encryptionData = await EncryptionData.CreateInternalV1_0( + contentEncryptionIv: aesProvider.IV, + keyWrapAlgorithm: _keyWrapAlgorithm, + contentEncryptionKey: generatedKey, + keyEncryptionKey: _keyEncryptionKey, + async: async, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var transformStream = new CryptoStream( + ciphertext, + aesProvider.CreateEncryptor(), + CryptoStreamMode.Write); + + if (async) + { + await plaintext.CopyToAsync(transformStream).ConfigureAwait(false); + } + else + { + plaintext.CopyTo(transformStream); + } + + transformStream.FlushFinalBlock(); + + bufferedCiphertext = ciphertext.ToArray(); + } + + return (bufferedCiphertext, encryptionData); + } + + /// + /// Securely generate a key. + /// + /// Key size. + /// The generated key bytes. + private static byte[] CreateKey(int numBits) + { + using (var secureRng = new RNGCryptoServiceProvider()) + { + var buff = new byte[numBits / 8]; + secureRng.GetBytes(buff); + return buff; + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs new file mode 100644 index 0000000000000..f54bacc82a6c7 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/ClientSideEncryptionAlgorithm.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.ComponentModel; + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Specifies the encryption algorithm used to encrypt and decrypt a blob. + /// + internal readonly struct ClientSideEncryptionAlgorithm + { + internal const string AesCbc256Value = "AES_CBC_256"; + + private readonly string _value; + + /// + /// Initializes a new instance of the structure. + /// + /// The string value of the instance. + public ClientSideEncryptionAlgorithm(string value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + /// + /// AES-CBC using a 256 bit key. + /// + public static ClientSideEncryptionAlgorithm AesCbc256 { get; } = new ClientSideEncryptionAlgorithm(AesCbc256Value); + + /// + /// Determines if two values are the same. + /// + /// The first to compare. + /// The second to compare. + /// True if and are the same; otherwise, false. + public static bool operator ==(ClientSideEncryptionAlgorithm left, ClientSideEncryptionAlgorithm right) => left.Equals(right); + + /// + /// Determines if two values are different. + /// + /// The first to compare. + /// The second to compare. + /// True if and are different; otherwise, false. + public static bool operator !=(ClientSideEncryptionAlgorithm left, ClientSideEncryptionAlgorithm right) => !left.Equals(right); + + /// + /// Converts a string to a . + /// + /// The string value to convert. + public static implicit operator ClientSideEncryptionAlgorithm(string value) => new ClientSideEncryptionAlgorithm(value); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object obj) => obj is ClientSideEncryptionAlgorithm other && Equals(other); + + /// + public bool Equals(ClientSideEncryptionAlgorithm other) => string.Equals(_value, other._value, StringComparison.Ordinal); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => _value?.GetHashCode() ?? 0; + + /// + public override string ToString() => _value; + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs new file mode 100644 index 0000000000000..c33ab8407c85c --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the encryption agent stored on the service. + /// + internal class EncryptionAgent + { + /// + /// The protocol version used for encryption. + /// + public ClientSideEncryptionVersion EncryptionVersion { get; set; } + + /// + /// The algorithm used for encryption. + /// + public ClientSideEncryptionAlgorithm EncryptionAlgorithm { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs new file mode 100644 index 0000000000000..ead5f7c6d42c6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the encryption data that is stored on the service. + /// + internal class EncryptionData + { + /// + /// The blob encryption mode. + /// + public string EncryptionMode { get; set; } + + /// + /// A object that stores the wrapping algorithm, key identifier and the encrypted key. + /// + public KeyEnvelope WrappedContentKey { get; set; } + + /// + /// The encryption agent. + /// + public EncryptionAgent EncryptionAgent { get; set; } + + /// + /// The content encryption IV. + /// + public byte[] ContentEncryptionIV { get; set; } + +#pragma warning disable CA2227 // Collection properties should be read only + /// + /// Metadata for encryption. Currently used only for storing the encryption library, but may contain other data. + /// + public Metadata KeyWrappingMetadata { get; set; } +#pragma warning restore CA2227 // Collection properties should be read only + + internal static async Task CreateInternalV1_0( + byte[] contentEncryptionIv, + string keyWrapAlgorithm, + byte[] contentEncryptionKey, + IKeyEncryptionKey keyEncryptionKey, + bool async, + CancellationToken cancellationToken) + => new EncryptionData() + { + EncryptionMode = Constants.ClientSideEncryption.EncryptionMode, + ContentEncryptionIV = contentEncryptionIv, + EncryptionAgent = new EncryptionAgent() + { + EncryptionAlgorithm = ClientSideEncryptionAlgorithm.AesCbc256, + EncryptionVersion = ClientSideEncryptionVersion.V1_0 + }, + KeyWrappingMetadata = new Dictionary() + { + { Constants.ClientSideEncryption.AgentMetadataKey, AgentString } + }, + WrappedContentKey = new KeyEnvelope() + { + Algorithm = keyWrapAlgorithm, + EncryptedKey = async + ? await keyEncryptionKey.WrapKeyAsync(keyWrapAlgorithm, contentEncryptionKey, cancellationToken).ConfigureAwait(false) + : keyEncryptionKey.WrapKey(keyWrapAlgorithm, contentEncryptionKey, cancellationToken), + KeyId = keyEncryptionKey.KeyId + } + }; + + /// + /// Singleton string identifying this encryption library. + /// + private static string AgentString { get; } = GenerateAgentString(); + + private static string GenerateAgentString() + { + Assembly assembly = typeof(EncryptionData).Assembly; + var platformInformation = $"({RuntimeInformation.FrameworkDescription}; {RuntimeInformation.OSDescription})"; + return $"azsdk-net-{assembly.GetName().Name}/{assembly.GetCustomAttribute().InformationalVersion} {platformInformation}"; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs new file mode 100644 index 0000000000000..96ed771efc32c --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionDataSerializer.cs @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; + +namespace Azure.Storage.Cryptography.Models +{ + internal static class EncryptionDataSerializer + { + private const string EncryptionAgent_EncryptionVersionName = "Protocol"; + + #region Serialize + + /// + /// Serializes an EncryptionData instance into JSON. + /// + /// Data to serialize. + /// The JSON string. + public static string Serialize(EncryptionData data) + { + return Encoding.UTF8.GetString(SerializeEncryptionData(data).ToArray()); + } + + /// + /// Serializes an EncryptionData instance into JSON. + /// + /// Data to serialize. + /// The JSON UTF8 bytes. + private static ReadOnlyMemory SerializeEncryptionData(EncryptionData data) + { + var writer = new Core.ArrayBufferWriter(); + using var json = new Utf8JsonWriter(writer); + + json.WriteStartObject(); + WriteEncryptionData(json, data); + json.WriteEndObject(); + + json.Flush(); + return writer.WrittenMemory; + } + + /// + /// Serializes an EncryptionData instance into JSON and writes it to the given JSON writer. + /// + /// The writer to write the serialization to. + /// Data to serialize. + public static void WriteEncryptionData(Utf8JsonWriter json, EncryptionData data) + { + json.WriteString(nameof(data.EncryptionMode), data.EncryptionMode); + + json.WriteStartObject(nameof(data.WrappedContentKey)); + WriteWrappedKey(json, data.WrappedContentKey); + json.WriteEndObject(); + + json.WriteStartObject(nameof(data.EncryptionAgent)); + WriteEncryptionAgent(json, data.EncryptionAgent); + json.WriteEndObject(); + + json.WriteString(nameof(data.ContentEncryptionIV), Convert.ToBase64String(data.ContentEncryptionIV)); + + json.WriteStartObject(nameof(data.KeyWrappingMetadata)); + WriteDictionary(json, data.KeyWrappingMetadata); + json.WriteEndObject(); + } + + private static void WriteWrappedKey(Utf8JsonWriter json, KeyEnvelope key) + { + json.WriteString(nameof(key.KeyId), key.KeyId); + json.WriteString(nameof(key.EncryptedKey), Convert.ToBase64String(key.EncryptedKey)); + json.WriteString(nameof(key.Algorithm), key.Algorithm); + } + + private static void WriteEncryptionAgent(Utf8JsonWriter json, EncryptionAgent encryptionAgent) + { + json.WriteString(EncryptionAgent_EncryptionVersionName, encryptionAgent.EncryptionVersion.Serialize()); + json.WriteString(nameof(encryptionAgent.EncryptionAlgorithm), encryptionAgent.EncryptionAlgorithm.ToString()); + } + + private static void WriteDictionary(Utf8JsonWriter json, IDictionary dictionary) + { + foreach (var entry in dictionary) + { + json.WriteString(entry.Key, entry.Value); + } + } + #endregion + + #region Deserialize + /// + /// Deserializes an EncryptionData instance from JSON. + /// + /// The serialized data string. + /// The instance. + public static EncryptionData Deserialize(string serializedData) + { + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(serializedData)); + return DeserializeEncryptionData(ref reader); + } + + /// + /// Reads an EncryptionData instance from a JSON reader. + /// + /// The reader to parse an EncryptionData isntance from. + /// The instance. + public static EncryptionData DeserializeEncryptionData(ref Utf8JsonReader reader) + { + using JsonDocument json = JsonDocument.ParseValue(ref reader); + JsonElement root = json.RootElement; + return ReadEncryptionData(root); + } + + /// + /// Reads an EncryptionData instance from a parsed JSON object. + /// + /// The JSON object model. + /// The instance. + public static EncryptionData ReadEncryptionData(JsonElement root) + { + var data = new EncryptionData(); + foreach (var property in root.EnumerateObject()) + { + ReadPropertyValue(data, property); + } + return data; + } + + private static void ReadPropertyValue(EncryptionData data, JsonProperty property) + { + if (property.NameEquals(nameof(data.EncryptionMode))) + { + data.EncryptionMode = property.Value.GetString(); + } + else if (property.NameEquals(nameof(data.WrappedContentKey))) + { + var key = new KeyEnvelope(); + foreach (var subProperty in property.Value.EnumerateObject()) + { + ReadPropertyValue(key, subProperty); + } + data.WrappedContentKey = key; + } + else if (property.NameEquals(nameof(data.EncryptionAgent))) + { + var agent = new EncryptionAgent(); + foreach (var subProperty in property.Value.EnumerateObject()) + { + ReadPropertyValue(agent, subProperty); + } + data.EncryptionAgent = agent; + } + else if (property.NameEquals(nameof(data.ContentEncryptionIV))) + { + data.ContentEncryptionIV = Convert.FromBase64String(property.Value.GetString()); + } + else if (property.NameEquals(nameof(data.KeyWrappingMetadata))) + { + var metadata = new Dictionary(); + foreach (var entry in property.Value.EnumerateObject()) + { + metadata.Add(entry.Name, entry.Value.GetString()); + } + data.KeyWrappingMetadata = metadata; + } + } + + private static void ReadPropertyValue(KeyEnvelope key, JsonProperty property) + { + if (property.NameEquals(nameof(key.Algorithm))) + { + key.Algorithm = property.Value.GetString(); + } + else if (property.NameEquals(nameof(key.EncryptedKey))) + { + key.EncryptedKey = Convert.FromBase64String(property.Value.GetString()); + } + else if (property.NameEquals(nameof(key.KeyId))) + { + key.KeyId = property.Value.GetString(); + } + } + + private static void ReadPropertyValue(EncryptionAgent agent, JsonProperty property) + { + if (property.NameEquals(nameof(agent.EncryptionAlgorithm))) + { + agent.EncryptionAlgorithm = new ClientSideEncryptionAlgorithm(property.Value.GetString()); + } + else if (property.NameEquals(EncryptionAgent_EncryptionVersionName)) + { + agent.EncryptionVersion = property.Value.GetString().ToClientSideEncryptionVersion(); + } + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs new file mode 100644 index 0000000000000..ce84702e548e0 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/KeyEnvelope.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Storage.Cryptography.Models +{ + /// + /// Represents the envelope key details JSON schema stored on the service. + /// In the envelope technique, a securely generated content encryption key (CEK) is generated + /// for every encryption operation. It is then encrypted (wrapped) with the user-provided key + /// encryption key (KEK), using a key-wrap algorithm. The wrapped CEK is stored with the + /// encrypted data, and needs the KEK to be unwrapped. The KEK and key-wrapping operation is + /// never seen by this SDK. + /// + internal class KeyEnvelope + { + /// + /// The key identifier string. + /// + public string KeyId { get; set; } + + /// + /// The encrypted content encryption key. + /// + public byte[] EncryptedKey { get; set; } + + /// + /// The algorithm used to wrap the content encryption key. + /// + public string Algorithm { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 61299182afba1..a7f8b5636d371 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -391,6 +391,29 @@ internal static class AccountResources } } + internal static class ClientSideEncryption + { + public const ClientSideEncryptionVersion CurrentVersion = ClientSideEncryptionVersion.V1_0; + + public const string AgentMetadataKey = "EncryptionLibrary"; + + public const string AesCbcPkcs5Padding = "AES/CBC/PKCS5Padding"; + + public const string AesCbcNoPadding = "AES/CBC/NoPadding"; + + public const string Aes = "AES"; + + public const string EncryptionDataKey = "encryptiondata"; + + public const string EncryptionMode = "FullBlob"; + + public const int EncryptionBlockSize = 16; + + public const int EncryptionKeySizeBits = 256; + + public const string XMsRange = "x-ms-range"; + } + /// /// XML Element Name constant values. /// diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index d4bbcd239d49a..7148d5626bbce 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -3,6 +3,7 @@ using System; using System.Globalization; +using System.Linq; using System.Security.Authentication; using Azure.Core.Pipeline; @@ -84,5 +85,34 @@ public static void VerifyHttpsTokenAuth(Uri uri) throw new ArgumentException("Cannot use TokenCredential without HTTPS."); } } + + public static class ClientSideEncryption + { + public static InvalidOperationException ClientSideEncryptionVersionNotSupported(string versionString = default) + => new InvalidOperationException("This library does not support the given version of client-side encryption." + + versionString == default ? "" : $" Version ID = {versionString}"); + + public static InvalidOperationException TypeNotSupported(Type type) + => new InvalidOperationException( + $"Client-side encryption is not supported for type \"{type.FullName}\". " + + "Please use a supported client type or create this client without specifying client-side encryption options."); + + public static InvalidOperationException MissingRequiredEncryptionResources(params string[] resourceNames) + => new InvalidOperationException("Cannot encrypt without specifying " + string.Join(",", resourceNames.AsEnumerable())); + + public static ArgumentException KeyNotFound(string keyId) + => new ArgumentException($"Resolution of id {keyId} returned null."); + + public static ArgumentException BadEncryptionAgent(string agent) + => new ArgumentException("Invalid Encryption Agent. This version of the client library does not understand" + + $"the Encryption Agent protocol \"{agent}\" set on the blob."); + + public static ArgumentException BadEncryptionAlgorithm(string algorithm) + => new ArgumentException($"Invalid Encryption Algorithm \"{algorithm}\" found on the resource. This version of the client" + + "library does not support the given encryption algorithm."); + + public static InvalidOperationException MissingEncryptionMetadata(string field) + => new InvalidOperationException($"Missing field \"{field}\" in encryption metadata"); + } } -} \ No newline at end of file +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs new file mode 100644 index 0000000000000..60267b05bc23b --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/WindowStream.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Storage.Shared +{ + /// + /// Exposes a predetermined slice of a larger stream using the same Stream interface. + /// + internal class WindowStream : Stream + { + private Stream InnerStream { get; } + private long WindowLength { get; } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + private long _position = 0; + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + + /// + /// Constructs a window of an underlying stream. While we can construct windows of unseekable + /// streams (no access to Length or Position), we must know the stream length to create a valid window. + /// + /// + /// Potentialy unseekable stream to expose a window of. + /// + /// + /// Maximum size of this window. + /// + public WindowStream(Stream stream, long windowLength) + { + InnerStream = stream; + WindowLength = windowLength; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override int ReadByte() + { + if (WindowLength - _position == 0) + { + return -1; + } + + int val = InnerStream.ReadByte(); + if (val != -1) + { + _position++; + } + + return val; + } + + public override int Read(byte[] buffer, int offset, int count) + { + count = (int)Math.Min(count, WindowLength - _position); + if (count == 0) + { + return 0; + } + int result = InnerStream.Read(buffer, offset, count); + _position += result; + return result; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + count = (int)Math.Min(count, WindowLength - _position); + if (count == 0) + { + return 0; + } + int result = await InnerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + _position += result; + return result; + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void WriteByte(byte value) => throw new NotSupportedException(); + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs new file mode 100644 index 0000000000000..aa329692928b3 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/KeyVaultConfiguration.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Azure.Storage.Test +{ + public class KeyVaultConfiguration + { + public string VaultName { get; private set; } + public string VaultEndpoint { get; private set; } + public string ActiveDirectoryApplicationId { get; private set; } + public string ActiveDirectoryTenantId { get; private set; } + public string ActiveDirectoryApplicationSecret { get; private set; } + public string ActiveDirectoryAuthEndpoint { get; private set; } + + /// + /// Parse an XML representation into a TenantConfiguration value. + /// + /// The XML element to parse. + /// A TenantConfiguration value. + public static KeyVaultConfiguration Parse(XElement tenant) + { + string Get(string name) => (string)tenant.Element(name); + + return new KeyVaultConfiguration + { + VaultName = Get("VaultName"), + VaultEndpoint = Get("VaultEndpoint"), + ActiveDirectoryApplicationId = Get("ActiveDirectoryApplicationId"), + ActiveDirectoryApplicationSecret = Get("ActiveDirectoryApplicationSecret"), + ActiveDirectoryTenantId = Get("ActiveDirectoryTenantId"), + ActiveDirectoryAuthEndpoint = Get("ActiveDirectoryAuthEndpoint") + }; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs index 523bb68b7da47..9366d333bb5b1 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurations.cs @@ -36,6 +36,15 @@ public class TestConfigurations /// private IDictionary Tenants { get; set; } + /// + /// Gets or sets a mapping of keyvault names to definitions. The + /// Target*TenantName properties define the keys to use with this + /// dictionary. You should only access the tenants via the GetTenant + /// method that will Assert.Inconclusive if the desired tenant wasn't + /// defined in this configuration. + /// + private IDictionary KeyVaults { get; set; } + /// /// Gets the name of the tenant in the Tenants dictionary to use by /// default for our tests. @@ -66,6 +75,12 @@ public class TestConfigurations /// private string TargetOAuthTenantName { get; set; } + /// + /// Gets the name of the tenant in the keyvaults dictionary to use for + /// any tests that require integration with key vault. + /// + private string TargetKeyVaultName { get; set; } + /// /// Gets the name of the tenant in the Tenants dictionary to use for /// any tests that require hierarchical namespace. @@ -110,6 +125,12 @@ public class TestConfigurations public static TenantConfiguration DefaultTargetOAuthTenant => GetTenant("TargetOAuthTenant", s_configurations.Value.TargetOAuthTenantName); + /// + /// Gets a keyvault to use for any tests that require keyvault access. + /// + public static KeyVaultConfiguration DefaultTargetKeyVault => + GetKeyVault("TargetKeyVault", s_configurations.Value.TargetKeyVaultName); + /// /// Gets a tenant to use for any tests that require hierarchical namespace. /// @@ -168,6 +189,26 @@ private static TenantConfiguration GetTenant(string type, string name) return config; } + /// + /// Get the live test configuration for a specific key vault type, or + /// stop running the test via Assert.Inconclusive if not found. + /// + /// + /// The name of the key vault type (XML element) to get. + /// + /// The name of the keyvault. + /// + /// The live test configuration for a specific tenant type. + /// + private static KeyVaultConfiguration GetKeyVault(string type, string name) + { + if (!s_configurations.Value.KeyVaults.TryGetValue(name, out KeyVaultConfiguration config)) + { + Assert.Inconclusive($"Live test configuration key vault type '{type}' named '{name}' was not found in file {TestConfigurationsPath}!"); + } + return config; + } + /// /// Load the test configurations file from the path pointed to by the /// AZ_STORAGE_CONFIG_PATH environment variable or the local copy of @@ -217,12 +258,17 @@ private static TestConfigurations ReadFromXml(XDocument doc) TargetPremiumBlobTenantName = Get("TargetPremiumBlobTenant"), TargetPreviewBlobTenantName = Get("TargetPreviewBlobTenant"), TargetOAuthTenantName = Get("TargetOAuthTenant"), + TargetKeyVaultName = Get("TargetKeyVault"), TargetHierarchicalNamespaceTenantName = Get("TargetHierarchicalNamespaceTenant"), TargetManagedDiskTenantName = Get("TargetManagedDiskTenant"), Tenants = config.Element("TenantConfigurations").Elements("TenantConfiguration") .Select(TenantConfiguration.Parse) - .ToDictionary(tenant => tenant.TenantName) + .ToDictionary(tenant => tenant.TenantName), + KeyVaults = + config.Element("KeyVaultConfigurations").Elements("KeyVaultConfiguration") + .Select(KeyVaultConfiguration.Parse) + .ToDictionary(keyvault => keyvault.VaultName) }; } } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml index d626b7f1da3b2..f5a5ddad108b0 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TestConfigurationsTemplate.xml @@ -4,6 +4,7 @@ SecondaryTenant NotInPreview OAuthTenant + ClientsideEncryptionKeyvault NamespaceTenant ManagedDiskTenant @@ -124,4 +125,14 @@ http://[ACCOUNT]-secondary.table.core.windows.net + + + ClientsideEncryptionKeyvault + https://[KEYVAULT NAME].vault.azure.net/ + [ActiveDirectoryApplicationId] + [ActiveDirectoryTenantId] + [ActiveDirectoryApplicationSecret] + https://login.microsoftonline.com/ + + diff --git a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs index 209865123d36c..da3eaacfca334 100644 --- a/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Queues/api/Azure.Storage.Queues.netstandard2.0.cs @@ -315,6 +315,28 @@ internal UpdateReceipt() { } public string PopReceipt { get { throw null; } } } } +namespace Azure.Storage.Queues.Specialized +{ + public partial class ClientSideDecryptionFailureEventArgs + { + internal ClientSideDecryptionFailureEventArgs() { } + public System.Exception Exception { get { throw null; } } + } + public partial class QueueClientSideEncryptionOptions : Azure.Storage.ClientSideEncryptionOptions + { + public QueueClientSideEncryptionOptions(Azure.Storage.ClientSideEncryptionVersion version) : base (default(Azure.Storage.ClientSideEncryptionVersion)) { } + public event System.EventHandler DecryptionFailed { add { } remove { } } + } + public partial class SpecializedQueueClientOptions : Azure.Storage.Queues.QueueClientOptions + { + public SpecializedQueueClientOptions(Azure.Storage.Queues.QueueClientOptions.ServiceVersion version = Azure.Storage.Queues.QueueClientOptions.ServiceVersion.V2019_07_07) : base (default(Azure.Storage.Queues.QueueClientOptions.ServiceVersion)) { } + public Azure.Storage.ClientSideEncryptionOptions ClientSideEncryption { get { throw null; } set { } } + } + public static partial class SpecializedQueueExtensions + { + public static Azure.Storage.Queues.QueueClient WithClientSideEncryptionOptions(this Azure.Storage.Queues.QueueClient client, Azure.Storage.ClientSideEncryptionOptions clientSideEncryptionOptions) { throw null; } + } +} namespace Azure.Storage.Sas { [System.FlagsAttribute] diff --git a/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj b/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj index 22499d097f263..efc3150cc7835 100644 --- a/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj +++ b/sdk/storage/Azure.Storage.Queues/src/Azure.Storage.Queues.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks) @@ -16,6 +16,9 @@ REST API Reference for Queue Service - https://docs.microsoft.com/en-us/rest/api/storageservices/queue-service-rest-api + + + @@ -31,6 +34,8 @@ + + diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs new file mode 100644 index 0000000000000..957943aecec84 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessage.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Queues.Specialized.Models +{ + internal class EncryptedMessage + { + public string EncryptedMessageText { get; set; } + + public EncryptionData EncryptionData { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs new file mode 100644 index 0000000000000..46990350d57aa --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/Models/EncryptedMessageSerializer.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Text; +using System.Text.Json; +using Azure.Storage.Cryptography.Models; + +namespace Azure.Storage.Queues.Specialized.Models +{ + internal static class EncryptedMessageSerializer + { + private const string EncryptedMessage_EncryptedMessageTextName = "EncryptedMessageContents"; + + #region Serialize + public static string Serialize(EncryptedMessage data) + { + return Encoding.UTF8.GetString(SerializeEncryptedMessage(data).ToArray()); + } + + public static ReadOnlyMemory SerializeEncryptedMessage(EncryptedMessage message) + { + var writer = new Core.ArrayBufferWriter(); + using var json = new Utf8JsonWriter(writer); + + json.WriteStartObject(); + WriteEncryptedMessage(json, message); + json.WriteEndObject(); + + json.Flush(); + return writer.WrittenMemory; + } + + public static void WriteEncryptedMessage(Utf8JsonWriter json, EncryptedMessage message) + { + json.WriteString(EncryptedMessage_EncryptedMessageTextName, message.EncryptedMessageText); + + json.WriteStartObject(nameof(message.EncryptionData)); + EncryptionDataSerializer.WriteEncryptionData(json, message.EncryptionData); + json.WriteEndObject(); + } + #endregion + + #region Deserialize + public static bool TryDeserialize(string serializedData, out EncryptedMessage encryptedMessage) + { + try + { + encryptedMessage = Deserialize(serializedData); + return true; + } + // JsonException does not actually cover everything. InvalidOperationException can be thrown + // on some string inputs, as we can't assume input is even JSON. + catch (Exception) + { + encryptedMessage = default; + return false; + } + } + + public static EncryptedMessage Deserialize(string serializedData) + { + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(serializedData)); + return DeserializeEncryptedMessage(ref reader); + } + + public static EncryptedMessage DeserializeEncryptedMessage(ref Utf8JsonReader reader) + { + using JsonDocument json = JsonDocument.ParseValue(ref reader); + JsonElement root = json.RootElement; + return ReadEncryptionData(root); + } + + private static EncryptedMessage ReadEncryptionData(JsonElement root) + { + var data = new EncryptedMessage(); + foreach (var property in root.EnumerateObject()) + { + ReadPropertyValue(data, property); + } + return data; + } + + private static void ReadPropertyValue(EncryptedMessage data, JsonProperty property) + { + if (property.NameEquals(EncryptedMessage_EncryptedMessageTextName)) + { + data.EncryptedMessageText = property.Value.GetString(); + } + else if (property.NameEquals(nameof(data.EncryptionData))) + { + data.EncryptionData = EncryptionDataSerializer.ReadEncryptionData(property.Value); + } + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs index f972d89da457e..46b354f411615 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClient.cs @@ -5,14 +5,17 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Net; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; using Metadata = System.Collections.Generic.IDictionary; +#pragma warning disable SA1402 // File may only contain a single type + namespace Azure.Storage.Queues { /// @@ -72,6 +75,18 @@ public class QueueClient /// internal virtual ClientDiagnostics ClientDiagnostics => _clientDiagnostics; + /// + /// The to be used when sending/receiving requests. + /// + private readonly QueueClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual QueueClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + + internal bool UsingClientSideEncryption => ClientSideEncryption != default; + /// /// QueueMaxMessagesPeek indicates the maximum number of messages /// you can retrieve with each call to Peek. @@ -178,6 +193,7 @@ public QueueClient(string connectionString, string queueName, QueueClientOptions _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -269,6 +285,7 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -290,13 +307,22 @@ internal QueueClient(Uri queueUri, HttpPipelinePolicy authentication, QueueClien /// The instance used to create /// diagnostic scopes every request. /// - internal QueueClient(Uri queueUri, HttpPipeline pipeline, QueueClientOptions.ServiceVersion version, ClientDiagnostics clientDiagnostics) + /// + /// Options for client-side encryption. + /// + internal QueueClient( + Uri queueUri, + HttpPipeline pipeline, + QueueClientOptions.ServiceVersion version, + ClientDiagnostics clientDiagnostics, + ClientSideEncryptionOptions encryptionOptions) { _uri = queueUri; _messagesUri = queueUri.AppendToPath(Constants.Queue.MessagesUri); _pipeline = pipeline; _version = version; _clientDiagnostics = clientDiagnostics; + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(encryptionOptions); } #endregion ctors @@ -1472,6 +1498,11 @@ private async Task> SendMessageInternal( $"{nameof(timeToLive)}: {timeToLive}"); try { + messageText = UsingClientSideEncryption + ? await new QueueClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) + .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) + : messageText; + Response> messages = await QueueRestClient.Messages.EnqueueAsync( ClientDiagnostics, @@ -1659,9 +1690,21 @@ private async Task> ReceiveMessagesInternal( .ConfigureAwait(false); // Return an exploding Response on 304 - return response.IsUnavailable() ? - response.GetRawResponse().AsNoBodyResponse() : - Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + if (response.IsUnavailable()) + { + return response.GetRawResponse().AsNoBodyResponse(); + } + else if (UsingClientSideEncryption) + { + return Response.FromValue( + await new QueueClientSideDecryptor(ClientSideEncryption) + .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + response.GetRawResponse()); + } + else + { + return Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + } } catch (Exception ex) { @@ -1766,9 +1809,21 @@ private async Task> PeekMessagesInternal( .ConfigureAwait(false); // Return an exploding Response on 304 - return response.IsUnavailable() ? - response.GetRawResponse().AsNoBodyResponse() : - Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + if (response.IsUnavailable()) + { + return response.GetRawResponse().AsNoBodyResponse(); + } + else if (UsingClientSideEncryption) + { + return Response.FromValue( + await new QueueClientSideDecryptor(ClientSideEncryption) + .ClientSideDecryptMessagesInternal(response.Value.ToArray(), async, cancellationToken).ConfigureAwait(false), + response.GetRawResponse()); + } + else + { + return Response.FromValue(response.Value.ToArray(), response.GetRawResponse()); + } } catch (Exception ex) { @@ -2018,6 +2073,11 @@ private async Task> UpdateMessageInternal( $"{nameof(visibilityTimeout)}: {visibilityTimeout}"); try { + messageText = UsingClientSideEncryption && messageText != default + ? await new QueueClientSideEncryptor(new ClientSideEncryptor(ClientSideEncryption)) + .ClientSideEncryptInternal(messageText, async, cancellationToken).ConfigureAwait(false) + : messageText; + return await QueueRestClient.MessageId.UpdateAsync( ClientDiagnostics, Pipeline, @@ -2045,3 +2105,27 @@ private async Task> UpdateMessageInternal( #endregion UpdateMessage } } + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Add methods to clients. + /// + public static partial class SpecializedQueueExtensions + { + /// + /// Creates a new instance of the class, maintaining all the same + /// internals but specifying new . + /// + /// Client to base off of. + /// New encryption options. Setting this to default will clear client-side encryption. + /// New instance with provided options and same internals otherwise. + public static QueueClient WithClientSideEncryptionOptions(this QueueClient client, ClientSideEncryptionOptions clientSideEncryptionOptions) + => new QueueClient( + client.Uri, + client.Pipeline, + client.Version, + client.ClientDiagnostics, + clientSideEncryptionOptions); + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs index 4330118d84c4e..ab3cdf9194163 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientOptions.cs @@ -4,6 +4,7 @@ using System; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Queues.Specialized; namespace Azure.Storage.Queues { @@ -82,6 +83,10 @@ public QueueClientOptions(ServiceVersion version = LatestVersion) /// public Uri GeoRedundantSecondaryUri { get; set; } + #region Advanced Options + internal ClientSideEncryptionOptions _clientSideEncryptionOptions; + #endregion + /// /// Add headers and query parameters in and /// diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs new file mode 100644 index 0000000000000..1408109e80f26 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideDecryptor.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Cryptography; +using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; +using Azure.Storage.Queues.Specialized.Models; + +namespace Azure.Storage.Queues +{ + internal class QueueClientSideDecryptor + { + private readonly ClientSideDecryptor _decryptor; + public QueueClientSideEncryptionOptions Options { get; } + + public QueueClientSideDecryptor(QueueClientSideEncryptionOptions options) + { + _decryptor = new ClientSideDecryptor(options); + Options = options; + } + + public async Task ClientSideDecryptMessagesInternal(QueueMessage[] messages, bool async, CancellationToken cancellationToken) + { + var filteredMessages = new List(); + foreach (var message in messages) + { + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (Exception e) when (Options.UsingDecryptionFailureHandler) + { + Options.OnDecryptionFailed(message, e); + } + } + return filteredMessages.ToArray(); + } + public async Task ClientSideDecryptMessagesInternal(PeekedMessage[] messages, bool async, CancellationToken cancellationToken) + { + var filteredMessages = new List(); + foreach (var message in messages) + { + try + { + message.MessageText = await ClientSideDecryptInternal(message.MessageText, async, cancellationToken).ConfigureAwait(false); + filteredMessages.Add(message); + } + catch (Exception e) when (Options.UsingDecryptionFailureHandler) + { + Options.OnDecryptionFailed(message, e); + } + } + return filteredMessages.ToArray(); + } + + private async Task ClientSideDecryptInternal(string downloadedMessage, bool async, CancellationToken cancellationToken) + { + if (!EncryptedMessageSerializer.TryDeserialize(downloadedMessage, out var encryptedMessage)) + { + return downloadedMessage; // not recognized as client-side encrypted message + } + + var encryptedMessageStream = new MemoryStream(Convert.FromBase64String(encryptedMessage.EncryptedMessageText)); + var decryptedMessageStream = await _decryptor.DecryptInternal( + encryptedMessageStream, + encryptedMessage.EncryptionData, + ivInStream: false, + noPadding: false, + async: async, + cancellationToken).ConfigureAwait(false); + // if we got back the stream we put in, then we couldn't decrypt and are supposed to return the original + // message to the user + if (encryptedMessageStream == decryptedMessageStream) + { + return downloadedMessage; + } + + return new StreamReader(decryptedMessageStream, Encoding.UTF8).ReadToEnd(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs new file mode 100644 index 0000000000000..c28c8a951c3ae --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptionOptions.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Storage.Cryptography; + +#pragma warning disable SA1402 // File may only contain a single type + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Contains Queues-specific options for client-side encryption. + /// + public class QueueClientSideEncryptionOptions : ClientSideEncryptionOptions + { + /// + /// Initializes a new instance of the class. + /// + /// The version of clientside encryption to use. + public QueueClientSideEncryptionOptions(ClientSideEncryptionVersion version) : base(version) + { + } + + /// + /// Event when failure to decrypt a message occurs. + /// + public event EventHandler DecryptionFailed; + + internal bool UsingDecryptionFailureHandler => DecryptionFailed.GetInvocationList().Length > 0; + + internal void OnDecryptionFailed(object message, Exception e) + { + DecryptionFailed?.Invoke(message, new ClientSideDecryptionFailureEventArgs(e)); + } + + /// + /// Clones the given as an instance of + /// . If the given instance is also a + /// , this clones it's specialty data as well. + /// + /// + internal static QueueClientSideEncryptionOptions CloneFrom(ClientSideEncryptionOptions options) + { + if (options == default) + { + return default; + } + var newOptions = new QueueClientSideEncryptionOptions(options.EncryptionVersion); + ClientSideEncryptionOptionsExtensions.CopyOptions(options, newOptions); + if (options is QueueClientSideEncryptionOptions queueOptions) + { + newOptions.DecryptionFailed = queueOptions.DecryptionFailed; + } + return newOptions; + } + } + + /// + /// Event args for when a queue message decryption fails. + /// + public class ClientSideDecryptionFailureEventArgs + { + /// + /// The exception thrown. + /// + public Exception Exception { get; } + + internal ClientSideDecryptionFailureEventArgs(Exception exception) + { + Exception = exception; + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs new file mode 100644 index 0000000000000..1c5917fd8bab6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/QueueClientSideEncryptor.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Specialized.Models; + +namespace Azure.Storage.Queues +{ + internal class QueueClientSideEncryptor + { + private readonly ClientSideEncryptor _encryptor; + + public QueueClientSideEncryptor(ClientSideEncryptor encryptor) + { + _encryptor = encryptor; + } + + public async Task ClientSideEncryptInternal(string messageToUpload, bool async, CancellationToken cancellationToken) + { + var bytesToEncrypt = Encoding.UTF8.GetBytes(messageToUpload); + (byte[] ciphertext, EncryptionData encryptionData) = await _encryptor.BufferedEncryptInternal( + new MemoryStream(bytesToEncrypt), + async, + cancellationToken).ConfigureAwait(false); + + return EncryptedMessageSerializer.Serialize(new EncryptedMessage + { + EncryptedMessageText = Convert.ToBase64String(ciphertext), + EncryptionData = encryptionData + }); + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs index 3aa8eae603aa8..cf0526a0fc748 100644 --- a/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs +++ b/sdk/storage/Azure.Storage.Queues/src/QueueServiceClient.cs @@ -8,7 +8,9 @@ using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; namespace Azure.Storage.Queues { @@ -59,6 +61,16 @@ public class QueueServiceClient /// internal virtual ClientDiagnostics ClientDiagnostics => _clientDiagnostics; + /// + /// The to be used when sending/receiving requests. + /// + private readonly ClientSideEncryptionOptions _clientSideEncryption; + + /// + /// The to be used when sending/receiving requests. + /// + internal virtual ClientSideEncryptionOptions ClientSideEncryption => _clientSideEncryption; + /// /// The Storage account name corresponding to the service client. /// @@ -128,6 +140,7 @@ public QueueServiceClient(string connectionString, QueueClientOptions options) _pipeline = options.Build(conn.Credentials); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } /// @@ -214,6 +227,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q _pipeline = options.Build(authentication); _version = options.Version; _clientDiagnostics = new ClientDiagnostics(options); + _clientSideEncryption = QueueClientSideEncryptionOptions.CloneFrom(options._clientSideEncryptionOptions); } #endregion ctors @@ -230,7 +244,7 @@ internal QueueServiceClient(Uri serviceUri, HttpPipelinePolicy authentication, Q /// A for the desired queue. /// public virtual QueueClient GetQueueClient(string queueName) - => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics); + => new QueueClient(Uri.AppendToPath(queueName), Pipeline, Version, ClientDiagnostics, ClientSideEncryption); #region GetQueues /// diff --git a/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs new file mode 100644 index 0000000000000..a8daa566e1b38 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/src/SpecializedQueueClientOptions.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Threading.Tasks; +using Azure.Storage.Queues.Models; + +namespace Azure.Storage.Queues.Specialized +{ + /// + /// Provides advanced client configuration options for connecting to Azure Queue + /// Storage. + /// +#pragma warning disable AZC0008 // ClientOptions should have a nested enum called ServiceVersion; This is an extension of existing public options that obey this. + public class SpecializedQueueClientOptions : QueueClientOptions +#pragma warning restore AZC0008 // ClientOptions should have a nested enum called ServiceVersion + { + /// + /// Initializes a new instance of the + /// class. + /// + /// + /// The of the service API used when + /// making requests. + /// + public SpecializedQueueClientOptions(ServiceVersion version = LatestVersion) : base(version) + { + } + + /// + /// Settings for data encryption within the SDK. Client-side encryption adds metadata to your queue + /// messages which is necessary to maintain for decryption. + /// + /// For more information, see . + /// + public ClientSideEncryptionOptions ClientSideEncryption + { + get => _clientSideEncryptionOptions; + set => _clientSideEncryptionOptions = value; + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index e767284f654c6..ce10e105c6a18 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -13,6 +13,9 @@ PreserveNewest + + + diff --git a/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs new file mode 100644 index 0000000000000..f6b2bb965cbd6 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs @@ -0,0 +1,734 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Cryptography; +using Azure.Core.TestFramework; +using Azure.Security.KeyVault.Keys.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Models; +using Azure.Storage.Queues.Specialized; +using Azure.Storage.Queues.Specialized.Models; +using Azure.Storage.Queues.Tests; +using Moq; +using NUnit.Framework; +using static Moq.It; + +namespace Azure.Storage.Queues.Test +{ + public class ClientSideEncryptionTests : QueueTestBase + { + + private const string s_algorithmName = "some algorithm name"; + private static readonly CancellationToken s_cancellationToken = new CancellationTokenSource().Token; + + private readonly string SampleUTF8String = Encoding.UTF8.GetString( + new byte[] { 0xe1, 0x9a, 0xa0, 0xe1, 0x9b, 0x87, 0xe1, 0x9a, 0xbb, 0x0a }); // valid UTF-8 bytes + + public ClientSideEncryptionTests(bool async) + : base(async, null /* RecordedTestMode.Record /* to re-record */) + { + } + + /// + /// Provides encryption functionality clone of client logic, letting us validate the client got it right end-to-end. + /// + private string EncryptData(string message, byte[] key, byte[] iv) + { + using (var aesProvider = new AesCryptoServiceProvider() { Key = key, IV = iv }) + using (var encryptor = aesProvider.CreateEncryptor()) + using (var memStream = new MemoryStream()) + using (var cryptoStream = new CryptoStream(memStream, encryptor, CryptoStreamMode.Write)) + { + var messageBytes = Encoding.UTF8.GetBytes(message); + cryptoStream.Write(messageBytes, 0, messageBytes.Length); + cryptoStream.FlushFinalBlock(); + return Convert.ToBase64String(memStream.ToArray()); + } + } + + private async Task GetKeyvaultIKeyEncryptionKey() + { + var keyClient = GetKeyClient_TargetKeyClient(); + Security.KeyVault.Keys.KeyVaultKey key = await keyClient.CreateRsaKeyAsync( + new Security.KeyVault.Keys.CreateRsaKeyOptions($"CloudRsaKey-{Guid.NewGuid()}", false)); + return new CryptographyClient(key.Id, GetTokenCredential_TargetKeyClient()); + } + + /// + /// Creates an encrypted queue client from a normal queue client. Note that this method does not copy over any + /// client options from the container client. You must pass in your own options. These options will be mutated. + /// + public async Task GetTestEncryptedQueueAsync( + ClientSideEncryptionOptions encryptionOptions, + string queueName = default, + IDictionary metadata = default) + { + // normally set through property on subclass; this is easier to hook up in current test infra with internals access + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; + + var service = GetServiceClient_SharedKey(options); + + metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + QueueClient queue = InstrumentClient(service.GetQueueClient(GetNewQueueName())); + return await DisposingQueue.CreateAsync(queue, metadata); + } + + /// + /// Generates a random string of the given size. + /// For implementation simplicity, this generates an ASCII string. + /// + /// Size of the string IN BYTES, not chars. + /// + public string GetRandomMessage(int size) + { + var buf = new byte[size]; + var random = new Random(); + for (int i = 0; i < size; i++) + { + buf[i] = (byte)random.Next(32, 127); // printable ASCII has values [32, 127) + } + + return Encoding.ASCII.GetString(buf); + } + + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + if (IsAsync) + { + keyMock.Setup(k => k.WrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Task.FromResult(Xor(userKeyBytes, key.ToArray()))); + keyMock.Setup(k => k.UnwrapKeyAsync(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey.ToArray()))); + } + else + { + keyMock.Setup(k => k.WrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(s_algorithmName, IsNotNull>(), s_cancellationToken)) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, wrappedKey.ToArray())); + } + + return keyMock; + } + + private Mock GetAlwaysFailsKeyResolver(bool throws) + { + var mock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + if (throws) + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns(Task.FromResult(null)); + } + } + else + { + if (throws) + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Throws(); + } + else + { + mock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((IKeyEncryptionKey)null); + } + } + + return mock; + } + + private Mock GetIKeyEncryptionKeyResolver(IKeyEncryptionKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + if (IsAsync) + { + resolverMock.Setup(r => r.ResolveAsync(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + } + else + { + resolverMock.Setup(r => r.Resolve(IsNotNull(), s_cancellationToken)) + .Returns((keyId, cancellationToken) => iKey?.KeyId == keyId ? iKey : throw new Exception("Mock resolver couldn't resolve key id.")); + } + + return resolverMock; + } + + private Mock GetTrackOneIKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.Kid).Returns(keyId); + keyMock.SetupGet(k => k.DefaultKeyWrapAlgorithm).Returns(s_algorithmName); + // track one had async-only key wrapping + keyMock.Setup(k => k.WrapKeyAsync(IsNotNull(), IsAny(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + // track 1 doesn't pass in the algorithm name, it lets the implementation return the default algorithm it chose + .Returns((key, algorithm, cancellationToken) => Task.FromResult(Tuple.Create(Xor(userKeyBytes, key), s_algorithmName))); + keyMock.Setup(k => k.UnwrapKeyAsync(IsNotNull(), s_algorithmName, IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((wrappedKey, algorithm, cancellationToken) => Task.FromResult(Xor(userKeyBytes, wrappedKey))); + + return keyMock; + } + + private Mock GetTrackOneIKeyResolver(Microsoft.Azure.KeyVault.Core.IKey iKey) + { + var resolverMock = new Mock(MockBehavior.Strict); + resolverMock.Setup(r => r.ResolveKeyAsync(IsNotNull(), IsNotNull())) // track 1 doesn't pass in the same cancellation token? + .Returns((keyId, cancellationToken) => iKey?.Kid == keyId ? Task.FromResult(iKey) : throw new Exception("Mock resolver couldn't resolve key id.")); + + return resolverMock; + } + + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + + [Test] + [LiveOnly] + public void CanSwapKey() + { + int options1EventCalled = 0; + int options2EventCalled = 0; + void Options1_DecryptionFailed(object sender, ClientSideDecryptionFailureEventArgs e) + { + options1EventCalled++; + } + void Options2_DecryptionFailed(object sender, ClientSideDecryptionFailureEventArgs e) + { + options2EventCalled++; + } + var options1 = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "foo" + }; + options1.DecryptionFailed += Options1_DecryptionFailed; + var options2 = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyResolver = GetIKeyEncryptionKeyResolver(default).Object, + KeyWrapAlgorithm = "bar" + }; + options2.DecryptionFailed += Options2_DecryptionFailed; + + var client = new QueueClient(new Uri("http://someuri.com"), new SpecializedQueueClientOptions() + { + ClientSideEncryption = options1 + }); + + Assert.AreEqual(options1.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options1.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options1.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + + Assert.AreEqual(0, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + client.ClientSideEncryption.OnDecryptionFailed(default, default); + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + + client = client.WithClientSideEncryptionOptions(options2); + + Assert.AreEqual(options2.KeyEncryptionKey, client.ClientSideEncryption.KeyEncryptionKey); + Assert.AreEqual(options2.KeyResolver, client.ClientSideEncryption.KeyResolver); + Assert.AreEqual(options2.KeyWrapAlgorithm, client.ClientSideEncryption.KeyWrapAlgorithm); + + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(0, options2EventCalled); + client.ClientSideEncryption.OnDecryptionFailed(default, default); + Assert.AreEqual(1, options1EventCalled); + Assert.AreEqual(1, options2EventCalled); + } + + [TestCase(16, false)] // a single cipher block + [TestCase(14, false)] // a single unalligned cipher block + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task UploadAsync(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = GetIKeyEncryptionKey().Object; + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); + + // download without decrypting + var receivedMessages = (await InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials())).ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; + Assert.AreEqual(1, receivedMessages.Length); + var encryptedMessage = receivedMessages[0].MessageText; // json of message and metadata + var parsedEncryptedMessage = EncryptedMessageSerializer.Deserialize(encryptedMessage); + + // encrypt original data manually for comparison + EncryptionData encryptionMetadata = parsedEncryptedMessage.EncryptionData; + Assert.NotNull(encryptionMetadata, "Never encrypted data."); + var explicitlyUnwrappedKey = IsAsync + ? await mockKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false) + : mockKey.UnwrapKey(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken); + string expectedEncryptedMessage = EncryptData( + message, + explicitlyUnwrappedKey, + encryptionMetadata.ContentEncryptionIV); + + // compare data + Assert.AreEqual(expectedEncryptedMessage, parsedEncryptedMessage.EncryptedMessageText); + } + } + + [TestCase(16, false)] // a single cipher block + [TestCase(14, false)] // a single unalligned cipher block + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task RoundtripAsync(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); + + // download with decryption + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message, downloadedMessage); + } + } + + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task Track2DownloadTrack1Blob(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object).Object; + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var track2Queue = disposable.Queue; + + // upload with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Queue = new Microsoft.Azure.Storage.Queue.CloudQueue( + track2Queue.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var track1RequestOptions = new Microsoft.Azure.Storage.Queue.QueueRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(mockKey, default) + }; + if (IsAsync) + { + await track1Queue.AddMessageAsync(new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), null, null, track1RequestOptions, null, s_cancellationToken); + } + else + { + track1Queue.AddMessage(new Microsoft.Azure.Storage.Queue.CloudQueueMessage(message), null, null, track1RequestOptions, null); + } + + // download with track 2 + var receivedMessages = (await track2Queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare original data to downloaded data + Assert.AreEqual(message, downloadedMessage); + } + } + + [TestCase(Constants.KB, false)] // multiple blocks + [TestCase(Constants.KB - 4, false)] // multiple unalligned blocks + [TestCase(0, true)] // utf8 support testing + [LiveOnly] // cannot seed content encryption key + public async Task Track1DownloadTrack2Blob(int messageSize, bool usePrebuiltMessage) + { + var message = usePrebuiltMessage + ? GetRandomMessage(messageSize) + : SampleUTF8String; + + const int keySizeBits = 256; + var keyEncryptionKeyBytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(keyEncryptionKeyBytes); + var keyId = Guid.NewGuid().ToString(); + + var mockKey = GetIKeyEncryptionKey(keyEncryptionKeyBytes, keyId).Object; + var mockKeyResolver = GetTrackOneIKeyResolver(GetTrackOneIKey(keyEncryptionKeyBytes, keyId).Object).Object; + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyWrapAlgorithm = s_algorithmName + })) + { + var track2Queue = disposable.Queue; + + // upload with track 2 + await track2Queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); + + // download with track 1 + var creds = GetNewSharedKeyCredentials(); + var track1Queue = new Microsoft.Azure.Storage.Queue.CloudQueue( + track2Queue.Uri, + new Microsoft.Azure.Storage.Auth.StorageCredentials(creds.AccountName, creds.GetAccountKey())); + var response = await track1Queue.GetMessageAsync( + null, + new Microsoft.Azure.Storage.Queue.QueueRequestOptions() + { + EncryptionPolicy = new Microsoft.Azure.Storage.Queue.QueueEncryptionPolicy(default, mockKeyResolver) + }, + null); + + // compare original data to downloaded data + Assert.AreEqual(message, response.AsString); + } + } + + [Test] + [LiveOnly] // need access to keyvault service && cannot seed content encryption key + public async Task RoundtripWithKeyvaultProvider() + { + var message = GetRandomMessage(Constants.KB); + IKeyEncryptionKey key = await GetKeyvaultIKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = key, + KeyWrapAlgorithm = "RSA-OAEP-256" + })) + { + var queue = disposable.Queue; + + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken); + + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + Assert.AreEqual(message, downloadedMessage); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task ReadPlaintextMessage() + { + var message = "any old message"; + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var encryptedQueueClient = disposable.Queue; + var plainQueueClient = new QueueClient(encryptedQueueClient.Uri, GetNewSharedKeyCredentials()); + + // upload with encryption + await plainQueueClient.SendMessageAsync(message, cancellationToken: s_cancellationToken); + + // download with decryption + var receivedMessages = (await encryptedQueueClient.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message, downloadedMessage); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task OnlyOneKeyWrapCall() + { + var message = "any old message"; + var mockKey = GetIKeyEncryptionKey(); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey.Object); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey.Object, + KeyResolver = mockKeyResolver.Object, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); + + var wrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("WrapKey"); + var wrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("WrapKeyAsync"); + + Assert.AreEqual(1, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == wrapAsyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == wrapSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == wrapSyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == wrapAsyncMethod)); + } + } + + [Test] + [LiveOnly] + public async Task UpdateEncryptedMessage() + { + var message1 = GetRandomMessage(Constants.KB); + var message2 = GetRandomMessage(Constants.KB); + var mockKey = GetIKeyEncryptionKey(); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey.Object, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + + // upload with encryption + await queue.SendMessageAsync(message1, cancellationToken: s_cancellationToken); // invokes key wrap first time + + // download and update message + var messageToUpdate = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value[0]; + await queue.UpdateMessageAsync(messageToUpdate.MessageId, messageToUpdate.PopReceipt, message2, cancellationToken: s_cancellationToken); // invokes key unwrap first time and key wrap second time + + // download with decryption + var receivedMessages = (await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken)).Value; // invokes key unwrap second time + Assert.AreEqual(1, receivedMessages.Length); + var downloadedMessage = receivedMessages[0].MessageText; + + // compare data + Assert.AreEqual(message2, downloadedMessage); + + // assert key wrap and unwrap were each invoked twice + MethodInfo keyWrap; + MethodInfo keyUnwrap; + if (IsAsync) + { + keyWrap = typeof(IKeyEncryptionKey).GetMethod("WrapKeyAsync"); + keyUnwrap = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); + } + else + { + keyWrap = typeof(IKeyEncryptionKey).GetMethod("WrapKey"); + keyUnwrap = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); + } + Assert.AreEqual(2, mockKey.Invocations.Count(invocation => invocation.Method == keyWrap)); + Assert.AreEqual(2, mockKey.Invocations.Count(invocation => invocation.Method == keyUnwrap)); + } + } + + [Test] + [LiveOnly] // cannot seed content encryption key + public async Task OnlyOneKeyResolveAndUnwrapCall() + { + var message = "any old message"; + var mockKey = GetIKeyEncryptionKey(); + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey.Object); + await using (var disposable = await GetTestEncryptedQueueAsync(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey.Object, + KeyResolver = mockKeyResolver.Object, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); + + // replace with client that has only key resolver + var options = GetOptions(); + options._clientSideEncryptionOptions = new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = default, // we want the key resolver to trigger; no cached key + KeyResolver = mockKeyResolver.Object + }; + queue = InstrumentClient(new QueueClient( + queue.Uri, + GetNewSharedKeyCredentials(), + options)); + + await queue.ReceiveMessagesAsync(cancellationToken: s_cancellationToken); + + System.Reflection.MethodInfo resolveSyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("Resolve"); + System.Reflection.MethodInfo resolveAsyncMethod = typeof(IKeyEncryptionKeyResolver).GetMethod("ResolveAsync"); + System.Reflection.MethodInfo unwrapSyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKey"); + System.Reflection.MethodInfo unwrapAsyncMethod = typeof(IKeyEncryptionKey).GetMethod("UnwrapKeyAsync"); + + Assert.AreEqual(1, IsAsync + ? mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveAsyncMethod) + : mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveSyncMethod) + : mockKeyResolver.Invocations.Count(invocation => invocation.Method == resolveAsyncMethod)); + + Assert.AreEqual(1, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == unwrapAsyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == unwrapSyncMethod)); + Assert.AreEqual(0, IsAsync + ? mockKey.Invocations.Count(invocation => invocation.Method == unwrapSyncMethod) + : mockKey.Invocations.Count(invocation => invocation.Method == unwrapAsyncMethod)); + } + } + + [TestCase(true, false, false)] + [TestCase(false, false, false)] + [TestCase(true, true, false)] + [TestCase(false, true, false)] + [TestCase(true, false, true)] + [TestCase(false, false, true)] + [TestCase(true, true, true)] + [TestCase(false, true, true)] + [LiveOnly] + public async Task CannotFindKeyAsync(bool useListener, bool resolverThrows, bool peek) + { + const int numMessages = 5; + var message = "any old message"; + var mockKey = GetIKeyEncryptionKey().Object; + var mockKeyResolver = GetIKeyEncryptionKeyResolver(mockKey).Object; + await using (var disposable = await GetTestEncryptedQueueAsync( + new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = mockKey, + KeyResolver = mockKeyResolver, + KeyWrapAlgorithm = s_algorithmName + })) + { + var queue = disposable.Queue; + foreach (var _ in Enumerable.Range(0, numMessages)) + { + await queue.SendMessageAsync(message, cancellationToken: s_cancellationToken).ConfigureAwait(false); + } + + bool threw = false; + var resolver = GetAlwaysFailsKeyResolver(resolverThrows); + int returnedMessages = int.MinValue; // obviously wrong value, but need to initialize to something before try block + int failureEventCalled = 0; + try + { + // download but can't find key + var encryptionOptions = new QueueClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + // note decryption will throw whether the resolver throws or just returns null + KeyResolver = resolver.Object, + KeyWrapAlgorithm = "test" + }; + if (useListener) + { + encryptionOptions.DecryptionFailed += (source, args) => failureEventCalled++; + } + var options = GetOptions(); + options._clientSideEncryptionOptions = encryptionOptions; + var badQueueClient = InstrumentClient(new QueueClient(queue.Uri, GetNewSharedKeyCredentials(), options)); + returnedMessages = peek + ? (await badQueueClient.PeekMessagesAsync(numMessages, cancellationToken: s_cancellationToken)).Value.Length + : (await badQueueClient.ReceiveMessagesAsync(numMessages, cancellationToken: s_cancellationToken)).Value.Length; + } + catch (MockException e) + { + Assert.Fail(e.Message); + } + catch (Exception) + { + threw = true; + } + finally + { + Assert.AreNotEqual(useListener, threw); + + if (useListener) + { + // we already asserted the correct method was called in `catch (MockException e)` + Assert.AreEqual(numMessages, resolver.Invocations.Count); + + // assert event was called for each message + Assert.AreEqual(numMessages, failureEventCalled); + + // assert all messages were filtered out of formal response + Assert.AreEqual(0, returnedMessages); + } + } + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs new file mode 100644 index 0000000000000..9d3bf67ca8e71 --- /dev/null +++ b/sdk/storage/Azure.Storage.Queues/tests/EncryptedMessageSerializerTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using Azure.Core.Cryptography; +using Azure.Core.Pipeline; +using Azure.Storage.Cryptography; +using Azure.Storage.Cryptography.Models; +using Azure.Storage.Queues.Specialized.Models; +using Moq; +using NUnit.Framework; +using static Moq.It; + +namespace Azure.Storage.Queues.Test +{ + public class EncryptedMessageSerializerTests // doesn't inherit our test base because there are no network tests + { + private const string TestMessage = "This can technically be a valid encrypted message."; + private const string KeyWrapAlgorithm = "my_key_wrap_algorithm"; + + private Mock GetIKeyEncryptionKey(byte[] userKeyBytes = default, string keyId = default) + { + if (userKeyBytes == default) + { + const int keySizeBits = 256; + var bytes = new byte[keySizeBits >> 3]; + new RNGCryptoServiceProvider().GetBytes(bytes); + userKeyBytes = bytes; + } + keyId ??= Guid.NewGuid().ToString(); + + var keyMock = new Mock(MockBehavior.Strict); + keyMock.SetupGet(k => k.KeyId).Returns(keyId); + keyMock.Setup(k => k.WrapKey(KeyWrapAlgorithm, IsNotNull>(), IsAny())) + .Returns, CancellationToken>((algorithm, key, cancellationToken) => Xor(userKeyBytes, key.ToArray())); + keyMock.Setup(k => k.UnwrapKey(KeyWrapAlgorithm, IsNotNull>(), IsAny())) + .Returns, CancellationToken>((algorithm, wrappedKey, cancellationToken) => Xor(userKeyBytes, userKeyBytes.ToArray())); + + return keyMock; + } + private static byte[] Xor(byte[] a, byte[] b) + { + if (a.Length != b.Length) + { + throw new ArgumentException("Keys must be the same length for this mock implementation."); + } + + var aBits = new System.Collections.BitArray(a); + var bBits = new System.Collections.BitArray(b); + + var result = new byte[a.Length]; + aBits.Xor(bBits).CopyTo(result, 0); + + return result; + } + + [Test] + public void SerializeEncryptedMessage() + { + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + // success = don't throw. test values in another test with deserialization (can't control serialization order) + } + + [Test] + public void DeserializeEncryptedMessage() + { + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + var parsedEncryptedMessage = EncryptedMessageSerializer.Deserialize(serializedMessage); + + Assert.IsTrue(AreEqual(encryptedMessage, parsedEncryptedMessage)); + } + + [Test] + public void TryDeserializeEncryptedMessage() + { + var result = new ClientSideEncryptor(new ClientSideEncryptionOptions(ClientSideEncryptionVersion.V1_0) + { + KeyEncryptionKey = GetIKeyEncryptionKey().Object, + KeyWrapAlgorithm = KeyWrapAlgorithm + }).BufferedEncryptInternal( + new MemoryStream(Encoding.UTF8.GetBytes(TestMessage)), + async: false, + default).EnsureCompleted(); + var encryptedMessage = new EncryptedMessage() + { + EncryptedMessageText = Convert.ToBase64String(result.ciphertext), + EncryptionData = result.encryptionData + }; + var serializedMessage = EncryptedMessageSerializer.Serialize(encryptedMessage); + + bool tryResult = EncryptedMessageSerializer.TryDeserialize(serializedMessage, out var parsedEncryptedMessage); + + Assert.AreEqual(true, tryResult); + Assert.IsTrue(AreEqual(encryptedMessage, parsedEncryptedMessage)); + } + + [TestCase("")] + [TestCase("\"aa\"")] // real world example + [TestCase("this is not even valid json")] + [TestCase("ᛁᚳ᛫ᛗᚨᚷ᛫ᚷᛚᚨᛋ᛫ᛖᚩᛏᚪᚾ᛫ᚩᚾᛞ᛫ᚻᛁᛏ᛫ᚾᛖ᛫ᚻᛖᚪᚱᛗᛁᚪᚧ᛫ᛗᛖ")] + public void TryDeserializeGracefulOnBadInput(string input) + { + bool tryResult = EncryptedMessageSerializer.TryDeserialize(input, out var parsedEncryptedMessage); + + Assert.AreEqual(false, tryResult); + Assert.AreEqual(default, parsedEncryptedMessage); + } + + #region ModelComparison + private static bool AreEqual(EncryptedMessage left, EncryptedMessage right) + => left.EncryptedMessageText.Equals(right.EncryptedMessageText, StringComparison.InvariantCulture) + && AreEqual(left.EncryptionData, right.EncryptionData); + + private static bool AreEqual(EncryptionData left, EncryptionData right) + => left.EncryptionMode.Equals(right.EncryptionMode, StringComparison.InvariantCulture) + && AreEqual(left.WrappedContentKey, right.WrappedContentKey) + && AreEqual(left.EncryptionAgent, right.EncryptionAgent) + && left.ContentEncryptionIV.SequenceEqual(right.ContentEncryptionIV) + && AreEqual(left.KeyWrappingMetadata, right.KeyWrappingMetadata); + + private static bool AreEqual(KeyEnvelope left, KeyEnvelope right) + => left.KeyId.Equals(right.KeyId, StringComparison.InvariantCulture) + && left.EncryptedKey.SequenceEqual(right.EncryptedKey) + && left.Algorithm.Equals(right.Algorithm, StringComparison.InvariantCulture); + + private static bool AreEqual(EncryptionAgent left, EncryptionAgent right) + => left.EncryptionAlgorithm.Equals(right.EncryptionAlgorithm) + && left.EncryptionVersion.Equals(right.EncryptionVersion); + + private static bool AreEqual(IDictionary left, IDictionary right) + { + if (left.Count != right.Count) + { + return false; + } + foreach (KeyValuePair leftPair in left) + { + if (!right.TryGetValue(leftPair.Key, out string rightValue)) + { + return false; + } + if (!leftPair.Value.Equals(rightValue, StringComparison.InvariantCulture)) + { + return false; + } + } + + return true; + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs b/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs index 601cfff3e5100..64ebbd250c93d 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs +++ b/sdk/storage/Azure.Storage.Queues/tests/QueueTestBase.cs @@ -55,14 +55,14 @@ public QueueClientOptions GetOptions() return Recording.InstrumentClientOptions(options); } - public QueueServiceClient GetServiceClient_SharedKey() + public QueueServiceClient GetServiceClient_SharedKey(QueueClientOptions options = default) => InstrumentClient( new QueueServiceClient( new Uri(TestConfigDefault.QueueServiceEndpoint), new StorageSharedKeyCredential( TestConfigDefault.AccountName, TestConfigDefault.AccountKey), - GetOptions())); + options ?? GetOptions())); public QueueServiceClient GetServiceClient_AccountSas(StorageSharedKeyCredential sharedKeyCredentials = default, SasQueryParameters sasCredentials = default) => InstrumentClient( @@ -76,6 +76,23 @@ public QueueServiceClient GetServiceClient_QueueServiceSas(string queueName, Sto new Uri($"{TestConfigDefault.QueueServiceEndpoint}?{sasCredentials ?? GetNewQueueServiceSasCredentials(queueName, sharedKeyCredentials ?? GetNewSharedKeyCredentials())}"), GetOptions())); + public Security.KeyVault.Keys.KeyClient GetKeyClient_TargetKeyClient() + => GetKeyClient(TestConfigurations.DefaultTargetKeyVault); + + public TokenCredential GetTokenCredential_TargetKeyClient() + => GetKeyClientTokenCredential(TestConfigurations.DefaultTargetKeyVault); + + private static Security.KeyVault.Keys.KeyClient GetKeyClient(KeyVaultConfiguration config) + => new Security.KeyVault.Keys.KeyClient( + new Uri(config.VaultEndpoint), + GetKeyClientTokenCredential(config)); + + private static TokenCredential GetKeyClientTokenCredential(KeyVaultConfiguration config) + => new Identity.ClientSecretCredential( + config.ActiveDirectoryTenantId, + config.ActiveDirectoryApplicationId, + config.ActiveDirectoryApplicationSecret); + public QueueServiceClient GetServiceClient_OauthAccount() => GetServiceClientFromOauthConfig(TestConfigOAuth); diff --git a/sdk/storage/Azure.Storage.sln b/sdk/storage/Azure.Storage.sln index 83df1576f3187..7d220db0125a4 100644 --- a/sdk/storage/Azure.Storage.sln +++ b/sdk/storage/Azure.Storage.sln @@ -85,15 +85,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.Shares. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Common.Samples.Tests", "Azure.Storage.Common\samples\Azure.Storage.Common.Samples.Tests.csproj", "{9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Cryptography", "Azure.Storage.Blobs.Cryptography\src\Azure.Storage.Blobs.Cryptography.csproj", "{4384B232-1222-44C7-9946-C23FE94DF6A9}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Cryptography", "Cryptography", "{9B235463-7D89-471E-87CA-A392661255E4}" - ProjectSection(SolutionItems) = preProject - Azure.Storage.Blobs.Cryptography\BreakingChanges.txt = Azure.Storage.Blobs.Cryptography\BreakingChanges.txt - Azure.Storage.Blobs.Cryptography\Changelog.txt = Azure.Storage.Blobs.Cryptography\Changelog.txt - Azure.Storage.Blobs.Cryptography\README.md = Azure.Storage.Blobs.Cryptography\README.md - EndProjectSection -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake", "Azure.Storage.Files.DataLake\src\Azure.Storage.Files.DataLake.csproj", "{DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Files.DataLake.Tests", "Azure.Storage.Files.DataLake\tests\Azure.Storage.Files.DataLake.Tests.csproj", "{EDBF4B6B-5D28-47AF-A9BE-B70568067407}" @@ -115,7 +106,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApiCompat", "..\..\eng\ApiC EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Storage.Blobs.Batch.Samples.Tests", "Azure.Storage.Blobs.Batch\samples\Azure.Storage.Blobs.Batch.Samples.Tests.csproj", "{73F575E6-FA87-40B0-9D36-6D9FAAC1E1C1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Core.TestFramework", "..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{23B3D5C8-3160-4BD6-8B25-0D33C98ABE70}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Core.TestFramework", "..\core\Azure.Core.TestFramework\src\Azure.Core.TestFramework.csproj", "{23B3D5C8-3160-4BD6-8B25-0D33C98ABE70}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -179,10 +170,6 @@ Global {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {9EBEFBC5-A5F4-41B5-946E-67F2DCF5E5B5}.Release|Any CPU.Build.0 = Release|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4384B232-1222-44C7-9946-C23FE94DF6A9}.Release|Any CPU.Build.0 = Release|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Debug|Any CPU.Build.0 = Debug|Any CPU {DE6EC5B8-5DBA-4D21-931E-3B042010B3C0}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 38c117eff19a6..cc0cf130fb709 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -42,8 +42,6 @@ stages: safeName: AzureStorageBlobs - name: Azure.Storage.Blobs.Batch safeName: AzureStorageBlobsBatch - - name: Azure.Storage.Blobs.Cryptography - safeName: AzureStorageBlobsCryptography - name: Azure.Storage.Common safeName: AzureStorageCommon - name: Azure.Storage.Files.Shares