diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp index 05aaf8de74..b6b74c4d5d 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp @@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeDirectoryClient( Azure::Core::Url directoryUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(directoryUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index a4ade4cc68..f9aa309ef3 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -282,8 +282,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeFileClient( Azure::Core::Url fileUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(fileUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index a1caa22248..9a7430e1c1 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -243,13 +243,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_fileSystemUrl; Blobs::BlobContainerClient m_blobContainerClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakeFileSystemClient( Azure::Core::Url fileSystemUrl, Blobs::BlobContainerClient blobContainerClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_fileSystemUrl(std::move(fileSystemUrl)), - m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)) + m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)), + m_customerProvidedKey(std::move(customerProvidedKey)) { } friend class DataLakeLeaseClient; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 4d73a3fe15..fc8e69a015 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -23,10 +23,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using SignedIdentifier = Blobs::Models::SignedIdentifier; using FileQueryArrowField = Blobs::Models::BlobQueryArrowField; using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType; + using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType; } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; + using EncryptionKey = Blobs::EncryptionKey; /** * @brief Client options used to initialize all DataLake clients. @@ -46,6 +48,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * API version used by this client. */ std::string ApiVersion; + + /** + * @brief Holds the customer provided key used when making requests. + */ + Azure::Nullable CustomerProvidedKey; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp index f6183a3e70..497b36ac0a 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp @@ -295,13 +295,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_pathUrl; Blobs::BlobClient m_blobClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakePathClient( Azure::Core::Url pathUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_pathUrl(std::move(pathUrl)), m_blobClient(std::move(blobClient)), - m_pipeline(std::move(pipeline)) + m_pipeline(std::move(pipeline)), m_customerProvidedKey(std::move(customerProvidedKey)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp index 633ddf57cc..564c971d02 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp @@ -113,5 +113,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_serviceUrl; Blobs::BlobServiceClient m_blobServiceClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index 0a58b8e565..c6e755f35c 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -374,6 +374,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable SourceIfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Owner; Nullable Group; Nullable Acl; @@ -478,6 +479,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; }; static Response Flush( Core::Http::_internal::HttpPipeline& pipeline, @@ -492,6 +494,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable LeaseId; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Flush; }; static Response Append( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 1ac31d8f52..7e81aa9f0b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -66,7 +66,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(fileName)); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeDirectoryClient::GetSubdirectoryClient( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(subdirectoryName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName)); - return DataLakeDirectoryClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeDirectoryClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } Azure::Response DataLakeDirectoryClient::RenameFile( @@ -119,10 +121,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(response.RawResponse)); } @@ -167,10 +172,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 407dbaaaa5..43606b0376 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -81,6 +81,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Append( *m_pipeline, m_pathUrl, content, protocolLayerOptions, context); } @@ -111,6 +117,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Flush(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c478c0cf50..38c6668b2b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -50,7 +50,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -78,7 +79,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -107,7 +109,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -129,7 +132,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobContainerClient.GetBlobClient(fileName); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeFileSystemClient::GetDirectoryClient( @@ -138,7 +142,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(directoryName)); return DataLakeDirectoryClient( - builder, m_blobContainerClient.GetBlobClient(directoryName), m_pipeline); + builder, + m_blobContainerClient.GetBlobClient(directoryName), + m_pipeline, + m_customerProvidedKey); } Azure::Response DataLakeFileSystemClient::Create( @@ -356,10 +363,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(result.RawResponse)); } @@ -404,10 +414,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(result.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 0d6990d864..9226372e12 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -48,7 +48,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -104,7 +106,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const std::string& pathUrl, const DataLakeClientOptions& options) : m_pathUrl(pathUrl), - m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)) + m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -202,6 +205,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata); protocolLayerOptions.Umask = options.Umask; protocolLayerOptions.Permissions = options.Permissions; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp index 4633d299ec..240abb8b65 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp @@ -44,7 +44,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -72,7 +73,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -101,7 +103,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -124,7 +127,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_serviceUrl; builder.AppendPath(_internal::UrlEncodePath(fileSystemName)); return DataLakeFileSystemClient( - builder, m_blobServiceClient.GetBlobContainerClient(fileSystemName), m_pipeline); + builder, + m_blobServiceClient.GetBlobContainerClient(fileSystemName), + m_pipeline, + m_customerProvidedKey); } ListFileSystemsPagedResponse DataLakeServiceClient::ListFileSystems( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 0d7c662f88..76a96546bb 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -96,6 +96,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam blobOptions.SecondaryHostForRetryReads = _detail::GetBlobUrlFromUrl(options.SecondaryHostForRetryReads); blobOptions.ApiVersion = options.ApiVersion; + blobOptions.CustomerProvidedKey = options.CustomerProvidedKey; return blobOptions; } diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 2d519a9487..f29440d19c 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -253,7 +253,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Owner.HasValue() && !options.Owner.Value().empty()) { request.SetHeader("x-ms-owner", options.Owner.Value()); @@ -661,7 +664,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -728,7 +734,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Flush.HasValue()) { request.GetUrl().AppendQueryParameter("flush", options.Flush.Value() ? "true" : "false"); diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 5344d267a8..751130bbde 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -181,6 +181,8 @@ directive: transform: > $.Continuation["x-ms-client-name"] = "ContinuationToken"; $.EncryptionKeySha256["format"] = "byte"; + delete $.EncryptionAlgorithm["enum"]; + delete $.EncryptionAlgorithm["x-ms-enum"]; - from: swagger-document where: $.definitions transform: > diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 615e1096d1..d299436ebc 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -397,6 +398,231 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey_LIVEONLY_) + { + + auto getRandomCustomerProvidedKey = [&]() { + Files::DataLake::EncryptionKey key; + std::vector aes256Key; + aes256Key.resize(32); + RandomBuffer(&aes256Key[0], aes256Key.size()); + key.Key = Azure::Core::Convert::Base64Encode(aes256Key); + key.KeyHash = Azure::Core::Cryptography::_internal::Sha256Hash().Final( + aes256Key.data(), aes256Key.size()); + key.Algorithm = Blobs::Models::EncryptionAlgorithmType::Aes256; + return key; + }; + + const int32_t bufferSize = 1024; // 1KB data size + auto buffer = std::make_shared>(bufferSize, 'x'); + Azure::Core::IO::MemoryBodyStream bodyStream(buffer->data(), buffer->size()); + + auto customerProvidedKey + = std::make_shared(getRandomCustomerProvidedKey()); + Files::DataLake::DataLakeClientOptions options; + options.CustomerProvidedKey = *customerProvidedKey; + auto fileServiceClient = std::make_shared( + Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), options)); + auto fileSystemClient = std::make_shared( + fileServiceClient->GetFileSystemClient(m_fileSystemName)); + + // fileSystem works + { + auto fileSystemClientWithoutEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + // Rename File + const std::string filename1 = GetTestName() + "file1"; + const std::string filename2 = GetTestName() + "file2"; + const std::string filename3 = GetTestName() + "file3"; + const std::string filename4 = GetTestName() + "file4"; + + auto oldFileClient = fileSystemClient->GetFileClient(filename1); + oldFileClient.Create(); + auto newFileClient = fileSystemClient->RenameFile(filename1, filename2).Value; + auto properties = std::make_shared( + newFileClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, filename2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); + + // Rename Directory + const std::string testName(GetTestName()); + const std::string oldDirectoryName = testName + "dir1"; + const std::string newDirectoryName = testName + "dir2"; + const std::string newDirectoryName2 = testName + "dir3"; + + auto oldDirectoryClient = fileSystemClient->GetDirectoryClient(oldDirectoryName); + oldDirectoryClient.Create(); + oldDirectoryClient.GetFileClient(testName + "file3").Create(); + oldDirectoryClient.GetSubdirectoryClient(testName + "dir4").Create(); + + auto newDirectoryClient + = fileSystemClient->RenameDirectory(oldDirectoryName, newDirectoryName).Value; + properties = std::make_shared( + newDirectoryClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto newDirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); + EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( + newDirectoryName, newDirectoryName2)); + + auto fileSystemClientWithEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, options); + auto created = std::make_shared( + fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + + // path works + { + const std::string pathName = "path"; + const std::string pathName2 = "path2"; + + auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); + EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(pathClient.SetMetadata(GetMetadata())); + auto properties = std::make_shared( + pathClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto pathClientWithoutEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName); + EXPECT_THROW(pathClientWithoutEncryptionKey.SetMetadata(GetMetadata()), StorageException); + EXPECT_THROW(pathClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.GetAccessControlList()); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetHttpHeaders( + Files::DataLake::Models::PathHttpHeaders())); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetPermissions("rwxrw-rw-")); + + auto pathClientWithEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName2, options); + auto created + = pathClientWithEncryptionKey.Create(Files::DataLake::Models::PathResourceType::File) + .Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created.EncryptionKeySha256.Value()); + } + + // file works + { + const std::string fileName = "file"; + const std::string fileName2 = "file2"; + auto fileClient = fileSystemClient->GetFileClient(fileName); + auto fileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName); + // upload test + EXPECT_NO_THROW(fileClient.Create()); + EXPECT_NO_THROW(fileClient.UploadFrom(buffer->data(), bufferSize)); + auto result = fileClient.Download(); + auto downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); + EXPECT_NO_THROW(fileClient.Delete()); + // append test + EXPECT_NO_THROW(fileClient.Create()); + bodyStream.Rewind(); + EXPECT_NO_THROW(fileClient.Append(bodyStream, 0)); + bodyStream.Rewind(); + EXPECT_THROW(fileClientWithoutEncryptionKey.Append(bodyStream, bufferSize), StorageException); + EXPECT_NO_THROW(fileClient.Flush(bufferSize)); + result = fileClient.Download(); + downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); + EXPECT_NO_THROW(fileClient.SetMetadata(GetMetadata())); + auto properties = std::make_shared( + fileClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); + EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); + + auto fileClientWithEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName2, options); + auto created = std::make_shared( + fileClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + // directory works + { + const std::string directoryName = "directory"; + const std::string directoryName2 = "directory2"; + const std::string subdirectoryName1 = "subdirectory1"; + const std::string subdirectoryName2 = "subdirectory2"; + const std::string subdirectoryName3 = "subdirectory3"; + const std::string fileName1 = "file1"; + const std::string fileName2 = "file2"; + const std::string fileName3 = "file3"; + auto directoryClient = fileSystemClient->GetDirectoryClient(directoryName); + auto directoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName); + // create subdirectory/file + EXPECT_NO_THROW(directoryClient.Create()); + auto subdirectoryClient = directoryClient.GetSubdirectoryClient(subdirectoryName1); + EXPECT_NO_THROW(subdirectoryClient.Create()); + auto fileClient = directoryClient.GetFileClient(fileName1); + EXPECT_NO_THROW(fileClient.Create()); + auto subdirectoryProperties = std::make_shared( + subdirectoryClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, subdirectoryProperties->EncryptionKeySha256.Value()); + auto fileProperties = fileClient.GetProperties(); + EXPECT_EQ(customerProvidedKey->KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); + + // rename file + auto newFileClient + = directoryClient.RenameFile(fileName1, directoryName + "/" + fileName2).Value; + auto newFileProperties = std::make_shared( + newFileClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, newFileProperties->EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName + "/" + fileName2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameFile( + fileName2, directoryName + "/" + fileName3)); + + auto newSubdirectoryClient + = directoryClient + .RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) + .Value; + auto newSubdirectoryProperties = std::make_shared( + newSubdirectoryClient.GetProperties().Value); + EXPECT_EQ( + customerProvidedKey->KeyHash, newSubdirectoryProperties->EncryptionKeySha256.Value()); + auto newsubdirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), + m_fileSystemName, + directoryName + "/" + subdirectoryName2); + EXPECT_THROW(newsubdirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameSubdirectory( + subdirectoryName2, directoryName + "/" + subdirectoryName3)); + + auto directoryClientWithEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName2, options); + auto created = std::make_shared( + directoryClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + } + TEST_F(DataLakeFileSystemClientTest, GetSetAccessPolicy_LIVEONLY_) { CHECK_SKIP_TEST();