From fa692d17ff14004483cd92e71da37984895918c8 Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 21 Sep 2022 14:55:15 +0800 Subject: [PATCH 1/5] cpk for datalake --- .../datalake/datalake_directory_client.hpp | 9 +- .../files/datalake/datalake_file_client.hpp | 9 +- .../datalake/datalake_file_system_client.hpp | 7 +- .../files/datalake/datalake_options.hpp | 7 + .../files/datalake/datalake_path_client.hpp | 6 +- .../datalake/datalake_service_client.hpp | 1 + .../storage/files/datalake/rest_client.hpp | 3 + .../src/datalake_directory_client.cpp | 24 ++- .../src/datalake_file_client.cpp | 12 ++ .../src/datalake_file_system_client.cpp | 35 ++-- .../src/datalake_path_client.cpp | 15 +- .../src/datalake_service_client.cpp | 14 +- .../src/datalake_utilities.cpp | 1 + .../src/rest_client.cpp | 15 +- .../swagger/README.md | 2 + .../ut/datalake_file_system_client_test.cpp | 182 ++++++++++++++++++ ...eSystemClientTest.CustomerProvidedKey.json | 46 +++++ 17 files changed, 351 insertions(+), 37 deletions(-) create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp index 05aaf8de74..b6b74c4d5d 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp @@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeDirectoryClient( Azure::Core::Url directoryUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(directoryUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index 9a4e7c851c..b1b1d78eb6 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -269,8 +269,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeFileClient( Azure::Core::Url fileUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(fileUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index a1caa22248..9a7430e1c1 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -243,13 +243,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_fileSystemUrl; Blobs::BlobContainerClient m_blobContainerClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakeFileSystemClient( Azure::Core::Url fileSystemUrl, Blobs::BlobContainerClient blobContainerClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_fileSystemUrl(std::move(fileSystemUrl)), - m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)) + m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)), + m_customerProvidedKey(std::move(customerProvidedKey)) { } friend class DataLakeLeaseClient; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index b0796cefde..76f8b25389 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -21,10 +21,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using PathHttpHeaders = Blobs::Models::BlobHttpHeaders; using ListFileSystemsIncludeFlags = Blobs::Models::ListBlobContainersIncludeFlags; using SignedIdentifier = Blobs::Models::SignedIdentifier; + using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType; } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; + using EncryptionKey = Blobs::EncryptionKey; /** * @brief Client options used to initialize all DataLake clients. @@ -44,6 +46,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * API version used by this client. */ std::string ApiVersion; + + /** + * @brief Holds the customer provided key used when making requests. + */ + Azure::Nullable CustomerProvidedKey; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp index f6183a3e70..497b36ac0a 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp @@ -295,13 +295,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_pathUrl; Blobs::BlobClient m_blobClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakePathClient( Azure::Core::Url pathUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_pathUrl(std::move(pathUrl)), m_blobClient(std::move(blobClient)), - m_pipeline(std::move(pipeline)) + m_pipeline(std::move(pipeline)), m_customerProvidedKey(std::move(customerProvidedKey)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp index 633ddf57cc..564c971d02 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp @@ -113,5 +113,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_serviceUrl; Blobs::BlobServiceClient m_blobServiceClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index 0a58b8e565..c6e755f35c 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -374,6 +374,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable SourceIfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Owner; Nullable Group; Nullable Acl; @@ -478,6 +479,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; }; static Response Flush( Core::Http::_internal::HttpPipeline& pipeline, @@ -492,6 +494,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable LeaseId; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Flush; }; static Response Append( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 1ac31d8f52..7e81aa9f0b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -66,7 +66,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(fileName)); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeDirectoryClient::GetSubdirectoryClient( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(subdirectoryName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName)); - return DataLakeDirectoryClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeDirectoryClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } Azure::Response DataLakeDirectoryClient::RenameFile( @@ -119,10 +121,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(response.RawResponse)); } @@ -167,10 +172,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index ded67cb674..c47d546655 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -81,6 +81,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Append( *m_pipeline, m_pathUrl, content, protocolLayerOptions, context); } @@ -111,6 +117,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Flush(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c478c0cf50..38c6668b2b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -50,7 +50,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -78,7 +79,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -107,7 +109,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -129,7 +132,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobContainerClient.GetBlobClient(fileName); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeFileSystemClient::GetDirectoryClient( @@ -138,7 +142,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(directoryName)); return DataLakeDirectoryClient( - builder, m_blobContainerClient.GetBlobClient(directoryName), m_pipeline); + builder, + m_blobContainerClient.GetBlobClient(directoryName), + m_pipeline, + m_customerProvidedKey); } Azure::Response DataLakeFileSystemClient::Create( @@ -356,10 +363,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(result.RawResponse)); } @@ -404,10 +414,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(result.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 0d6990d864..9226372e12 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -48,7 +48,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -104,7 +106,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const std::string& pathUrl, const DataLakeClientOptions& options) : m_pathUrl(pathUrl), - m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)) + m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -202,6 +205,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata); protocolLayerOptions.Umask = options.Umask; protocolLayerOptions.Permissions = options.Permissions; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp index 4633d299ec..240abb8b65 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp @@ -44,7 +44,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -72,7 +73,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -101,7 +103,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -124,7 +127,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_serviceUrl; builder.AppendPath(_internal::UrlEncodePath(fileSystemName)); return DataLakeFileSystemClient( - builder, m_blobServiceClient.GetBlobContainerClient(fileSystemName), m_pipeline); + builder, + m_blobServiceClient.GetBlobContainerClient(fileSystemName), + m_pipeline, + m_customerProvidedKey); } ListFileSystemsPagedResponse DataLakeServiceClient::ListFileSystems( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 0d7c662f88..76a96546bb 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -96,6 +96,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam blobOptions.SecondaryHostForRetryReads = _detail::GetBlobUrlFromUrl(options.SecondaryHostForRetryReads); blobOptions.ApiVersion = options.ApiVersion; + blobOptions.CustomerProvidedKey = options.CustomerProvidedKey; return blobOptions; } diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 2d519a9487..f29440d19c 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -253,7 +253,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Owner.HasValue() && !options.Owner.Value().empty()) { request.SetHeader("x-ms-owner", options.Owner.Value()); @@ -661,7 +664,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -728,7 +734,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Flush.HasValue()) { request.GetUrl().AppendQueryParameter("flush", options.Flush.Value() ? "true" : "false"); diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 5344d267a8..751130bbde 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -181,6 +181,8 @@ directive: transform: > $.Continuation["x-ms-client-name"] = "ContinuationToken"; $.EncryptionKeySha256["format"] = "byte"; + delete $.EncryptionAlgorithm["enum"]; + delete $.EncryptionAlgorithm["x-ms-enum"]; - from: swagger-document where: $.definitions transform: > diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 615e1096d1..f8dfc9b544 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -397,6 +398,187 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey_LIVEONLY_) + { + // will skip test under some cased where test can't run (usually LIVE only tests) + CHECK_SKIP_TEST(); + + auto getRandomCustomerProvidedKey = [&]() { + Files::DataLake::EncryptionKey key; + std::vector aes256Key; + aes256Key.resize(32); + RandomBuffer(&aes256Key[0], aes256Key.size()); + key.Key = Azure::Core::Convert::Base64Encode(aes256Key); + key.KeyHash = Azure::Core::Cryptography::_internal::Sha256Hash().Final( + aes256Key.data(), aes256Key.size()); + key.Algorithm = Blobs::Models::EncryptionAlgorithmType::Aes256; + return key; + }; + + const int32_t bufferSize = 1024; // 1KB data size + std::vector buffer(bufferSize, 'x'); + Azure::Core::IO::MemoryBodyStream bodyStream(buffer.data(), buffer.size()); + + auto customerProvidedKey = getRandomCustomerProvidedKey(); + Files::DataLake::DataLakeClientOptions options; + options.CustomerProvidedKey = customerProvidedKey; + auto fileServiceClient + = Azure::Storage::Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), options); + auto fileSystemClient = fileServiceClient.GetFileSystemClient(m_fileSystemName); + + // fileSystem works + { + auto fileSystemClientWithoutEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + // Rename File + const std::string oldFilename = GetTestName() + "file1"; + const std::string newFilename = GetTestName() + "file2"; + const std::string newFilename2 = GetTestName() + "file3"; + + auto oldFileClient = fileSystemClient.GetFileClient(oldFilename); + oldFileClient.Create(); + auto newFileClient = fileSystemClient.RenameFile(oldFilename, newFilename).Value; + auto properties = newFileClient.GetProperties(); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, newFilename); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(newFilename, newFilename2)); + + // Rename Directory + const std::string testName(GetTestName()); + const std::string oldDirectoryName = testName + "dir1"; + const std::string newDirectoryName = testName + "dir2"; + const std::string newDirectoryName2 = testName + "dir3"; + + auto oldDirectoryClient = fileSystemClient.GetDirectoryClient(oldDirectoryName); + oldDirectoryClient.Create(); + oldDirectoryClient.GetFileClient(testName + "file3").Create(); + oldDirectoryClient.GetSubdirectoryClient(testName + "dir4").Create(); + + auto newDirectoryClient + = fileSystemClient.RenameDirectory(oldDirectoryName, newDirectoryName).Value; + properties = newDirectoryClient.GetProperties(); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto newDirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); + EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( + newDirectoryName, newDirectoryName2)); + } + + // path works + { + const std::string pathName = "path"; + auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); + EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(pathClient.SetMetadata(GetMetadata())); + auto properties = pathClient.GetProperties(); + EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto pathClientWithoutEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName); + EXPECT_THROW(pathClientWithoutEncryptionKey.SetMetadata(GetMetadata()), StorageException); + EXPECT_THROW(pathClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.GetAccessControlList()); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetHttpHeaders( + Files::DataLake::Models::PathHttpHeaders())); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetPermissions("rwxrw-rw-")); + } + + // file works + { + const std::string fileName = "file"; + auto fileClient = fileSystemClient.GetFileClient(fileName); + auto fileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName); + // upload test + EXPECT_NO_THROW(fileClient.Create()); + EXPECT_NO_THROW(fileClient.UploadFrom(buffer.data(), bufferSize)); + auto result = fileClient.Download(); + auto downloaded = ReadBodyStream(result.Value.Body); + EXPECT_EQ(buffer, downloaded); + EXPECT_NO_THROW(fileClient.Delete()); + // append test + EXPECT_NO_THROW(fileClient.Create()); + bodyStream.Rewind(); + EXPECT_NO_THROW(fileClient.Append(bodyStream, 0)); + bodyStream.Rewind(); + EXPECT_THROW(fileClientWithoutEncryptionKey.Append(bodyStream, bufferSize), StorageException); + EXPECT_NO_THROW(fileClient.Flush(bufferSize)); + result = fileClient.Download(); + downloaded = ReadBodyStream(result.Value.Body); + EXPECT_EQ(buffer, downloaded); + EXPECT_NO_THROW(fileClient.SetMetadata(GetMetadata())); + auto properties = fileClient.GetProperties(); + EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + + EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); + EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); + } + // directory works + { + const std::string directoryName = "directory"; + const std::string subdirectoryName1 = "subdirectory1"; + const std::string subdirectoryName2 = "subdirectory2"; + const std::string subdirectoryName3 = "subdirectory3"; + const std::string fileName1 = "file1"; + const std::string fileName2 = "file2"; + const std::string fileName3 = "file3"; + auto directoryClient = fileSystemClient.GetDirectoryClient(directoryName); + auto directoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName); + // create subdirectory/file + EXPECT_NO_THROW(directoryClient.Create()); + auto subdirectoryClient = directoryClient.GetSubdirectoryClient(subdirectoryName1); + EXPECT_NO_THROW(subdirectoryClient.Create()); + auto fileClient = directoryClient.GetFileClient(fileName1); + EXPECT_NO_THROW(fileClient.Create()); + auto subdirectoryProperties = subdirectoryClient.GetProperties(); + EXPECT_EQ( + customerProvidedKey.KeyHash, subdirectoryProperties.Value.EncryptionKeySha256.Value()); + auto fileProperties = fileClient.GetProperties(); + EXPECT_EQ(customerProvidedKey.KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); + + // rename file + auto newFileClient + = directoryClient.RenameFile(fileName1, directoryName + "/" + fileName2).Value; + auto newFileProperties = newFileClient.GetProperties(); + EXPECT_EQ(customerProvidedKey.KeyHash, newFileProperties.Value.EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName + "/" + fileName2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameFile( + fileName2, directoryName + "/" + fileName3)); + + auto newSubdirectoryClient + = directoryClient + .RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) + .Value; + auto newSubdirectoryProperties = newSubdirectoryClient.GetProperties(); + EXPECT_EQ( + customerProvidedKey.KeyHash, newSubdirectoryProperties.Value.EncryptionKeySha256.Value()); + auto newsubdirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), + m_fileSystemName, + directoryName + "/" + subdirectoryName2); + EXPECT_THROW(newsubdirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameSubdirectory( + subdirectoryName2, directoryName + "/" + subdirectoryName3)); + } + } + TEST_F(DataLakeFileSystemClientTest, GetSetAccessPolicy_LIVEONLY_) { CHECK_SKIP_TEST(); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json new file mode 100644 index 0000000000..e54369fd52 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json @@ -0,0 +1,46 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5d22ed71-0afa-4cb3-5797-398079facac4", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Wed, 21 Sep 2022 06:43:09 GMT", + "etag": "\"0x8DA9B9C8C0EDDEE\"", + "last-modified": "Wed, 21 Sep 2022 06:43:09 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "5d22ed71-0afa-4cb3-5797-398079facac4", + "x-ms-request-id": "1d067a95-b01e-0058-4285-cd0af0000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestcustomerprovidedkey?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f4ae2973-c1f5-443e-7e56-7284e0d78eb3", + "x-ms-version": "2021-04-10" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Wed, 21 Sep 2022 06:44:43 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f4ae2973-c1f5-443e-7e56-7284e0d78eb3", + "x-ms-request-id": "fc8205fe-701e-0025-6f85-cd7bd3000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestcustomerprovidedkey?restype=container" + } + ] +} From 39040f4dab169813069a93d0eba4c457471e2177 Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 21 Sep 2022 15:15:12 +0800 Subject: [PATCH 2/5] update test --- .../ut/datalake_file_system_client_test.cpp | 51 ++++++++++++++++--- ...eSystemClientTest.CustomerProvidedKey.json | 46 ----------------- 2 files changed, 43 insertions(+), 54 deletions(-) delete mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index f8dfc9b544..add36e1387 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -433,20 +433,21 @@ namespace Azure { namespace Storage { namespace Test { = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName); // Rename File - const std::string oldFilename = GetTestName() + "file1"; - const std::string newFilename = GetTestName() + "file2"; - const std::string newFilename2 = GetTestName() + "file3"; + const std::string filename1 = GetTestName() + "file1"; + const std::string filename2 = GetTestName() + "file2"; + const std::string filename3 = GetTestName() + "file3"; + const std::string filename4 = GetTestName() + "file4"; - auto oldFileClient = fileSystemClient.GetFileClient(oldFilename); + auto oldFileClient = fileSystemClient.GetFileClient(filename1); oldFileClient.Create(); - auto newFileClient = fileSystemClient.RenameFile(oldFilename, newFilename).Value; + auto newFileClient = fileSystemClient.RenameFile(filename1, filename2).Value; auto properties = newFileClient.GetProperties(); EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); auto newFileClientWithoutEncryptionKey = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, newFilename); + AdlsGen2ConnectionString(), m_fileSystemName, filename2); EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(newFilename, newFilename2)); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); // Rename Directory const std::string testName(GetTestName()); @@ -462,6 +463,7 @@ namespace Azure { namespace Storage { namespace Test { auto newDirectoryClient = fileSystemClient.RenameDirectory(oldDirectoryName, newDirectoryName).Value; properties = newDirectoryClient.GetProperties(); + EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); auto newDirectoryClientWithoutEncryptionKey = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( @@ -469,11 +471,20 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( newDirectoryName, newDirectoryName2)); + + auto fileSystemClientWithEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, options); + auto created = fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); } // path works { const std::string pathName = "path"; + const std::string pathName2 = "path2"; + auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); @@ -490,11 +501,21 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetHttpHeaders( Files::DataLake::Models::PathHttpHeaders())); EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetPermissions("rwxrw-rw-")); + + auto pathClientWithEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName2, options); + auto created + = pathClientWithEncryptionKey.Create(Files::DataLake::Models::PathResourceType::File) + .Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); } // file works { const std::string fileName = "file"; + const std::string fileName2 = "file2"; auto fileClient = fileSystemClient.GetFileClient(fileName); auto fileClientWithoutEncryptionKey = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( @@ -520,13 +541,20 @@ namespace Azure { namespace Storage { namespace Test { auto properties = fileClient.GetProperties(); EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); - EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); + + auto fileClientWithEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName2, options); + auto created = fileClientWithEncryptionKey.Create().Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); } // directory works { const std::string directoryName = "directory"; + const std::string directoryName2 = "directory2"; const std::string subdirectoryName1 = "subdirectory1"; const std::string subdirectoryName2 = "subdirectory2"; const std::string subdirectoryName3 = "subdirectory3"; @@ -576,6 +604,13 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(newsubdirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameSubdirectory( subdirectoryName2, directoryName + "/" + subdirectoryName3)); + + auto directoryClientWithEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName2, options); + auto created = directoryClientWithEncryptionKey.Create().Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); } } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json deleted file mode 100644 index e54369fd52..0000000000 --- a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.CustomerProvidedKey.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "networkCallRecords": [ - { - "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", - "x-ms-client-request-id": "5d22ed71-0afa-4cb3-5797-398079facac4", - "x-ms-version": "2021-04-10" - }, - "Method": "PUT", - "Response": { - "BODY": "", - "REASON_PHRASE": "Created", - "STATUS_CODE": "201", - "content-length": "0", - "date": "Wed, 21 Sep 2022 06:43:09 GMT", - "etag": "\"0x8DA9B9C8C0EDDEE\"", - "last-modified": "Wed, 21 Sep 2022 06:43:09 GMT", - "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "5d22ed71-0afa-4cb3-5797-398079facac4", - "x-ms-request-id": "1d067a95-b01e-0058-4285-cd0af0000000", - "x-ms-version": "2021-04-10" - }, - "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestcustomerprovidedkey?restype=container" - }, - { - "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", - "x-ms-client-request-id": "f4ae2973-c1f5-443e-7e56-7284e0d78eb3", - "x-ms-version": "2021-04-10" - }, - "Method": "DELETE", - "Response": { - "BODY": "", - "REASON_PHRASE": "Accepted", - "STATUS_CODE": "202", - "content-length": "0", - "date": "Wed, 21 Sep 2022 06:44:43 GMT", - "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "f4ae2973-c1f5-443e-7e56-7284e0d78eb3", - "x-ms-request-id": "fc8205fe-701e-0025-6f85-cd7bd3000000", - "x-ms-version": "2021-04-10" - }, - "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestcustomerprovidedkey?restype=container" - } - ] -} From 70aca8ee0b223930bb1f3fd326fa397f47162ba1 Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 21 Sep 2022 15:18:23 +0800 Subject: [PATCH 3/5] remove chek skip list --- .../test/ut/datalake_file_system_client_test.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index add36e1387..8c53334001 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -400,8 +400,6 @@ namespace Azure { namespace Storage { namespace Test { TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey_LIVEONLY_) { - // will skip test under some cased where test can't run (usually LIVE only tests) - CHECK_SKIP_TEST(); auto getRandomCustomerProvidedKey = [&]() { Files::DataLake::EncryptionKey key; From 5e7d725265df65204427c0691ebd445937af43b8 Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 21 Sep 2022 16:47:06 +0800 Subject: [PATCH 4/5] move stack variables to heap --- .../ut/datalake_file_system_client_test.cpp | 132 ++++++++++-------- 1 file changed, 74 insertions(+), 58 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 8c53334001..d721481ea2 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -414,38 +414,45 @@ namespace Azure { namespace Storage { namespace Test { }; const int32_t bufferSize = 1024; // 1KB data size - std::vector buffer(bufferSize, 'x'); - Azure::Core::IO::MemoryBodyStream bodyStream(buffer.data(), buffer.size()); + auto buffer = std::make_shared>(bufferSize, 'x'); + Azure::Core::IO::MemoryBodyStream bodyStream(buffer->data(), buffer->size()); - auto customerProvidedKey = getRandomCustomerProvidedKey(); + auto customerProvidedKey + = std::make_shared(getRandomCustomerProvidedKey()); Files::DataLake::DataLakeClientOptions options; - options.CustomerProvidedKey = customerProvidedKey; - auto fileServiceClient - = Azure::Storage::Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), options); - auto fileSystemClient = fileServiceClient.GetFileSystemClient(m_fileSystemName); + options.CustomerProvidedKey = *customerProvidedKey; + auto fileServiceClient = std::make_shared( + Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), options)); + auto fileSystemClient = std::make_shared( + fileServiceClient->GetFileSystemClient(m_fileSystemName)); // fileSystem works { auto fileSystemClientWithoutEncryptionKey - = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName); + = std::make_shared( + Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName)); // Rename File const std::string filename1 = GetTestName() + "file1"; const std::string filename2 = GetTestName() + "file2"; const std::string filename3 = GetTestName() + "file3"; const std::string filename4 = GetTestName() + "file4"; - auto oldFileClient = fileSystemClient.GetFileClient(filename1); - oldFileClient.Create(); - auto newFileClient = fileSystemClient.RenameFile(filename1, filename2).Value; - auto properties = newFileClient.GetProperties(); - EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto oldFileClient = std::make_shared( + fileSystemClient->GetFileClient(filename1)); + oldFileClient->Create(); + auto newFileClient = std::make_shared( + fileSystemClient->RenameFile(filename1, filename2).Value); + auto properties = std::make_shared( + newFileClient->GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); auto newFileClientWithoutEncryptionKey - = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, filename2); - EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); + = std::make_shared( + Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, filename2)); + EXPECT_THROW(newFileClientWithoutEncryptionKey->GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey->RenameFile(filename2, filename3)); // Rename Directory const std::string testName(GetTestName()); @@ -453,29 +460,32 @@ namespace Azure { namespace Storage { namespace Test { const std::string newDirectoryName = testName + "dir2"; const std::string newDirectoryName2 = testName + "dir3"; - auto oldDirectoryClient = fileSystemClient.GetDirectoryClient(oldDirectoryName); + auto oldDirectoryClient = fileSystemClient->GetDirectoryClient(oldDirectoryName); oldDirectoryClient.Create(); oldDirectoryClient.GetFileClient(testName + "file3").Create(); oldDirectoryClient.GetSubdirectoryClient(testName + "dir4").Create(); auto newDirectoryClient - = fileSystemClient.RenameDirectory(oldDirectoryName, newDirectoryName).Value; - properties = newDirectoryClient.GetProperties(); - EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + = fileSystemClient->RenameDirectory(oldDirectoryName, newDirectoryName).Value; + properties = std::make_shared( + newDirectoryClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); auto newDirectoryClientWithoutEncryptionKey = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey->RenameDirectory( newDirectoryName, newDirectoryName2)); auto fileSystemClientWithEncryptionKey - = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, options); - auto created = fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value; - EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); + = std::make_shared( + Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, options)); + auto created = std::make_shared( + fileSystemClientWithEncryptionKey->GetFileClient(filename4).Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); } // path works @@ -487,9 +497,10 @@ namespace Azure { namespace Storage { namespace Test { AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); EXPECT_NO_THROW(pathClient.SetMetadata(GetMetadata())); - auto properties = pathClient.GetProperties(); - EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto properties = std::make_shared( + pathClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); auto pathClientWithoutEncryptionKey = Files::DataLake::DataLakePathClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, pathName); @@ -507,23 +518,23 @@ namespace Azure { namespace Storage { namespace Test { = pathClientWithEncryptionKey.Create(Files::DataLake::Models::PathResourceType::File) .Value; EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); + EXPECT_EQ(customerProvidedKey->KeyHash, created.EncryptionKeySha256.Value()); } // file works { const std::string fileName = "file"; const std::string fileName2 = "file2"; - auto fileClient = fileSystemClient.GetFileClient(fileName); + auto fileClient = fileSystemClient->GetFileClient(fileName); auto fileClientWithoutEncryptionKey = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, fileName); // upload test EXPECT_NO_THROW(fileClient.Create()); - EXPECT_NO_THROW(fileClient.UploadFrom(buffer.data(), bufferSize)); + EXPECT_NO_THROW(fileClient.UploadFrom(buffer->data(), bufferSize)); auto result = fileClient.Download(); - auto downloaded = ReadBodyStream(result.Value.Body); - EXPECT_EQ(buffer, downloaded); + auto downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); EXPECT_NO_THROW(fileClient.Delete()); // append test EXPECT_NO_THROW(fileClient.Create()); @@ -533,21 +544,23 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_THROW(fileClientWithoutEncryptionKey.Append(bodyStream, bufferSize), StorageException); EXPECT_NO_THROW(fileClient.Flush(bufferSize)); result = fileClient.Download(); - downloaded = ReadBodyStream(result.Value.Body); - EXPECT_EQ(buffer, downloaded); + downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); EXPECT_NO_THROW(fileClient.SetMetadata(GetMetadata())); - auto properties = fileClient.GetProperties(); - EXPECT_TRUE(properties.Value.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, properties.Value.EncryptionKeySha256.Value()); + auto properties = std::make_shared( + fileClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); auto fileClientWithEncryptionKey = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, fileName2, options); - auto created = fileClientWithEncryptionKey.Create().Value; - EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); + auto created = std::make_shared( + fileClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); } // directory works { @@ -559,7 +572,7 @@ namespace Azure { namespace Storage { namespace Test { const std::string fileName1 = "file1"; const std::string fileName2 = "file2"; const std::string fileName3 = "file3"; - auto directoryClient = fileSystemClient.GetDirectoryClient(directoryName); + auto directoryClient = fileSystemClient->GetDirectoryClient(directoryName); auto directoryClientWithoutEncryptionKey = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, directoryName); @@ -569,17 +582,18 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(subdirectoryClient.Create()); auto fileClient = directoryClient.GetFileClient(fileName1); EXPECT_NO_THROW(fileClient.Create()); - auto subdirectoryProperties = subdirectoryClient.GetProperties(); - EXPECT_EQ( - customerProvidedKey.KeyHash, subdirectoryProperties.Value.EncryptionKeySha256.Value()); + auto subdirectoryProperties = std::make_shared( + subdirectoryClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, subdirectoryProperties->EncryptionKeySha256.Value()); auto fileProperties = fileClient.GetProperties(); - EXPECT_EQ(customerProvidedKey.KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); + EXPECT_EQ(customerProvidedKey->KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); // rename file auto newFileClient = directoryClient.RenameFile(fileName1, directoryName + "/" + fileName2).Value; - auto newFileProperties = newFileClient.GetProperties(); - EXPECT_EQ(customerProvidedKey.KeyHash, newFileProperties.Value.EncryptionKeySha256.Value()); + auto newFileProperties = std::make_shared( + newFileClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, newFileProperties->EncryptionKeySha256.Value()); auto newFileClientWithoutEncryptionKey = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, directoryName + "/" + fileName2); @@ -591,9 +605,10 @@ namespace Azure { namespace Storage { namespace Test { = directoryClient .RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) .Value; - auto newSubdirectoryProperties = newSubdirectoryClient.GetProperties(); + auto newSubdirectoryProperties = std::make_shared( + newSubdirectoryClient.GetProperties().Value); EXPECT_EQ( - customerProvidedKey.KeyHash, newSubdirectoryProperties.Value.EncryptionKeySha256.Value()); + customerProvidedKey->KeyHash, newSubdirectoryProperties->EncryptionKeySha256.Value()); auto newsubdirectoryClientWithoutEncryptionKey = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( AdlsGen2ConnectionString(), @@ -606,9 +621,10 @@ namespace Azure { namespace Storage { namespace Test { auto directoryClientWithEncryptionKey = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, directoryName2, options); - auto created = directoryClientWithEncryptionKey.Create().Value; - EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); - EXPECT_EQ(customerProvidedKey.KeyHash, created.EncryptionKeySha256.Value()); + auto created = std::make_shared( + directoryClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); } } From 0cad52a96e0b266e11b468b4a0ef916cca666b9b Mon Sep 17 00:00:00 2001 From: Zhendong Chang Date: Wed, 21 Sep 2022 16:55:06 +0800 Subject: [PATCH 5/5] update make_shared --- .../ut/datalake_file_system_client_test.cpp | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index d721481ea2..d299436ebc 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -430,29 +430,25 @@ namespace Azure { namespace Storage { namespace Test { // fileSystem works { auto fileSystemClientWithoutEncryptionKey - = std::make_shared( - Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName)); + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); // Rename File const std::string filename1 = GetTestName() + "file1"; const std::string filename2 = GetTestName() + "file2"; const std::string filename3 = GetTestName() + "file3"; const std::string filename4 = GetTestName() + "file4"; - auto oldFileClient = std::make_shared( - fileSystemClient->GetFileClient(filename1)); - oldFileClient->Create(); - auto newFileClient = std::make_shared( - fileSystemClient->RenameFile(filename1, filename2).Value); + auto oldFileClient = fileSystemClient->GetFileClient(filename1); + oldFileClient.Create(); + auto newFileClient = fileSystemClient->RenameFile(filename1, filename2).Value; auto properties = std::make_shared( - newFileClient->GetProperties().Value); + newFileClient.GetProperties().Value); EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); auto newFileClientWithoutEncryptionKey - = std::make_shared( - Files::DataLake::DataLakeFileClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, filename2)); - EXPECT_THROW(newFileClientWithoutEncryptionKey->GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey->RenameFile(filename2, filename3)); + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, filename2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); // Rename Directory const std::string testName(GetTestName()); @@ -475,15 +471,14 @@ namespace Azure { namespace Storage { namespace Test { = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); - EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey->RenameDirectory( + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( newDirectoryName, newDirectoryName2)); auto fileSystemClientWithEncryptionKey - = std::make_shared( - Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( - AdlsGen2ConnectionString(), m_fileSystemName, options)); + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, options); auto created = std::make_shared( - fileSystemClientWithEncryptionKey->GetFileClient(filename4).Create().Value); + fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value); EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); }