Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cpk for datalake #3955

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
explicit DataLakeDirectoryClient(
Azure::Core::Url directoryUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
: DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline)
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
: DataLakePathClient(
std::move(directoryUrl),
std::move(blobClient),
pipeline,
customerProvidedKey)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -282,8 +282,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
explicit DataLakeFileClient(
Azure::Core::Url fileUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
: DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline)
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
: DataLakePathClient(
std::move(fileUrl),
std::move(blobClient),
pipeline,
customerProvidedKey)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,13 +243,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url m_fileSystemUrl;
Blobs::BlobContainerClient m_blobContainerClient;
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
Azure::Nullable<EncryptionKey> m_customerProvidedKey;

explicit DataLakeFileSystemClient(
Azure::Core::Url fileSystemUrl,
Blobs::BlobContainerClient blobContainerClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
: m_fileSystemUrl(std::move(fileSystemUrl)),
m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline))
m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)),
m_customerProvidedKey(std::move(customerProvidedKey))
{
}
friend class DataLakeLeaseClient;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
using SignedIdentifier = Blobs::Models::SignedIdentifier;
using FileQueryArrowField = Blobs::Models::BlobQueryArrowField;
using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType;
using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType;
} // namespace Models

using DownloadFileToOptions = Blobs::DownloadBlobToOptions;
using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions;
using EncryptionKey = Blobs::EncryptionKey;

/**
* @brief Client options used to initialize all DataLake clients.
Expand All @@ -46,6 +48,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* API version used by this client.
*/
std::string ApiVersion;

/**
* @brief Holds the customer provided key used when making requests.
*/
Azure::Nullable<EncryptionKey> CustomerProvidedKey;
};

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,13 +295,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url m_pathUrl;
Blobs::BlobClient m_blobClient;
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
Azure::Nullable<EncryptionKey> m_customerProvidedKey;

explicit DataLakePathClient(
Azure::Core::Url pathUrl,
Blobs::BlobClient blobClient,
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
: m_pathUrl(std::move(pathUrl)), m_blobClient(std::move(blobClient)),
m_pipeline(std::move(pipeline))
m_pipeline(std::move(pipeline)), m_customerProvidedKey(std::move(customerProvidedKey))
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,5 +113,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Azure::Core::Url m_serviceUrl;
Blobs::BlobServiceClient m_blobServiceClient;
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
Azure::Nullable<EncryptionKey> m_customerProvidedKey;
};
}}}} // namespace Azure::Storage::Files::DataLake
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<DateTime> SourceIfUnmodifiedSince;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<std::string> EncryptionAlgorithm;
microzchang marked this conversation as resolved.
Show resolved Hide resolved
Nullable<std::string> Owner;
Nullable<std::string> Group;
Nullable<std::string> Acl;
Expand Down Expand Up @@ -478,6 +479,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<DateTime> IfUnmodifiedSince;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<std::string> EncryptionAlgorithm;
};
static Response<Models::FlushFileResult> Flush(
Core::Http::_internal::HttpPipeline& pipeline,
Expand All @@ -492,6 +494,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<std::string> LeaseId;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<std::string> EncryptionAlgorithm;
Nullable<bool> Flush;
};
static Response<Models::AppendFileResult> Append(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
builder.AppendPath(_internal::UrlEncodePath(fileName));
auto blobClient = m_blobClient;
blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(fileName));
return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline);
return DataLakeFileClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
}

DataLakeDirectoryClient DataLakeDirectoryClient::GetSubdirectoryClient(
Expand All @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
builder.AppendPath(_internal::UrlEncodePath(subdirectoryName));
auto blobClient = m_blobClient;
blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName));
return DataLakeDirectoryClient(std::move(builder), std::move(blobClient), m_pipeline);
return DataLakeDirectoryClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
}

Azure::Response<DataLakeFileClient> DataLakeDirectoryClient::RenameFile(
Expand Down Expand Up @@ -119,10 +121,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto response = _detail::PathClient::Create(
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);

auto renamedBlobClient
= Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
auto renamedFileClient = DataLakeFileClient(
std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline);
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
return Azure::Response<DataLakeFileClient>(
std::move(renamedFileClient), std::move(response.RawResponse));
}
Expand Down Expand Up @@ -167,10 +172,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto response = _detail::PathClient::Create(
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);

auto renamedBlobClient
= Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
auto renamedDirectoryClient = DataLakeDirectoryClient(
std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline);
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
return Azure::Response<DataLakeDirectoryClient>(
std::move(renamedDirectoryClient), std::move(response.RawResponse));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
}
}
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
if (m_customerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
return _detail::FileClient::Append(
*m_pipeline, m_pathUrl, content, protocolLayerOptions, context);
}
Expand Down Expand Up @@ -111,6 +117,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
if (m_customerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
return _detail::FileClient::Flush(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
Expand Down Expand Up @@ -78,7 +79,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand Down Expand Up @@ -107,7 +109,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const DataLakeClientOptions& options)
: m_fileSystemUrl(fileSystemUrl), m_blobContainerClient(
_detail::GetBlobUrlFromUrl(fileSystemUrl),
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand All @@ -129,7 +132,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto builder = m_fileSystemUrl;
builder.AppendPath(_internal::UrlEncodePath(fileName));
auto blobClient = m_blobContainerClient.GetBlobClient(fileName);
return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline);
return DataLakeFileClient(
std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey);
}

DataLakeDirectoryClient DataLakeFileSystemClient::GetDirectoryClient(
Expand All @@ -138,7 +142,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto builder = m_fileSystemUrl;
builder.AppendPath(_internal::UrlEncodePath(directoryName));
return DataLakeDirectoryClient(
builder, m_blobContainerClient.GetBlobClient(directoryName), m_pipeline);
builder,
m_blobContainerClient.GetBlobClient(directoryName),
m_pipeline,
m_customerProvidedKey);
}

Azure::Response<Models::CreateFileSystemResult> DataLakeFileSystemClient::Create(
Expand Down Expand Up @@ -356,10 +363,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto result = _detail::PathClient::Create(
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);

auto renamedBlobClient
= Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
auto renamedFileClient = DataLakeFileClient(
std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline);
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
return Azure::Response<DataLakeFileClient>(
std::move(renamedFileClient), std::move(result.RawResponse));
}
Expand Down Expand Up @@ -404,10 +414,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto result = _detail::PathClient::Create(
*m_pipeline, destinationDfsUrl, protocolLayerOptions, context);

auto renamedBlobClient
= Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline);
auto renamedBlobClient = Blobs::BlobClient(
_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey);
auto renamedDirectoryClient = DataLakeDirectoryClient(
std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline);
std::move(destinationDfsUrl),
std::move(renamedBlobClient),
m_pipeline,
m_customerProvidedKey);
return Azure::Response<DataLakeDirectoryClient>(
std::move(renamedDirectoryClient), std::move(result.RawResponse));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_pathUrl(pathUrl), m_blobClient(
_detail::GetBlobUrlFromUrl(pathUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
Expand Down Expand Up @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_pathUrl(pathUrl), m_blobClient(
_detail::GetBlobUrlFromUrl(pathUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand Down Expand Up @@ -104,7 +106,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const std::string& pathUrl,
const DataLakeClientOptions& options)
: m_pathUrl(pathUrl),
m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options))
m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand Down Expand Up @@ -202,6 +205,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata);
protocolLayerOptions.Umask = options.Umask;
protocolLayerOptions.Permissions = options.Permissions;
if (m_customerProvidedKey.HasValue())
{
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
DataLakeClientOptions newOptions = options;
newOptions.PerRetryPolicies.emplace_back(
Expand Down Expand Up @@ -72,7 +73,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
credential,
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand Down Expand Up @@ -101,7 +103,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const DataLakeClientOptions& options)
: m_serviceUrl(serviceUrl), m_blobServiceClient(
_detail::GetBlobUrlFromUrl(serviceUrl),
_detail::GetBlobClientOptions(options))
_detail::GetBlobClientOptions(options)),
m_customerProvidedKey(options.CustomerProvidedKey)
{
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perRetryPolicies;
std::vector<std::unique_ptr<Azure::Core::Http::Policies::HttpPolicy>> perOperationPolicies;
Expand All @@ -124,7 +127,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
auto builder = m_serviceUrl;
builder.AppendPath(_internal::UrlEncodePath(fileSystemName));
return DataLakeFileSystemClient(
builder, m_blobServiceClient.GetBlobContainerClient(fileSystemName), m_pipeline);
builder,
m_blobServiceClient.GetBlobContainerClient(fileSystemName),
m_pipeline,
m_customerProvidedKey);
}

ListFileSystemsPagedResponse DataLakeServiceClient::ListFileSystems(
Expand Down
Loading