Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

protocol layer for data lake STG82 #3927

Merged
merged 3 commits into from
Sep 14, 2022
Merged

protocol layer for data lake STG82 #3927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88138fe
protocol layer for data lake STG82
microzchang Sep 9, 2022
0fd902b
fix conversation
microzchang Sep 13, 2022
96062ab
fix conversation
microzchang Sep 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 64 additions & 2 deletions sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
/**
* The version used for the operations to Azure storage services.
*/
constexpr static const char* ApiVersion = "2020-02-10";
constexpr static const char* ApiVersion = "2021-06-08";
} // namespace _detail
namespace Models {
namespace _detail {
Expand Down Expand Up @@ -92,6 +92,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
std::string Owner;
std::string Group;
std::string Permissions;
/**
* The name of the encryption scope under which the blob is encrypted.
*/
Nullable<std::string> EncryptionScope;
std::string ETag;
};
namespace _detail {
Expand Down Expand Up @@ -152,6 +156,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* The size of the resource in bytes.
*/
Nullable<int64_t> FileSize;
/**
* The value of this header is set to true if the contents of the request are successfully
* encrypted using the specified algorithm, and false otherwise.
*/
Nullable<bool> IsServerEncrypted;
/**
* The SHA-256 hash of the encryption key used to encrypt the blob. This header is only
* returned when the blob was encrypted with a customer-provided key.
*/
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
/**
* @brief Response type for #Azure::Storage::Files::DataLake::PathClient::Delete.
Expand Down Expand Up @@ -220,6 +234,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
*/
Nullable<std::string> ContinuationToken;
};
/**
* @brief Response type for #Azure::Storage::Files::DataLake::PathClient::Undelete.
*/
struct UndeletePathResult final
{
/**
* The type of the resource. The value may be "file" or "directory". If not set, the value
* is "file".
*/
Nullable<std::string> ResourceType;
};
/**
* @brief Response type for
* #Azure::Storage::Files::DataLake::PathClient::GetAccessControlList.
Expand Down Expand Up @@ -267,6 +292,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* The size of the resource in bytes.
*/
int64_t FileSize = int64_t();
/**
* The value of this header is set to true if the contents of the request are successfully
* encrypted using the specified algorithm, and false otherwise.
*/
Nullable<bool> IsServerEncrypted;
/**
* The SHA-256 hash of the encryption key used to encrypt the blob. This header is only
* returned when the blob was encrypted with a customer-provided key.
*/
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
/**
* @brief Response type for #Azure::Storage::Files::DataLake::FileClient::Append.
Expand All @@ -282,7 +317,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* The value of this header is set to true if the contents of the request are successfully
* encrypted using the specified algorithm, and false otherwise.
*/
bool IsServerEncrypted = bool();
Nullable<bool> IsServerEncrypted;
/**
* The SHA-256 hash of the encryption key used to encrypt the blob. This header is only
* returned when the blob was encrypted with a customer-provided key.
*/
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
} // namespace Models
namespace _detail {
Expand Down Expand Up @@ -332,6 +372,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
ETag SourceIfNoneMatch;
Nullable<DateTime> SourceIfModifiedSince;
Nullable<DateTime> SourceIfUnmodifiedSince;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<std::string> Owner;
Nullable<std::string> Group;
Nullable<std::string> Acl;
Nullable<std::string> ProposedLeaseId;
Nullable<int64_t> LeaseDuration;
Nullable<std::string> ExpiresOn;
};
static Response<Models::CreatePathResult> Create(
Core::Http::_internal::HttpPipeline& pipeline,
Expand Down Expand Up @@ -386,6 +434,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const Core::Url& url,
const SetPathAccessControlListRecursiveOptions& options,
const Core::Context& context);
struct UndeletePathOptions final
{
Nullable<std::string> UndeleteSource;
};
static Response<Models::_detail::UndeletePathResult> Undelete(
Core::Http::_internal::HttpPipeline& pipeline,
const Core::Url& url,
const UndeletePathOptions& options,
const Core::Context& context);
struct GetPathAccessControlListOptions final
{
Nullable<bool> Upn;
Expand Down Expand Up @@ -419,6 +476,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
ETag IfNoneMatch;
Nullable<DateTime> IfModifiedSince;
Nullable<DateTime> IfUnmodifiedSince;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
};
static Response<Models::FlushFileResult> Flush(
Core::Http::_internal::HttpPipeline& pipeline,
Expand All @@ -431,6 +490,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<std::vector<uint8_t>> TransactionalContentHash;
Nullable<std::vector<uint8_t>> TransactionalContentCrc64;
Nullable<std::string> LeaseId;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<bool> Flush;
};
static Response<Models::AppendFileResult> Append(
Core::Http::_internal::HttpPipeline& pipeline,
Expand Down
143 changes: 133 additions & 10 deletions sdk/storage/azure-storage-files-datalake/src/rest_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
if (options.ContinuationToken.HasValue() && !options.ContinuationToken.Value().empty())
{
request.GetUrl().AppendQueryParameter(
Expand Down Expand Up @@ -110,6 +110,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
vectorElement2.Owner = var0["owner"].get<std::string>();
vectorElement2.Group = var0["group"].get<std::string>();
vectorElement2.Permissions = var0["permissions"].get<std::string>();
if (var0.count("EncryptionScope") != 0)
{
vectorElement2.EncryptionScope = var0["EncryptionScope"].get<std::string>();
}
if (var0.count("etag") != 0)
{
vectorElement2.ETag = var0["etag"].get<std::string>();
Expand Down Expand Up @@ -138,7 +142,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
if (options.Resource.HasValue() && !options.Resource.Value().ToString().empty())
{
request.GetUrl().AppendQueryParameter(
Expand Down Expand Up @@ -238,6 +242,42 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"x-ms-source-if-unmodified-since",
options.SourceIfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value());
}
if (options.EncryptionKeySha256.HasValue()
&& !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty())
{
request.SetHeader(
"x-ms-encryption-key-sha256",
Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()));
}
request.SetHeader("x-ms-encryption-algorithm", "AES256");
if (options.Owner.HasValue() && !options.Owner.Value().empty())
{
request.SetHeader("x-ms-owner", options.Owner.Value());
}
if (options.Group.HasValue() && !options.Group.Value().empty())
{
request.SetHeader("x-ms-group", options.Group.Value());
}
if (options.Acl.HasValue() && !options.Acl.Value().empty())
{
request.SetHeader("x-ms-acl", options.Acl.Value());
}
if (options.ProposedLeaseId.HasValue() && !options.ProposedLeaseId.Value().empty())
{
request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value());
}
if (options.LeaseDuration.HasValue())
{
request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value()));
}
if (options.ExpiresOn.HasValue() && !options.ExpiresOn.Value().empty())
{
request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value());
}
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Created)
Expand All @@ -252,6 +292,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
response.FileSize = std::stoll(pRawResponse->GetHeaders().at("Content-Length"));
}
if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0)
{
response.IsServerEncrypted
= pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true");
}
if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0)
{
response.EncryptionKeySha256 = Core::Convert::Base64Decode(
pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256"));
}
return Response<Models::CreatePathResult>(std::move(response), std::move(pRawResponse));
}
Response<Models::DeletePathResult> PathClient::Delete(
Expand All @@ -269,7 +319,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value()));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
if (options.Recursive.HasValue())
{
request.GetUrl().AppendQueryParameter(
Expand Down Expand Up @@ -361,7 +411,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
Expand Down Expand Up @@ -408,7 +458,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-acl", options.Acl.Value());
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
Expand Down Expand Up @@ -449,6 +499,33 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
return Response<Models::_detail::SetAccessControlListRecursiveResult>(
std::move(response), std::move(pRawResponse));
}
Response<Models::_detail::UndeletePathResult> PathClient::Undelete(
Core::Http::_internal::HttpPipeline& pipeline,
const Core::Url& url,
const UndeletePathOptions& options,
const Core::Context& context)
{
auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url);
request.GetUrl().AppendQueryParameter("comp", "undelete");
if (options.UndeleteSource.HasValue() && !options.UndeleteSource.Value().empty())
{
request.SetHeader("x-ms-undelete-source", options.UndeleteSource.Value());
}
request.SetHeader("x-ms-version", "2021-06-08");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
{
throw StorageException::CreateFromResponse(std::move(pRawResponse));
}
Models::_detail::UndeletePathResult response;
if (pRawResponse->GetHeaders().count("x-ms-resource-type") != 0)
{
response.ResourceType = pRawResponse->GetHeaders().at("x-ms-resource-type");
}
return Response<Models::_detail::UndeletePathResult>(
std::move(response), std::move(pRawResponse));
}
Response<Models::_detail::GetPathAccessControlListResult> PathClient::GetAccessControlList(
Core::Http::_internal::HttpPipeline& pipeline,
const Core::Url& url,
Expand Down Expand Up @@ -485,7 +562,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
Expand Down Expand Up @@ -572,7 +649,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
"If-Unmodified-Since",
options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123));
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value());
}
if (options.EncryptionKeySha256.HasValue()
&& !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty())
{
request.SetHeader(
"x-ms-encryption-key-sha256",
Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()));
}
request.SetHeader("x-ms-encryption-algorithm", "AES256");
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
Expand All @@ -584,6 +673,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
response.LastModified = DateTime::Parse(
pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123);
response.FileSize = std::stoll(pRawResponse->GetHeaders().at("Content-Length"));
if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0)
{
response.IsServerEncrypted
= pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true");
}
if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0)
{
response.EncryptionKeySha256 = Core::Convert::Base64Decode(
pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256"));
}
return Response<Models::FlushFileResult>(std::move(response), std::move(pRawResponse));
}
Response<Models::AppendFileResult> FileClient::Append(
Expand Down Expand Up @@ -617,7 +716,23 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-lease-id", options.LeaseId.Value());
}
request.SetHeader("x-ms-version", "2020-02-10");
request.SetHeader("x-ms-version", "2021-06-08");
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value());
}
if (options.EncryptionKeySha256.HasValue()
&& !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty())
{
request.SetHeader(
"x-ms-encryption-key-sha256",
Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()));
}
request.SetHeader("x-ms-encryption-algorithm", "AES256");
if (options.Flush.HasValue())
{
request.GetUrl().AppendQueryParameter("flush", options.Flush.Value() ? "true" : "false");
}
auto pRawResponse = pipeline.Send(request, context);
auto httpStatusCode = pRawResponse->GetStatusCode();
if (httpStatusCode != Core::Http::HttpStatusCode::Accepted)
Expand All @@ -639,8 +754,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
= Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64"));
response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64;
}
response.IsServerEncrypted
= pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true");
if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0)
{
response.IsServerEncrypted
= pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true");
}
if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0)
{
response.EncryptionKeySha256 = Core::Convert::Base64Decode(
pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256"));
}
return Response<Models::AppendFileResult>(std::move(response), std::move(pRawResponse));
}
} // namespace _detail
Expand Down
Loading