Skip to content

Commit

Permalink
datalake append/flush with lease (#4110)
Browse files Browse the repository at this point in the history
  • Loading branch information
@microzchang
microzchang authored Nov 15, 2022
1 parent ab63938 commit 6a3fe23
Show file tree
Hide file tree
Showing 8 changed files with 1,282 additions and 0 deletions.
44 changes: 44 additions & 0 deletions ...torage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,28 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* If true, the file will be flushed after the append.
*/
Azure::Nullable<bool> Flush;

/**
* If "acquire" it will acquire the lease.
* If "auto-renew" it will renew the lease.
* If "release" it will release the lease only on flush. Only applicable if Flush is set to
* true.
* If "acquire-release" it will acquire & complete the operation & release the lease once
* operation is done. Only applicable if Flush is set to true.
*/
Azure::Nullable<Models::LeaseAction> LeaseAction;

/**
* Proposed LeaseId.
*/
Azure::Nullable<std::string> LeaseId;

/**
* Specifies the duration of the lease, in seconds, or InfiniteLeaseDuration for a lease that
* never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot
* be changed using renew or change.
*/
Azure::Nullable<std::chrono::seconds> LeaseDuration;
};

/**
Expand Down Expand Up @@ -418,6 +440,28 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* Specify the access condition for the path.
*/
PathAccessConditions AccessConditions;

/**
* If "acquire" it will acquire the lease.
* If "auto-renew" it will renew the lease.
* If "release" it will release the lease only on flush. Only applicable if Flush is set to
* true.
* If "acquire-release" it will acquire & complete the operation & release the lease once
* operation is done. Only applicable if Flush is set to true.
*/
Azure::Nullable<Models::LeaseAction> LeaseAction;

/**
* Proposed LeaseId.
*/
Azure::Nullable<std::string> LeaseId;

/**
* Specifies the duration of the lease, in seconds, or InfiniteLeaseDuration for a lease that
* never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot
* be changed using renew or change.
*/
Azure::Nullable<std::chrono::seconds> LeaseDuration;
};

/**
Expand Down
34 changes: 34 additions & 0 deletions sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,26 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
std::string Acl;
};
} // namespace _detail
/**
* @brief Optional. If "acquire" it will acquire the lease. If "auto-renew" it will renew the
* lease. If "release" it will release the lease only on flush. If "acquire-release" it will
* acquire & complete the operation & release the lease once operation is done.
*/
class LeaseAction final {
public:
LeaseAction() = default;
explicit LeaseAction(std::string value) : m_value(std::move(value)) {}
bool operator==(const LeaseAction& other) const { return m_value == other.m_value; }
bool operator!=(const LeaseAction& other) const { return !(*this == other); }
const std::string& ToString() const { return m_value; }
AZ_STORAGE_FILES_DATALAKE_DLLEXPORT const static LeaseAction Acquire;
AZ_STORAGE_FILES_DATALAKE_DLLEXPORT const static LeaseAction AutoRenew;
AZ_STORAGE_FILES_DATALAKE_DLLEXPORT const static LeaseAction Release;
AZ_STORAGE_FILES_DATALAKE_DLLEXPORT const static LeaseAction AcquireRelease;

private:
std::string m_value;
};
/**
* @brief Response type for #Azure::Storage::Files::DataLake::FileClient::Flush.
*/
Expand Down Expand Up @@ -304,6 +324,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* returned when the blob was encrypted with a customer-provided key.
*/
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
/**
* If the lease was auto-renewed with this request.
*/
Nullable<bool> LeaseRenewed;
};
/**
* @brief Response type for #Azure::Storage::Files::DataLake::FileClient::Append.
Expand All @@ -325,6 +349,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
* returned when the blob was encrypted with a customer-provided key.
*/
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
/**
* If the lease was auto-renewed with this request.
*/
Nullable<bool> LeaseRenewed;
};
} // namespace Models
namespace _detail {
Expand Down Expand Up @@ -471,6 +499,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<bool> Close;
Nullable<std::vector<uint8_t>> ContentMD5;
Nullable<std::string> LeaseId;
Nullable<Models::LeaseAction> LeaseAction;
Nullable<int64_t> LeaseDuration;
Nullable<std::string> ProposedLeaseId;
Nullable<std::string> CacheControl;
Nullable<std::string> ContentType;
Nullable<std::string> ContentDisposition;
Expand All @@ -495,6 +526,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
Nullable<std::vector<uint8_t>> TransactionalContentHash;
Nullable<std::vector<uint8_t>> TransactionalContentCrc64;
Nullable<std::string> LeaseId;
Nullable<Models::LeaseAction> LeaseAction;
Nullable<int64_t> LeaseDuration;
Nullable<std::string> ProposedLeaseId;
Nullable<std::string> EncryptionKey;
Nullable<std::vector<uint8_t>> EncryptionKeySha256;
Nullable<std::string> EncryptionAlgorithm;
Expand Down
12 changes: 12 additions & 0 deletions sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.LeaseAction = options.LeaseAction;
protocolLayerOptions.ProposedLeaseId = options.LeaseId;
if (options.LeaseDuration.HasValue())
{
protocolLayerOptions.LeaseDuration = static_cast<int64_t>(options.LeaseDuration->count());
}
return _detail::FileClient::Append(
*m_pipeline, m_pathUrl, content, protocolLayerOptions, context);
}
Expand Down Expand Up @@ -124,6 +130,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
}
protocolLayerOptions.LeaseAction = options.LeaseAction;
protocolLayerOptions.ProposedLeaseId = options.LeaseId;
if (options.LeaseDuration.HasValue())
{
protocolLayerOptions.LeaseDuration = static_cast<int64_t>(options.LeaseDuration->count());
}
return _detail::FileClient::Flush(*m_pipeline, m_pathUrl, protocolLayerOptions, context);
}

Expand Down
38 changes: 38 additions & 0 deletions sdk/storage/azure-storage-files-datalake/src/rest_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
const PublicAccessType PublicAccessType::Path("blob");
const PathResourceType PathResourceType::Directory("directory");
const PathResourceType PathResourceType::File("file");
const LeaseAction LeaseAction::Acquire("acquire");
const LeaseAction LeaseAction::AutoRenew("auto-renew");
const LeaseAction LeaseAction::Release("release");
const LeaseAction LeaseAction::AcquireRelease("acquire-release");
} // namespace Models
namespace _detail {
Response<Models::_detail::PathList> FileSystemClient::ListPaths(
Expand Down Expand Up @@ -624,6 +628,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-lease-id", options.LeaseId.Value());
}
if (options.LeaseAction.HasValue() && !options.LeaseAction.Value().ToString().empty())
{
request.SetHeader("x-ms-lease-action", options.LeaseAction.Value().ToString());
}
if (options.LeaseDuration.HasValue())
{
request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value()));
}
if (options.ProposedLeaseId.HasValue() && !options.ProposedLeaseId.Value().empty())
{
request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value());
}
if (options.CacheControl.HasValue() && !options.CacheControl.Value().empty())
{
request.SetHeader("x-ms-cache-control", options.CacheControl.Value());
Expand Down Expand Up @@ -698,6 +714,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
response.EncryptionKeySha256 = Core::Convert::Base64Decode(
pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256"));
}
if (pRawResponse->GetHeaders().count("x-ms-lease-renewed") != 0)
{
response.LeaseRenewed
= pRawResponse->GetHeaders().at("x-ms-lease-renewed") == std::string("true");
}
return Response<Models::FlushFileResult>(std::move(response), std::move(pRawResponse));
}
Response<Models::AppendFileResult> FileClient::Append(
Expand Down Expand Up @@ -731,6 +752,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
{
request.SetHeader("x-ms-lease-id", options.LeaseId.Value());
}
if (options.LeaseAction.HasValue() && !options.LeaseAction.Value().ToString().empty())
{
request.SetHeader("x-ms-lease-action", options.LeaseAction.Value().ToString());
}
if (options.LeaseDuration.HasValue())
{
request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value()));
}
if (options.ProposedLeaseId.HasValue() && !options.ProposedLeaseId.Value().empty())
{
request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value());
}
request.SetHeader("x-ms-version", "2021-06-08");
if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty())
{
Expand Down Expand Up @@ -779,6 +812,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
response.EncryptionKeySha256 = Core::Convert::Base64Decode(
pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256"));
}
if (pRawResponse->GetHeaders().count("x-ms-lease-renewed") != 0)
{
response.LeaseRenewed
= pRawResponse->GetHeaders().at("x-ms-lease-renewed") == std::string("true");
}
return Response<Models::AppendFileResult>(std::move(response), std::move(pRawResponse));
}
} // namespace _detail
Expand Down
2 changes: 2 additions & 0 deletions sdk/storage/azure-storage-files-datalake/swagger/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,7 @@ directive:
$["x-ms-content-crc64"]["x-ms-client-name"] = "TransactionalContentHash";
$["x-ms-content-crc64"]["x-nullable"] = true;
$["x-ms-encryption-key-sha256"]["x-nullable"] = true;
$["x-ms-lease-renewed"]["x-nullable"] = true;
delete $["ETag"];
```
Expand All @@ -450,4 +451,5 @@ directive:
transform: >
$["Content-Length"]["x-ms-client-name"] = "FileSize";
$["x-ms-encryption-key-sha256"]["x-nullable"] = true;
$["x-ms-lease-renewed"]["x-nullable"] = true;
```
Loading

0 comments on commit 6a3fe23

Please sign in to comment.