From d3c5dd41b43563109c89fde13730c9393cf593dc Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Tue, 13 Sep 2022 22:24:26 -0700 Subject: [PATCH 01/11] protocol layer for data lake STG82 (#3927) --- .../storage/files/datalake/rest_client.hpp | 66 +++++++- .../src/rest_client.cpp | 143 ++++++++++++++++-- .../swagger/README.md | 100 +++++++++++- 3 files changed, 292 insertions(+), 17 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index 9118b085cd..0a58b8e565 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -26,7 +26,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { /** * The version used for the operations to Azure storage services. */ - constexpr static const char* ApiVersion = "2020-02-10"; + constexpr static const char* ApiVersion = "2021-06-08"; } // namespace _detail namespace Models { namespace _detail { @@ -92,6 +92,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::string Owner; std::string Group; std::string Permissions; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; std::string ETag; }; namespace _detail { @@ -152,6 +156,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The size of the resource in bytes. */ Nullable FileSize; + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + Nullable IsServerEncrypted; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only + * returned when the blob was encrypted with a customer-provided key. + */ + Nullable> EncryptionKeySha256; }; /** * @brief Response type for #Azure::Storage::Files::DataLake::PathClient::Delete. @@ -220,6 +234,17 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { */ Nullable ContinuationToken; }; + /** + * @brief Response type for #Azure::Storage::Files::DataLake::PathClient::Undelete. + */ + struct UndeletePathResult final + { + /** + * The type of the resource. The value may be "file" or "directory". If not set, the value + * is "file". + */ + Nullable ResourceType; + }; /** * @brief Response type for * #Azure::Storage::Files::DataLake::PathClient::GetAccessControlList. @@ -267,6 +292,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The size of the resource in bytes. */ int64_t FileSize = int64_t(); + /** + * The value of this header is set to true if the contents of the request are successfully + * encrypted using the specified algorithm, and false otherwise. + */ + Nullable IsServerEncrypted; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only + * returned when the blob was encrypted with a customer-provided key. + */ + Nullable> EncryptionKeySha256; }; /** * @brief Response type for #Azure::Storage::Files::DataLake::FileClient::Append. @@ -282,7 +317,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The value of this header is set to true if the contents of the request are successfully * encrypted using the specified algorithm, and false otherwise. */ - bool IsServerEncrypted = bool(); + Nullable IsServerEncrypted; + /** + * The SHA-256 hash of the encryption key used to encrypt the blob. This header is only + * returned when the blob was encrypted with a customer-provided key. + */ + Nullable> EncryptionKeySha256; }; } // namespace Models namespace _detail { @@ -332,6 +372,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ETag SourceIfNoneMatch; Nullable SourceIfModifiedSince; Nullable SourceIfUnmodifiedSince; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable Owner; + Nullable Group; + Nullable Acl; + Nullable ProposedLeaseId; + Nullable LeaseDuration; + Nullable ExpiresOn; }; static Response Create( Core::Http::_internal::HttpPipeline& pipeline, @@ -386,6 +434,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const Core::Url& url, const SetPathAccessControlListRecursiveOptions& options, const Core::Context& context); + struct UndeletePathOptions final + { + Nullable UndeleteSource; + }; + static Response Undelete( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const UndeletePathOptions& options, + const Core::Context& context); struct GetPathAccessControlListOptions final { Nullable Upn; @@ -419,6 +476,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ETag IfNoneMatch; Nullable IfModifiedSince; Nullable IfUnmodifiedSince; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; }; static Response Flush( Core::Http::_internal::HttpPipeline& pipeline, @@ -431,6 +490,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable> TransactionalContentHash; Nullable> TransactionalContentCrc64; Nullable LeaseId; + Nullable EncryptionKey; + Nullable> EncryptionKeySha256; + Nullable Flush; }; static Response Append( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index a19745cfdc..2d519a9487 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -58,7 +58,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value())); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); if (options.ContinuationToken.HasValue() && !options.ContinuationToken.Value().empty()) { request.GetUrl().AppendQueryParameter( @@ -110,6 +110,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { vectorElement2.Owner = var0["owner"].get(); vectorElement2.Group = var0["group"].get(); vectorElement2.Permissions = var0["permissions"].get(); + if (var0.count("EncryptionScope") != 0) + { + vectorElement2.EncryptionScope = var0["EncryptionScope"].get(); + } if (var0.count("etag") != 0) { vectorElement2.ETag = var0["etag"].get(); @@ -138,7 +142,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value())); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); if (options.Resource.HasValue() && !options.Resource.Value().ToString().empty()) { request.GetUrl().AppendQueryParameter( @@ -238,6 +242,42 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-source-if-unmodified-since", options.SourceIfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.Owner.HasValue() && !options.Owner.Value().empty()) + { + request.SetHeader("x-ms-owner", options.Owner.Value()); + } + if (options.Group.HasValue() && !options.Group.Value().empty()) + { + request.SetHeader("x-ms-group", options.Group.Value()); + } + if (options.Acl.HasValue() && !options.Acl.Value().empty()) + { + request.SetHeader("x-ms-acl", options.Acl.Value()); + } + if (options.ProposedLeaseId.HasValue() && !options.ProposedLeaseId.Value().empty()) + { + request.SetHeader("x-ms-proposed-lease-id", options.ProposedLeaseId.Value()); + } + if (options.LeaseDuration.HasValue()) + { + request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value())); + } + if (options.ExpiresOn.HasValue() && !options.ExpiresOn.Value().empty()) + { + request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Created) @@ -252,6 +292,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { response.FileSize = std::stoll(pRawResponse->GetHeaders().at("Content-Length")); } + if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0) + { + response.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) + { + response.EncryptionKeySha256 = Core::Convert::Base64Decode( + pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256")); + } return Response(std::move(response), std::move(pRawResponse)); } Response PathClient::Delete( @@ -269,7 +319,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.GetUrl().AppendQueryParameter("timeout", std::to_string(options.Timeout.Value())); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); if (options.Recursive.HasValue()) { request.GetUrl().AppendQueryParameter( @@ -361,7 +411,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -408,7 +458,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.SetHeader("x-ms-acl", options.Acl.Value()); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -449,6 +499,33 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { return Response( std::move(response), std::move(pRawResponse)); } + Response PathClient::Undelete( + Core::Http::_internal::HttpPipeline& pipeline, + const Core::Url& url, + const UndeletePathOptions& options, + const Core::Context& context) + { + auto request = Core::Http::Request(Core::Http::HttpMethod::Put, url); + request.GetUrl().AppendQueryParameter("comp", "undelete"); + if (options.UndeleteSource.HasValue() && !options.UndeleteSource.Value().empty()) + { + request.SetHeader("x-ms-undelete-source", options.UndeleteSource.Value()); + } + request.SetHeader("x-ms-version", "2021-06-08"); + auto pRawResponse = pipeline.Send(request, context); + auto httpStatusCode = pRawResponse->GetStatusCode(); + if (httpStatusCode != Core::Http::HttpStatusCode::Ok) + { + throw StorageException::CreateFromResponse(std::move(pRawResponse)); + } + Models::_detail::UndeletePathResult response; + if (pRawResponse->GetHeaders().count("x-ms-resource-type") != 0) + { + response.ResourceType = pRawResponse->GetHeaders().at("x-ms-resource-type"); + } + return Response( + std::move(response), std::move(pRawResponse)); + } Response PathClient::GetAccessControlList( Core::Http::_internal::HttpPipeline& pipeline, const Core::Url& url, @@ -485,7 +562,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -572,7 +649,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "If-Unmodified-Since", options.IfUnmodifiedSince.Value().ToString(Azure::DateTime::DateFormat::Rfc1123)); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + request.SetHeader("x-ms-encryption-algorithm", "AES256"); auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -584,6 +673,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { response.LastModified = DateTime::Parse( pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); response.FileSize = std::stoll(pRawResponse->GetHeaders().at("Content-Length")); + if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0) + { + response.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) + { + response.EncryptionKeySha256 = Core::Convert::Base64Decode( + pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256")); + } return Response(std::move(response), std::move(pRawResponse)); } Response FileClient::Append( @@ -617,7 +716,23 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.SetHeader("x-ms-lease-id", options.LeaseId.Value()); } - request.SetHeader("x-ms-version", "2020-02-10"); + request.SetHeader("x-ms-version", "2021-06-08"); + if (options.EncryptionKey.HasValue() && !options.EncryptionKey.Value().empty()) + { + request.SetHeader("x-ms-encryption-key", options.EncryptionKey.Value()); + } + if (options.EncryptionKeySha256.HasValue() + && !Core::Convert::Base64Encode(options.EncryptionKeySha256.Value()).empty()) + { + request.SetHeader( + "x-ms-encryption-key-sha256", + Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); + } + request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.Flush.HasValue()) + { + request.GetUrl().AppendQueryParameter("flush", options.Flush.Value() ? "true" : "false"); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Accepted) @@ -639,8 +754,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64")); response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; } - response.IsServerEncrypted - = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); + if (pRawResponse->GetHeaders().count("x-ms-request-server-encrypted") != 0) + { + response.IsServerEncrypted + = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); + } + if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) + { + response.EncryptionKeySha256 = Core::Convert::Base64Decode( + pRawResponse->GetHeaders().at("x-ms-encryption-key-sha256")); + } return Response(std::move(response), std::move(pRawResponse)); } } // namespace _detail diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 0e9b87c617..5344d267a8 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -9,7 +9,7 @@ package-name: azure-storage-files-datalake namespace: Azure::Storage::Files::DataLake output-folder: generated clear-output-folder: true -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/storage-main/specification/storage/data-plane/Microsoft.StorageDataLake/stable/2020-06-12/DataLakeStorage.json +input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2021-06-08/DataLakeStorage.json ``` ## ModelFour Options @@ -66,12 +66,11 @@ directive: delete $["/{filesystem}"].patch; delete $["/{filesystem}"].head; delete $["/{filesystem}"].delete; - delete $["/{filesystem}?restype=container&comp=list&hierarchy"]; delete $["/{filesystem}/{path}"].post; delete $["/{filesystem}/{path}"].get; delete $["/{filesystem}/{path}"].patch; delete $["/{filesystem}/{path}?comp=expiry"]; - delete $["/{filesystem}/{path}?comp=undelete"]; + delete $["/{filesystem}?restype=container&comp=list&hierarchy"]; ``` ### API Version @@ -89,13 +88,13 @@ directive: "name": "ApiVersion", "modelAsString": false }, - "enum": ["2020-02-10"], + "enum": ["2021-06-08"], "description": "The version used for the operations to Azure storage services." }; - from: swagger-document where: $.parameters transform: > - $.ApiVersionParameter.enum[0] = "2020-02-10"; + $.ApiVersionParameter.enum[0] = "2021-06-08"; ``` ### Rename Operations @@ -118,14 +117,70 @@ directive: } ``` +### Return Type namespace + +```yaml +directive: + - from: swagger-document + where: $ + transform: > + const operations = [ + "Path_Undelete", + ]; + for (const url in $["x-ms-paths"]) { + for (const verb in $["x-ms-paths"][url]) { + if (!operations.includes($["x-ms-paths"][url][verb].operationId)) continue; + const operation = $["x-ms-paths"][url][verb]; + + const status_codes = Object.keys(operation.responses).filter(s => s !== "default"); + status_codes.forEach((status_code, i) => { + if (!operation.responses[status_code].schema) { + const operationId = operation.operationId; + const clientName = operationId.substr(0, operationId.indexOf("_")); + const operationName = operationId.substr(operationId.indexOf("_") + 1); + let operationWords = operationName.split(/(?=[A-Z])/); + operationWords.splice(1, 0, clientName); + const defaultReturnTypeName = operationWords.join("") + "Result"; + operation.responses[status_code].schema = { + "type": "object", + "x-ms-sealed": false, + "x-ms-client-name": defaultReturnTypeName, + "x-namespace": "_detail", + "properties": { + "__placeHolder": {"type": "integer"} + } + }; + } else if (operation.responses[status_code].schema["$ref"]) { + let obj = $; + for (const p of operation.responses[status_code].schema["$ref"].split("/").slice(1)) { + obj = obj[p]; + } + obj["x-namespace"] = "_detail"; + } else { + operation.responses[status_code].schema["x-namespace"] = "_detail"; + } + }); + } + } +``` + ### Global Changes for Definitions, Types etc. ```yaml directive: + - from: swagger-document + where: $["x-ms-paths"].*.*.responses.*.headers + transform: > + for (const h in $) { + if (h === "x-ms-encryption-key-sha256") { + $[h]["format"] = "byte"; + } + } - from: swagger-document where: $.parameters transform: > $.Continuation["x-ms-client-name"] = "ContinuationToken"; + $.EncryptionKeySha256["format"] = "byte"; - from: swagger-document where: $.definitions transform: > @@ -175,6 +230,9 @@ directive: $.Path.properties["lastModified"]["format"] = "date-time-rfc1123"; $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize"; $.Path.properties["isDirectory"]["x-ms-client-default"] = false; + $.Path.properties["EncryptionScope"]["x-nullable"] = true; + delete $.Path.properties["creationTime"]; + delete $.Path.properties["expiryTime"]; $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"}; delete $.Path.properties["eTag"]; $.PathList["x-namespace"] = "_detail"; @@ -192,11 +250,17 @@ directive: ```yaml directive: + - from: swagger-document + where: $["x-ms-paths"]["/{filesystem}/{path}"].put.parameters + transform: > + $ = $.filter(p => !(p["$ref"] && (p["$ref"].endsWith("#/parameters/PathExpiryOptionsOptional")))); - from: swagger-document where: $["x-ms-paths"]["/{filesystem}/{path}"].put.responses transform: > $["201"].headers["Content-Length"]["x-ms-client-name"] = "FileSize"; $["201"].headers["Content-Length"]["x-nullable"] = true; + $["201"].headers["x-ms-request-server-encrypted"]["x-nullable"] = true; + $["201"].headers["x-ms-encryption-key-sha256"]["x-nullable"] = true; delete $["201"].headers["x-ms-continuation"]; $["201"].schema = { "type": "object", @@ -208,6 +272,18 @@ directive: }; ``` + +### SetExpiry + +```yaml +directive: + - from: swagger-document + where: $.parameters + transform: > + delete $["PathExpiryOptions"]; + delete $["PathExpiryOptionsOptional"]; +``` + ### DeletePath ```yaml @@ -253,6 +329,16 @@ directive: } ``` +### UndeletePath + +```yaml +directive: + - from: swagger-document + where: $["x-ms-paths"]["/{filesystem}/{path}?comp=undelete"].put.responses + transform: > + $["200"].headers["x-ms-resource-type"]["x-nullable"] = true; +``` + ### GetPathAccessControlList ```yaml @@ -349,6 +435,8 @@ directive: $["Content-MD5"]["x-nullable"] = true; $["x-ms-content-crc64"]["x-ms-client-name"] = "TransactionalContentHash"; $["x-ms-content-crc64"]["x-nullable"] = true; + $["x-ms-request-server-encrypted"]["x-nullable"] = true; + $["x-ms-encryption-key-sha256"]["x-nullable"] = true; delete $["ETag"]; ``` @@ -360,4 +448,6 @@ directive: where: $["x-ms-paths"]["/{filesystem}/{path}?action=flush"].patch.responses["200"].headers transform: > $["Content-Length"]["x-ms-client-name"] = "FileSize"; + $["x-ms-request-server-encrypted"]["x-nullable"] = true; + $["x-ms-encryption-key-sha256"]["x-nullable"] = true; ``` \ No newline at end of file From ff1cd0f5e5b354db8be8be4f399d7ff3f50dccd8 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Mon, 19 Sep 2022 14:38:11 +0800 Subject: [PATCH 02/11] list system containers (#3929) --- .../test/ut/datalake_service_client_test.cpp | 21 + ...viceClientTest.ListFileSystemsSegment.json | 368 +++++++++--------- ...rviceClientTest.ListSystemFileSystems.json | 25 ++ 3 files changed, 230 insertions(+), 184 deletions(-) create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListSystemFileSystems.json diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp index 2d8c7b4846..ca7458b166 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp @@ -126,6 +126,27 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeServiceClientTest, DISABLED_ListSystemFileSystems) + { + // Disabled temporarily because the test account on the pipeline hasn't system fileSystems. + // List system type FileSystems + Files::DataLake::ListFileSystemsOptions options; + options.Include = Files::DataLake::Models::ListFileSystemsIncludeFlags::System; + std::vector fileSystems; + for (auto pageResult = m_dataLakeServiceClient->ListFileSystems(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + for (const auto& c : pageResult.FileSystems) + { + if (c.Name[0] == '$') + { + fileSystems.push_back(c.Name); + } + } + } + EXPECT_FALSE(fileSystems.empty()); + } + TEST_F(DataLakeServiceClientTest, AnonymousConstructorsWorks_LIVEONLY_) { CHECK_SKIP_TEST(); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListFileSystemsSegment.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListFileSystemsSegment.json index 9596a5d006..7deaa0747a 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListFileSystemsSegment.json +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListFileSystemsSegment.json @@ -2,9 +2,9 @@ "networkCallRecords": [ { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "bf53e4c5-25ec-4e68-726a-60228c7cd73f", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "310854f2-ce79-46ae-51d6-087239c131e8", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -12,21 +12,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C558D96\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:39 GMT", + "etag": "\"0x8DA99E307B315D2\"", + "last-modified": "Mon, 19 Sep 2022 02:02:39 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "bf53e4c5-25ec-4e68-726a-60228c7cd73f", - "x-ms-request-id": "4b990357-b01e-0030-50d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "310854f2-ce79-46ae-51d6-087239c131e8", + "x-ms-request-id": "78851512-a01e-0036-27cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea0?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "917bdd8e-fdf2-41a2-553b-7080cdbc8247", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1363400b-85b6-41e3-4372-6e33c9ae451a", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -34,21 +34,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C5B8030\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:39 GMT", + "etag": "\"0x8DA99E308112D7D\"", + "last-modified": "Mon, 19 Sep 2022 02:02:39 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "917bdd8e-fdf2-41a2-553b-7080cdbc8247", - "x-ms-request-id": "4b990370-b01e-0030-66d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "1363400b-85b6-41e3-4372-6e33c9ae451a", + "x-ms-request-id": "788516e3-a01e-0036-59cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb0?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "863b9955-4183-4b71-6f4a-c2b39c70012b", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "ff9c7bc1-aa9c-4a23-6de8-6a06aac863d3", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -56,21 +56,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C5FC557\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:40 GMT", + "etag": "\"0x8DA99E308516153\"", + "last-modified": "Mon, 19 Sep 2022 02:02:40 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "863b9955-4183-4b71-6f4a-c2b39c70012b", - "x-ms-request-id": "4b990384-b01e-0030-77d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "ff9c7bc1-aa9c-4a23-6de8-6a06aac863d3", + "x-ms-request-id": "78851868-a01e-0036-1acb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea1?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "66a23a38-8a9a-4fda-5360-ab6018123eaa", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5eaba000-6a08-48f9-4c8d-d42d97306edb", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -78,21 +78,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C64318A\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:40 GMT", + "etag": "\"0x8DA99E308825522\"", + "last-modified": "Mon, 19 Sep 2022 02:02:40 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "66a23a38-8a9a-4fda-5360-ab6018123eaa", - "x-ms-request-id": "4b990398-b01e-0030-09d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "5eaba000-6a08-48f9-4c8d-d42d97306edb", + "x-ms-request-id": "788519e0-a01e-0036-78cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb1?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "f76a8403-9342-4cb2-5005-ea4a4eecce57", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "c268d310-da0a-44b3-6174-050703c11fb5", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -100,21 +100,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C684F9A\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:40 GMT", + "etag": "\"0x8DA99E308B1E99E\"", + "last-modified": "Mon, 19 Sep 2022 02:02:40 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "f76a8403-9342-4cb2-5005-ea4a4eecce57", - "x-ms-request-id": "4b9903a7-b01e-0030-16d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "c268d310-da0a-44b3-6174-050703c11fb5", + "x-ms-request-id": "78851af5-a01e-0036-7dcb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea2?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "0ca86a5b-3323-4b00-4676-e3341a6b0580", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a362d29c-aa6c-49d4-66ea-5523a9e23e66", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -122,21 +122,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C6DCD0A\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:41 GMT", + "etag": "\"0x8DA99E308E48AE7\"", + "last-modified": "Mon, 19 Sep 2022 02:02:41 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "0ca86a5b-3323-4b00-4676-e3341a6b0580", - "x-ms-request-id": "4b9903bd-b01e-0030-28d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "a362d29c-aa6c-49d4-66ea-5523a9e23e66", + "x-ms-request-id": "78851c17-a01e-0036-0acb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb2?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "2279452a-9291-4159-41ce-a0465314a9aa", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1fce1c5e-3775-406f-7727-ee5f769b3b8b", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -144,21 +144,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C721241\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:41 GMT", + "etag": "\"0x8DA99E309141F6E\"", + "last-modified": "Mon, 19 Sep 2022 02:02:41 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "2279452a-9291-4159-41ce-a0465314a9aa", - "x-ms-request-id": "4b9903ca-b01e-0030-34d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "1fce1c5e-3775-406f-7727-ee5f769b3b8b", + "x-ms-request-id": "78851d0f-a01e-0036-6ecb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea3?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "495f60b3-52e2-4fa7-45a0-7a92622f5e4a", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "020cdf3a-63cd-43eb-5f44-6ef046ca9f08", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -166,21 +166,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C767E5F\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:41 GMT", + "etag": "\"0x8DA99E30943B3D7\"", + "last-modified": "Mon, 19 Sep 2022 02:02:41 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "495f60b3-52e2-4fa7-45a0-7a92622f5e4a", - "x-ms-request-id": "4b9903d4-b01e-0030-3dd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "020cdf3a-63cd-43eb-5f44-6ef046ca9f08", + "x-ms-request-id": "78851e8c-a01e-0036-56cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb3?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "887208f0-9813-4a11-5373-2347240dc6d1", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f768fad7-e31d-431d-719e-dd5c16c84adc", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -188,21 +188,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C7AC385\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:42 GMT", + "etag": "\"0x8DA99E30977B482\"", + "last-modified": "Mon, 19 Sep 2022 02:02:42 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "887208f0-9813-4a11-5373-2347240dc6d1", - "x-ms-request-id": "4b9903e4-b01e-0030-4cd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "f768fad7-e31d-431d-719e-dd5c16c84adc", + "x-ms-request-id": "78851ffe-a01e-0036-30cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea4?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "e21707e2-aa61-4558-72ba-4d39c8fbb592", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a62399b5-59b8-434a-55de-defacbdacad8", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -210,84 +210,84 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", - "etag": "\"0x8D9C4E91C7EBA8B\"", - "last-modified": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:42 GMT", + "etag": "\"0x8DA99E309A6D3CE\"", + "last-modified": "Mon, 19 Sep 2022 02:02:42 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "e21707e2-aa61-4558-72ba-4d39c8fbb592", - "x-ms-request-id": "4b9903fd-b01e-0030-61d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "a62399b5-59b8-434a-55de-defacbdacad8", + "x-ms-request-id": "78852132-a01e-0036-4dcb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb4?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "aeba04e9-d97d-4dc3-6ebf-761ed23dcd0a", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "45b66566-3378-4bd3-70b7-3f7630dd260b", + "x-ms-version": "2021-04-10" }, "Method": "GET", "Response": { - "BODY": "datalakeserviceclienttestlistfilesystemssegmea0Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C558D96\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C5FC557\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea2Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C684F9A\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea3Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C721241\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea4Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C7AC385\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb0Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C5B8030\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb1Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C64318A\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb2Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C6DCD0A\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb3Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C767E5F\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb4Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C7EBA8B\"unlockedavailable$account-encryption-keyfalsefalsefalse", + "BODY": "datalakeserviceclienttestlistfilesystemssegmea0Mon, 19 Sep 2022 02:02:39 GMT\"0x8DA99E307B315D2\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308516153\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea2Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308B1E99E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea3Mon, 19 Sep 2022 02:02:41 GMT\"0x8DA99E309141F6E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea4Mon, 19 Sep 2022 02:02:42 GMT\"0x8DA99E30977B482\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb0Mon, 19 Sep 2022 02:02:39 GMT\"0x8DA99E308112D7D\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb1Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308825522\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb2Mon, 19 Sep 2022 02:02:41 GMT\"0x8DA99E308E48AE7\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb3Mon, 19 Sep 2022 02:02:41 GMT\"0x8DA99E30943B3D7\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmeb4Mon, 19 Sep 2022 02:02:42 GMT\"0x8DA99E309A6D3CE\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:42 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "aeba04e9-d97d-4dc3-6ebf-761ed23dcd0a", - "x-ms-request-id": "4b99040c-b01e-0030-6ed1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "45b66566-3378-4bd3-70b7-3f7630dd260b", + "x-ms-request-id": "78852219-a01e-0036-20cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net?comp=list" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "9d9e768f-9ee3-45b3-54fe-2e98860d3614", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f1918a86-b048-481e-4370-69b8f1ad8b3b", + "x-ms-version": "2021-04-10" }, "Method": "GET", "Response": { - "BODY": "datalakeserviceclienttestlistfilesystemssegmeadatalakeserviceclienttestlistfilesystemssegmea0Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C558D96\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C5FC557\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea2Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C684F9A\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea3Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C721241\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea4Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C7AC385\"unlockedavailable$account-encryption-keyfalsefalsefalse", + "BODY": "datalakeserviceclienttestlistfilesystemssegmeadatalakeserviceclienttestlistfilesystemssegmea0Mon, 19 Sep 2022 02:02:39 GMT\"0x8DA99E307B315D2\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308516153\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea2Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308B1E99E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea3Mon, 19 Sep 2022 02:02:41 GMT\"0x8DA99E309141F6E\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea4Mon, 19 Sep 2022 02:02:42 GMT\"0x8DA99E30977B482\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "9d9e768f-9ee3-45b3-54fe-2e98860d3614", - "x-ms-request-id": "4b990415-b01e-0030-77d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "f1918a86-b048-481e-4370-69b8f1ad8b3b", + "x-ms-request-id": "78852420-a01e-0036-09cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net?comp=list&prefix=datalakeserviceclienttestlistfilesystemssegmea" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "dd5bf53c-180e-4700-767d-bca8f65603da", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "2b915247-b739-4b57-7a13-f1e8edd3f76c", + "x-ms-version": "2021-04-10" }, "Method": "GET", "Response": { - "BODY": "2datalakeserviceclienttestlistfilesystemssegmea0Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C558D96\"unlockedavailable$account-encryption-keyfalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Wed, 22 Dec 2021 01:19:33 GMT\"0x8D9C4E91C5FC557\"unlockedavailable$account-encryption-keyfalsefalsefalse/datalakelivenow/datalakeserviceclienttestlistfilesystemssegmea2", + "BODY": "2datalakeserviceclienttestlistfilesystemssegmea0Mon, 19 Sep 2022 02:02:39 GMT\"0x8DA99E307B315D2\"unlockedavailable$account-encryption-keyfalsefalsefalsefalsedatalakeserviceclienttestlistfilesystemssegmea1Mon, 19 Sep 2022 02:02:40 GMT\"0x8DA99E308516153\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse/zchangdatalake/datalakeserviceclienttestlistfilesystemssegmea2", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/xml", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:43 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "dd5bf53c-180e-4700-767d-bca8f65603da", - "x-ms-request-id": "4b990420-b01e-0030-02d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "2b915247-b739-4b57-7a13-f1e8edd3f76c", + "x-ms-request-id": "788525c2-a01e-0036-12cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net?comp=list&maxresults=2" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "a5048e62-0520-4fe7-6dec-ab0a2d55b0f4", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "57fb18e6-0df1-4c98-48f7-e127ebf3961c", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -295,19 +295,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:44 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "a5048e62-0520-4fe7-6dec-ab0a2d55b0f4", - "x-ms-request-id": "4b990425-b01e-0030-06d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "57fb18e6-0df1-4c98-48f7-e127ebf3961c", + "x-ms-request-id": "788526af-a01e-0036-6dcb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea0?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "147b4502-e3a5-4bed-40cd-9d9299272688", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "0bc8f713-b433-4829-7c00-7ed23ee9575f", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -315,19 +315,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:44 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "147b4502-e3a5-4bed-40cd-9d9299272688", - "x-ms-request-id": "4b99042c-b01e-0030-0dd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "0bc8f713-b433-4829-7c00-7ed23ee9575f", + "x-ms-request-id": "78852798-a01e-0036-42cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea1?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "b1f24459-581c-444d-7768-f18f48058170", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "178eb256-dad1-4e63-4faf-1a44b7f2543a", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -335,19 +335,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:44 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "b1f24459-581c-444d-7768-f18f48058170", - "x-ms-request-id": "4b990437-b01e-0030-18d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "178eb256-dad1-4e63-4faf-1a44b7f2543a", + "x-ms-request-id": "788528b0-a01e-0036-3bcb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea2?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "89f523bc-0d08-4a1f-5333-edd04f4bec17", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "896bc83a-5b7c-41c5-52cf-8bd9853b83a6", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -355,19 +355,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:45 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "89f523bc-0d08-4a1f-5333-edd04f4bec17", - "x-ms-request-id": "4b99043d-b01e-0030-1dd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "896bc83a-5b7c-41c5-52cf-8bd9853b83a6", + "x-ms-request-id": "7885299b-a01e-0036-05cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea3?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "88db7494-cac3-4d30-7382-a888662c6ad3", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1a769cfa-7205-4cf0-4c95-e6371db3c543", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -375,19 +375,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:45 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "88db7494-cac3-4d30-7382-a888662c6ad3", - "x-ms-request-id": "4b990442-b01e-0030-22d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "1a769cfa-7205-4cf0-4c95-e6371db3c543", + "x-ms-request-id": "78852a8c-a01e-0036-68cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmea4?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "864c74f8-88b1-4542-4249-e9d32993fea5", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "97fe93d1-0a6b-4913-5355-5a8ec21283d6", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -395,19 +395,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:45 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "864c74f8-88b1-4542-4249-e9d32993fea5", - "x-ms-request-id": "4b99044b-b01e-0030-2bd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "97fe93d1-0a6b-4913-5355-5a8ec21283d6", + "x-ms-request-id": "78852c3f-a01e-0036-75cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb0?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "c5590fb7-82b7-4088-7c40-b671d288a647", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "bf519a38-36b8-49e4-5ec9-1dd89e7ad736", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -415,19 +415,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:45 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "c5590fb7-82b7-4088-7c40-b671d288a647", - "x-ms-request-id": "4b990451-b01e-0030-31d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "bf519a38-36b8-49e4-5ec9-1dd89e7ad736", + "x-ms-request-id": "78852e02-a01e-0036-0fcb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb1?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "fd37808c-25f7-4847-406e-5f76c41d3afa", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1020fc6a-79c8-48fa-6b06-641436621490", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -435,19 +435,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:46 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "fd37808c-25f7-4847-406e-5f76c41d3afa", - "x-ms-request-id": "4b990459-b01e-0030-39d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "1020fc6a-79c8-48fa-6b06-641436621490", + "x-ms-request-id": "78852f14-a01e-0036-0ccb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb2?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "0d459a84-32ba-41d8-4361-3937776e6e01", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a9052ccc-e02a-4aef-710a-0b21a9f473e6", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -455,19 +455,19 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:46 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "0d459a84-32ba-41d8-4361-3937776e6e01", - "x-ms-request-id": "4b99045c-b01e-0030-3cd1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "a9052ccc-e02a-4aef-710a-0b21a9f473e6", + "x-ms-request-id": "78852fd8-a01e-0036-43cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb3?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Linux 5.4.0-1063-azure x86_64 #66~18.04.1-Ubuntu SMP Thu Oct 21 09:59:28 UTC 2021)", - "x-ms-client-request-id": "3395363d-006d-4cc9-6bfc-576bd439bdd4", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "cb9175c7-7e21-4e00-6fc0-6a203318f2d5", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -475,11 +475,11 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Wed, 22 Dec 2021 01:19:33 GMT", + "date": "Mon, 19 Sep 2022 02:02:46 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "3395363d-006d-4cc9-6bfc-576bd439bdd4", - "x-ms-request-id": "4b990462-b01e-0030-42d1-f67b20000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "cb9175c7-7e21-4e00-6fc0-6a203318f2d5", + "x-ms-request-id": "788530b0-a01e-0036-80cb-cb5fdf000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakeserviceclienttestlistfilesystemssegmeb4?restype=container" } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListSystemFileSystems.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListSystemFileSystems.json new file mode 100644 index 0000000000..55aeab96c2 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.ListSystemFileSystems.json @@ -0,0 +1,25 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f88eba0a-e316-459d-7a43-322398709e83", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "$logsMon, 15 Aug 2022 08:45:51 GMT\"0x8DA7E9A8EB2CBB3\"unlockedavailable$account-encryption-keyfalsefalsefalsefalse", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Mon, 19 Sep 2022 02:02:47 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "f88eba0a-e316-459d-7a43-322398709e83", + "x-ms-request-id": "788531a2-a01e-0036-5ecb-cb5fdf000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=list&include=system" + } + ] +} From 44270c09ba7afbbf24f9f195300ba5aa90ae4a80 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Wed, 21 Sep 2022 11:00:04 +0800 Subject: [PATCH 03/11] encryption scope sas (#3936) * encryption scope sas * format * remove EncryptionScope properties * update scope --- .../files/datalake/datalake_sas_builder.hpp | 5 +++ .../src/datalake_sas_builder.cpp | 19 ++++++++--- .../test/ut/datalake_sas_test.cpp | 33 +++++++++++++++++++ sdk/storage/test-resources-post.ps1 | 2 ++ 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp index 85c85100ac..dba43e533e 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_sas_builder.hpp @@ -293,6 +293,11 @@ namespace Azure { namespace Storage { namespace Sas { */ std::string CorrelationId; + /** + * @brief Optional encryption scope to use when sending requests authorized with this SAS url. + */ + std::string EncryptionScope; + /** * @brief Sets the permissions for the filesystem SAS. * diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp index 53aadf9246..aecadd89da 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_sas_builder.cpp @@ -10,7 +10,7 @@ namespace Azure { namespace Storage { namespace Sas { namespace { - constexpr static const char* SasVersion = "2020-02-10"; + constexpr static const char* SasVersion = "2021-06-08"; std::string DataLakeSasResourceToString(DataLakeSasResource resource) { @@ -138,8 +138,9 @@ namespace Azure { namespace Storage { namespace Sas { std::string stringToSign = Permissions + "\n" + startsOnStr + "\n" + expiresOnStr + "\n" + canonicalName + "\n" + Identifier + "\n" + (IPRange.HasValue() ? IPRange.Value() : "") - + "\n" + protocol + "\n" + SasVersion + "\n" + resource + "\n" + "\n" + CacheControl + "\n" - + ContentDisposition + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType; + + "\n" + protocol + "\n" + SasVersion + "\n" + resource + "\n" + "\n" + EncryptionScope + + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding + "\n" + + ContentLanguage + "\n" + ContentType; std::string signature = Azure::Core::Convert::Base64Encode(_internal::HmacSha256( std::vector(stringToSign.begin(), stringToSign.end()), @@ -190,6 +191,10 @@ namespace Azure { namespace Storage { namespace Sas { { builder.AppendQueryParameter("rsct", _internal::UrlEncodeQueryParameter(ContentType)); } + if (!EncryptionScope.empty()) + { + builder.AppendQueryParameter("ses", _internal::UrlEncodeQueryParameter(EncryptionScope)); + } return builder.GetAbsoluteUrl(); } @@ -223,8 +228,8 @@ namespace Azure { namespace Storage { namespace Sas { + "\n" + userDelegationKey.SignedService + "\n" + userDelegationKey.SignedVersion + "\n" + PreauthorizedAgentObjectId + "\n" + AgentObjectId + "\n" + CorrelationId + "\n" + (IPRange.HasValue() ? IPRange.Value() : "") + "\n" + protocol + "\n" + SasVersion + "\n" - + resource + "\n" + "\n" + CacheControl + "\n" + ContentDisposition + "\n" + ContentEncoding - + "\n" + ContentLanguage + "\n" + ContentType; + + resource + "\n" + "\n" + EncryptionScope + "\n" + CacheControl + "\n" + ContentDisposition + + "\n" + ContentEncoding + "\n" + ContentLanguage + "\n" + ContentType; std::string signature = Azure::Core::Convert::Base64Encode(_internal::HmacSha256( std::vector(stringToSign.begin(), stringToSign.end()), @@ -292,6 +297,10 @@ namespace Azure { namespace Storage { namespace Sas { { builder.AppendQueryParameter("rsct", _internal::UrlEncodeQueryParameter(ContentType)); } + if (!EncryptionScope.empty()) + { + builder.AppendQueryParameter("ses", _internal::UrlEncodeQueryParameter(EncryptionScope)); + } builder.AppendQueryParameter("sig", _internal::UrlEncodeQueryParameter(signature)); return builder.GetAbsoluteUrl(); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp index 4615159f45..75c25f8404 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_sas_test.cpp @@ -441,6 +441,39 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(p.Value.HttpHeaders.CacheControl, headers.CacheControl); EXPECT_EQ(p.Value.HttpHeaders.ContentEncoding, headers.ContentEncoding); } + + // Encryption scope + const auto encryptionScope = GetTestEncryptionScope(); + { + auto sasBuilderWithEncryptionScope = fileSasBuilder; + sasBuilderWithEncryptionScope.EncryptionScope = encryptionScope; + sasBuilderWithEncryptionScope.SetPermissions(Sas::DataLakeSasPermissions::All); + auto fileClientEncryptionScopeSas = Files::DataLake::DataLakeFileClient( + fileUrl + sasBuilderWithEncryptionScope.GenerateSasToken(*keyCredential)); + fileClientEncryptionScopeSas.Create(); + auto pRawResponse = fileClientEncryptionScopeSas.GetProperties().RawResponse; + ASSERT_TRUE(pRawResponse->GetHeaders().count("x-ms-encryption-scope") != 0); + EXPECT_EQ(pRawResponse->GetHeaders().at("x-ms-encryption-scope"), encryptionScope); + + fileClientEncryptionScopeSas = Files::DataLake::DataLakeFileClient( + fileUrl + sasBuilderWithEncryptionScope.GenerateSasToken(userDelegationKey, accountName)); + fileClientEncryptionScopeSas.Create(); + pRawResponse = fileClientEncryptionScopeSas.GetProperties().RawResponse; + ASSERT_TRUE(pRawResponse->GetHeaders().count("x-ms-encryption-scope") != 0); + EXPECT_EQ(pRawResponse->GetHeaders().at("x-ms-encryption-scope"), encryptionScope); + } + { + auto sasBuilderWithEncryptionScope = directorySasBuilder; + sasBuilderWithEncryptionScope.EncryptionScope = encryptionScope; + sasBuilderWithEncryptionScope.SetPermissions(Sas::DataLakeSasPermissions::All); + auto directoryClientEncryptionScopeSas = Files::DataLake::DataLakeDirectoryClient( + directory1Url + + sasBuilderWithEncryptionScope.GenerateSasToken(userDelegationKey, accountName)); + directoryClientEncryptionScopeSas.Create(); + auto pRawResponse = directoryClientEncryptionScopeSas.GetProperties().RawResponse; + ASSERT_TRUE(pRawResponse->GetHeaders().count("x-ms-encryption-scope") != 0); + EXPECT_EQ(pRawResponse->GetHeaders().at("x-ms-encryption-scope"), encryptionScope); + } } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/test-resources-post.ps1 b/sdk/storage/test-resources-post.ps1 index 539c242e10..a02df05f80 100644 --- a/sdk/storage/test-resources-post.ps1 +++ b/sdk/storage/test-resources-post.ps1 @@ -8,6 +8,8 @@ param( New-AzStorageEncryptionScope -ResourceGroupName $ResourceGroupName -StorageAccountName $DeploymentOutputs['ACCOUNT_NAME'] -EncryptionScopeName "EncryptionScopeForTest" -StorageEncryption +New-AzStorageEncryptionScope -ResourceGroupName $ResourceGroupName -StorageAccountName $DeploymentOutputs['DATALAKE_ACCOUNT_NAME'] -EncryptionScopeName "EncryptionScopeForTest" -StorageEncryption + # This script is used to wait until XCache is refreshed for the service properties (30s), and role assignment takes effect (300s). Start-Sleep -s 300 From ca567ee00a368cf4982fe83e54974361c0256d5f Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Wed, 21 Sep 2022 16:02:46 +0800 Subject: [PATCH 04/11] Datalakestg82 quick query parquet format (#3942) --- .../azure-storage-blobs/src/rest_client.cpp | 12 +- .../azure-storage-blobs/swagger/README.md | 4 + .../files/datalake/datalake_file_client.hpp | 13 + .../files/datalake/datalake_options.hpp | 33 ++ .../files/datalake/datalake_responses.hpp | 38 ++ .../src/datalake_file_client.cpp | 29 ++ .../test/ut/CMakeLists.txt | 1 + .../test/ut/datalake_file_client_test.cpp | 2 + .../test/ut/datalake_file_query_test.cpp | 430 ++++++++++++++++++ 9 files changed, 559 insertions(+), 3 deletions(-) create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 1678ac9b72..7742fd6087 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -5573,9 +5573,15 @@ namespace Azure { namespace Storage { namespace Blobs { response.LeaseDuration = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration")); } - response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); - response.LeaseStatus - = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0) + { + response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state")); + } + if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0) + { + response.LeaseStatus + = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status")); + } response.IsServerEncrypted = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true"); return Response(std::move(response), std::move(pRawResponse)); diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 2b34011ca2..493c96ad29 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -1292,6 +1292,10 @@ directive: delete $[status_code].headers["x-ms-blob-content-md5"]; delete $[status_code].headers["x-ms-content-crc64"]; $[status_code].headers["x-ms-lease-duration"]["x-nullable"] = true; + $[status_code].headers["x-ms-lease-state"]["x-ms-client-default"] = ""; + $[status_code].headers["x-ms-lease-state"]["x-nullable"] = true; + $[status_code].headers["x-ms-lease-status"]["x-ms-client-default"] = ""; + $[status_code].headers["x-ms-lease-status"]["x-nullable"] = true; } ``` diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index 9a4e7c851c..a4ade4cc68 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -265,6 +265,19 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const ScheduleFileDeletionOptions& options = ScheduleFileDeletionOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Returns the result of a query against the file. + * + * @param querySqlExpression The query expression in SQL. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A QueryFileResult describing the query result. + */ + Azure::Response Query( + const std::string& querySqlExpression, + const QueryFileOptions& options = QueryFileOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: explicit DataLakeFileClient( Azure::Core::Url fileUrl, diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index b0796cefde..4d73a3fe15 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -21,6 +21,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using PathHttpHeaders = Blobs::Models::BlobHttpHeaders; using ListFileSystemsIncludeFlags = Blobs::Models::ListBlobContainersIncludeFlags; using SignedIdentifier = Blobs::Models::SignedIdentifier; + using FileQueryArrowField = Blobs::Models::BlobQueryArrowField; + using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType; } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; @@ -638,4 +640,35 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using ReleaseLeaseOptions = Blobs::ReleaseLeaseOptions; using ChangeLeaseOptions = Blobs::ChangeLeaseOptions; + using FileQueryInputTextOptions = Blobs::BlobQueryInputTextOptions; + using FileQueryOutputTextOptions = Blobs::BlobQueryOutputTextOptions; + using FileQueryError = Blobs::BlobQueryError; + + /** + * @brief Optional parameters for #Azure::Storage::Files::DataLake::DataLakeFileClient::Query. + */ + struct QueryFileOptions final + { + /** + * @brief Input text configuration. + */ + FileQueryInputTextOptions InputTextConfiguration; + /** + * @brief Output text configuration. + */ + FileQueryOutputTextOptions OutputTextConfiguration; + /** + * @brief Optional conditions that must be met to perform this operation. + */ + PathAccessConditions AccessConditions; + /** + * @brief Callback for progress handling. + */ + std::function ProgressHandler; + /** + * @brief Callback for error handling. If you don't specify one, the default will be used, which + * will ignore all non-fatal errors and throw for fatal errors. + */ + std::function ErrorHandler; + }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index e8b9b90ce8..bc076a4cae 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -449,6 +449,44 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using ScheduleFileDeletionResult = Blobs::Models::SetBlobExpiryResult; using CopyStatus = Blobs::Models::CopyStatus; + /** + * @brief Response type for #Azure::Storage::Files::DataLake::FileClient::Query. + */ + struct QueryFileResult final + { + std::unique_ptr BodyStream; + /** + * Returns the date and time the container was last modified. Any operation that modifies the + * file, including an update of the file's metadata or properties, changes the last-modified + * time of the file. + */ + DateTime LastModified; + /** + * The ETag contains a value that you can use to perform operations conditionally. If the + * request version is 2011-08-18 or newer, the ETag value will be in quotes. + */ + Azure::ETag ETag; + /** + * When a file is leased, specifies whether the lease is of infinite or fixed duration. + */ + Nullable LeaseDuration; + /** + * Lease state of the file. + */ + Models::LeaseState LeaseState; + /** + * The current lease status of the file. + */ + Models::LeaseStatus LeaseStatus; + /** + * The value of this header is set to true if the file data and application metadata are + * completely encrypted using the specified algorithm. Otherwise, the value is set to false + * (when the file is unencrypted, or if only parts of the file/application metadata are + * encrypted). + */ + bool IsServerEncrypted = bool(); + }; + /** * @brief The detailed information returned when downloading a file. */ diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index ded67cb674..407dbaaaa5 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -327,4 +327,33 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { *m_pipeline, m_blobClient.m_blobUrl, protocolLayerOptions, context); } + Azure::Response DataLakeFileClient::Query( + const std::string& querySqlExpression, + const QueryFileOptions& options, + const Azure::Core::Context& context) const + { + Blobs::QueryBlobOptions blobOptions; + blobOptions.InputTextConfiguration = options.InputTextConfiguration; + blobOptions.OutputTextConfiguration = options.OutputTextConfiguration; + blobOptions.ErrorHandler = options.ErrorHandler; + blobOptions.ProgressHandler = options.ProgressHandler; + blobOptions.AccessConditions.IfMatch = options.AccessConditions.IfMatch; + blobOptions.AccessConditions.IfNoneMatch = options.AccessConditions.IfNoneMatch; + blobOptions.AccessConditions.IfModifiedSince = options.AccessConditions.IfModifiedSince; + blobOptions.AccessConditions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + blobOptions.AccessConditions.LeaseId = options.AccessConditions.LeaseId; + auto response + = m_blobClient.AsBlockBlobClient().Query(querySqlExpression, blobOptions, context); + Models::QueryFileResult ret; + ret.BodyStream = std::move(response.Value.BodyStream); + ret.ETag = std::move(response.Value.ETag); + ret.LastModified = std::move(response.Value.LastModified); + ret.LeaseDuration = std::move(response.Value.LeaseDuration); + ret.LeaseState = std::move(response.Value.LeaseState); + ret.LeaseStatus = std::move(response.Value.LeaseStatus); + ret.IsServerEncrypted = response.Value.IsServerEncrypted; + return Azure::Response( + std::move(ret), std::move(response.RawResponse)); + } + }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt index a8d6fe32fe..ea2c7d2fd0 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt +++ b/sdk/storage/azure-storage-files-datalake/test/ut/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable ( datalake_directory_client_test.hpp datalake_file_client_test.cpp datalake_file_client_test.hpp + datalake_file_query_test.cpp datalake_file_system_client_test.cpp datalake_file_system_client_test.hpp datalake_path_client_test.cpp diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index 099135cd54..327b6f1245 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -29,6 +29,7 @@ namespace Azure { namespace Storage { namespace Test { void DataLakeFileClientTest::SetUp() { DataLakeFileSystemClientTest::SetUp(); + CHECK_SKIP_TEST(); m_fileName = GetFileSystemValidName(); m_fileClient = std::make_shared( m_fileSystemClient->GetFileClient(m_fileName)); @@ -37,6 +38,7 @@ namespace Azure { namespace Storage { namespace Test { void DataLakeFileClientTest::TearDown() { + CHECK_SKIP_TEST(); m_fileSystemClient->GetFileClient(m_fileName).Delete(); DataLakeFileSystemClientTest::TearDown(); } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp new file mode 100644 index 0000000000..1aa6a8c3d8 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_query_test.cpp @@ -0,0 +1,430 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: MIT + +#include "datalake_file_client_test.hpp" + +#include +#include +#include + +// cspell:ignore sapote + +namespace Azure { namespace Storage { namespace Test { + + const std::string JsonQueryTestData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": 50} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": 99} +{"id": 104, "name": "clementines", "price": 399} +{"id": 105, "name": "grapes", "price": 150} +{"id": 106, "name": "lemons", "price": 69} +{"id": 107, "name": "pears", "price": 100} +{"id": 108, "name": "cherries", "price": 281} +{"id": 109, "name": "coconut", "price": 178} +{"id": 110, "name": "bananas", "price": 39} +{"id": 111, "name": "peaches", "price": 117} +{"id": 112, "name": "sapote,\"mamey", "price": 50} +)json"; + + const std::string CsvQueryTestData = R"csv( +id,name,price +100,oranges,100 +101,limes,50 +102,berries,199 +103,apples,99 +104,clementines,399 +105,grapes,150 +106,lemons,69 +107,pears,100 +108,cherries,281 +109,coconut,178 +110,bananas,39 +111,peaches,117 +112,sapote\,mamey,50 +)csv"; + + const std::vector ParquetQueryTestData = Core::Convert::Base64Decode( + "UEFSMRUAFewBFewBLBUaFQAVBhUIAAACAAAAGgFkAAAAAAAAAGUAAAAAAAAAZgAAAAAAAABnAAAAAAAAAGgAAAAAAAAA" + "aQAAAAAAAABqAAAAAAAAAGsAAAAAAAAAbAAAAAAAAABtAAAAAAAAAG4AAAAAAAAAbwAAAAAAAABwAAAAAAAAAAAAAAAA" + "AAAAFQAVxAIVxAIsFRoVABUGFQgAAAIAAAAaAQcAAABvcmFuZ2VzBQAAAGxpbWVzBwAAAGJlcnJpZXMGAAAAYXBwbGVz" + "CwAAAGNsZW1lbnRpbmVzBgAAAGdyYXBlcwYAAABsZW1vbnMFAAAAcGVhcnMIAAAAY2hlcnJpZXMHAAAAY29jb251dAcA" + "AABiYW5hbmFzBwAAAHBlYWNoZXMOAAAAc2Fwb3RlLCJtYW1leSIAAAAAAAAAABUAFewBFewBLBUaFQAVBhUIAAACAAAA" + "GgFkAAAAAAAAADIAAAAAAAAAxwAAAAAAAABjAAAAAAAAAI8BAAAAAAAAlgAAAAAAAABFAAAAAAAAAGQAAAAAAAAAGQEA" + "AAAAAACyAAAAAAAAACcAAAAAAAAAdQAAAAAAAAAyAAAAAAAAAAAAAAAAAAAAFQIZTEgGc2NoZW1hFQYAFQQVgAEVAhgC" + "aWQAFQwlAhgEbmFtZSUAABUEFYABFQIYBXByaWNlABYaGRwZPCaaAhwVBBkVABkYAmlkFQAWGhaSAhaSAhkAFgg8GAhw" + "AAAAAAAAABgIZAAAAAAAAAAWAAAZHBUAFQAVAgAAACaEBRwVDBkVABkYBG5hbWUVABYaFuoCFuoCGQAWmgI8GA5zYXBv" + "dGUsIm1hbWV5IhgGYXBwbGVzFgAAGRwVABUAFQIAAAAmlgccFQQZFQAZGAVwcmljZRUAFhoWkgIWkgIZABaEBTwYCI8B" + "AAAAAAAAGAgnAAAAAAAAABYAABkcFQAVABUCAAAAFo4HFhoAGRwYBnBhbmRhcxiRBXsiY29sdW1uX2luZGV4ZXMiOiBb" + "eyJmaWVsZF9uYW1lIjogbnVsbCwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiBudWxsLCAibnVtcHlfdHlwZSI6ICJv" + "YmplY3QiLCAicGFuZGFzX3R5cGUiOiAibWl4ZWQtaW50ZWdlciJ9XSwgImNvbHVtbnMiOiBbeyJmaWVsZF9uYW1lIjog" + "ImlkIiwgIm1ldGFkYXRhIjogbnVsbCwgIm5hbWUiOiAiaWQiLCAibnVtcHlfdHlwZSI6ICJpbnQ2NCIsICJwYW5kYXNf" + "dHlwZSI6ICJpbnQ2NCJ9LCB7ImZpZWxkX25hbWUiOiAibmFtZSIsICJtZXRhZGF0YSI6IG51bGwsICJuYW1lIjogIm5h" + "bWUiLCAibnVtcHlfdHlwZSI6ICJvYmplY3QiLCAicGFuZGFzX3R5cGUiOiAidW5pY29kZSJ9LCB7ImZpZWxkX25hbWUi" + "OiAicHJpY2UiLCAibWV0YWRhdGEiOiBudWxsLCAibmFtZSI6ICJwcmljZSIsICJudW1weV90eXBlIjogImludDY0Iiwg" + "InBhbmRhc190eXBlIjogImludDY0In1dLCAiY3JlYXRvciI6IHsibGlicmFyeSI6ICJmYXN0cGFycXVldCIsICJ2ZXJz" + "aW9uIjogIjAuOC4xIn0sICJpbmRleF9jb2x1bW5zIjogW3sia2luZCI6ICJyYW5nZSIsICJuYW1lIjogbnVsbCwgInN0" + "YXJ0IjogMCwgInN0ZXAiOiAxLCAic3RvcCI6IDEzfV0sICJwYW5kYXNfdmVyc2lvbiI6ICIxLjQuMiIsICJwYXJ0aXRp" + "b25fY29sdW1ucyI6IFtdfQAYKmZhc3RwYXJxdWV0LXB5dGhvbiB2ZXJzaW9uIDAuOC4xIChidWlsZCAwKQDXAwAAUEFS" + "MQ=="); + + TEST_F(DataLakeFileClientTest, QueryJsonInputCsvOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + client.UploadFrom( + reinterpret_cast(JsonQueryTestData.data()), JsonQueryTestData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateJsonTextOptions(); + + { + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateCsvTextOptions(); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103,apples,99 +106,lemons,69 +110,bananas,39 +112,"sapote,""mamey",50 +)csv"); + } + + { + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateCsvTextOptions( + "|", ".", "[", "\\", true); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"csv(103.apples.99|106.lemons.69|110.bananas.39|112.sapote,"mamey.50|)csv"); + } + } + + TEST_F(DataLakeFileClientTest, QueryCsvInputJsonOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateCsvTextOptions( + "\n", ",", "\"", "\\", true); + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateJsonTextOptions("|"); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + EXPECT_EQ( + std::string(data.begin(), data.end()), + R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); + } + + TEST_F(DataLakeFileClientTest, QueryCsvInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + client.UploadFrom( + reinterpret_cast(CsvQueryTestData.data()), CsvQueryTestData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateCsvTextOptions( + "\n", ",", "\"", "\\", true); + std::vector fields; + Files::DataLake::Models::FileQueryArrowField field; + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::Decimal; + field.Name = "price"; + field.Precision = 10; + field.Scale = 2; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+gAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAACAAAAAQAAAAAQAAAC" + "c////AAABBxAAAAAgAAAABAAAAAAAAAAFAAAAcHJpY2UAAAAIAAwABAAIAAgAAAAKAAAAAgAAANT///" + "8AAAEFEAAAABwAAAAEAAAAAAAAAAQAAABuYW1lAAAAAAQABAAEAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEC" + "EAAAABwAAAAEAAAAAAAAAAIAAABpZAAACAAMAAgABwAIAAAAAAAAAUAAAAAAAAAA//////" + "AAAAAUAAAAAAAAAAwAGgAGAAUACAAMAAwAAAAAAwQAHAAAAAgAAAAAAAAAAAAAAAAACgAMAAAABAAIAAoAAACA" + "AAAABAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAQAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAMA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAP/////" + "4AAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAACYAAAAAAAAAAAACgAYAAwABAAIAAoAAACMAAAAE" + "AAAAAQAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAAAAAAAA" + "AAAAgAAAAAAAAABQAAAAAAAAAOAAAAAAAAAAfAAAAAAAAAFgAAAAAAAAAAAAAAAAAAABYAAAAAAAAAEAAAAAAA" + "AAAAAAAAAMAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAABnAAAAAAA" + "AAGoAAAAAAAAAbgAAAAAAAABwAAAAAAAAAAAAAAAGAAAADAAAABMAAAAfAAAAAAAAAGFwcGxlc2xlbW9uc2Jhb" + "mFuYXNzYXBvdGUsbWFtZXkAYwAAAAAAAAAAAAAAAAAAAEUAAAAAAAAAAAAAAAAAAAAnAAAAAAAAAAAAAAAAAAA" + "AMgAAAAAAAAAAAAAAAAAAAA=="); + EXPECT_EQ(data, expectedData); + } + + TEST_F(DataLakeFileClientTest, QueryParquetInputArrowOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + client.UploadFrom(ParquetQueryTestData.data(), ParquetQueryTestData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateParquetTextOptions(); + std::vector fields; + Files::DataLake::Models::FileQueryArrowField field; + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::Int64; + field.Name = "id"; + fields.push_back(field); + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::String; + field.Name = "name"; + fields.push_back(field); + field.Type = Files::DataLake::Models::FileQueryArrowFieldType::Int64; + field.Name = "price"; + fields.push_back(field); + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateArrowTextOptions(std::move(fields)); + auto queryResponse + = client.Query("SELECT * from BlobStorage WHERE id > 101 AND price < 100;", queryOptions); + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + const auto expectedData = Core::Convert::Base64Decode( + "/////" + "+AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAMAAAB4AAAAOAAAAAQAAAC" + "k////AAABAhAAAAAYAAAABAAAAAAAAAAFAAAAcHJpY2UAAACY////AAAAAUAAAADU////" + "AAABBRAAAAAcAAAABAAAAAAAAAAEAAAAbmFtZQAAAAAEAAQABAAAABAAFAAIAAYABwAMAAAAEAAQAAAAAAABAh" + "AAAAAcAAAABAAAAAAAAAACAAAAaWQAAAgADAAIAAcACAAAAAAAAAFAAAAAAAAAAP/////" + "wAAAAFAAAAAAAAAAMABoABgAFAAgADAAMAAAAAAMEABwAAAAIAAAAAAAAAAAAAAAAAAoADAAAAAQACAAKAAAAg" + "AAAAAQAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAEAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAADA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAZwAAAAAA" + "AAAAAAAABgAAAGFwcGxlcwAAYwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABgAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAagAAAAAA" + "AAAAAAAABgAAAGxlbW9ucwAARQAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAIAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAABwAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAbgAAAAAA" + "AAAAAAAABwAAAGJhbmFuYXMAJwAAAAAAAAD/////" + "+AAAABQAAAAAAAAADAAWAAYABQAIAAwADAAAAAADBAAYAAAAKAAAAAAAAAAAAAoAGAAMAAQACAAKAAAAjAAAAB" + "AAAAABAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAAAAAAAAA" + "AAAACAAAAAAAAAAIAAAAAAAAABAAAAAAAAAADgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAIAAAAAA" + "AAAAAAAAADAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAcAAAAAAA" + "AAAAAAAADgAAAHNhcG90ZSwibWFtZXkiAAAyAAAAAAAAAP////8AAAAA"); + EXPECT_EQ(data, expectedData); + } + + TEST_F(DataLakeFileClientTest, QueryWithError_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + const std::string malformedData = + R"json( +{"id": 100, "name": "oranges", "price": 100} +{"id": 101, "name": "limes", "price": "aa"} +{"id": 102, "name": "berries", "price": 199} +{"id": 103, "name": "apples", "price": "bb"} +{"id": 104, "name": "clementines", "price": 399} +xx +)json"; + client.UploadFrom(reinterpret_cast(malformedData.data()), malformedData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateJsonTextOptions(); + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + + try + { + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + FAIL(); + } + catch (StorageException& e) + { + EXPECT_EQ(e.StatusCode, Core::Http::HttpStatusCode::Ok); + EXPECT_EQ(e.ReasonPhrase, "OK"); + EXPECT_FALSE(e.RequestId.empty()); + EXPECT_FALSE(e.ClientRequestId.empty()); + EXPECT_EQ(e.ErrorCode, "ParseError"); + EXPECT_FALSE(e.Message.empty()); + EXPECT_FALSE(std::string(e.what()).empty()); + } + + bool progressCallbackCalled = false; + queryOptions.ProgressHandler + = [&malformedData, &progressCallbackCalled](int64_t offset, int64_t totalBytes) { + EXPECT_EQ(totalBytes, static_cast(malformedData.size())); + EXPECT_TRUE(offset >= 0 && offset <= totalBytes); + progressCallbackCalled = true; + }; + int numNonFatalErrors = 0; + int numFatalErrors = 0; + queryOptions.ErrorHandler + = [&numNonFatalErrors, &numFatalErrors](Files::DataLake::FileQueryError e) { + if (e.IsFatal) + { + ++numFatalErrors; + } + else + { + ++numNonFatalErrors; + } + }; + queryResponse = client.Query("SELECT * FROM BlobStorage WHERE price > 0;", queryOptions); + queryResponse.Value.BodyStream->ReadToEnd(); + + EXPECT_EQ(numNonFatalErrors, 2); + EXPECT_EQ(numFatalErrors, 1); + EXPECT_TRUE(progressCallbackCalled); + } + + TEST_F(DataLakeFileClientTest, QueryDefaultInputOutput_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + const std::string csvData = "100,oranges,100"; + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + auto queryResponse = client.Query("SELECT * from BlobStorage;"); + + auto data = queryResponse.Value.BodyStream->ReadToEnd(); + } + + TEST_F(DataLakeFileClientTest, QueryLargeBlob_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + + constexpr size_t DataSize = static_cast(32_MB); + + int recordCounter = 0; + std::string csvData; + std::string jsonData; + while (csvData.size() < DataSize) + { + std::string counter = std::to_string(recordCounter++); + std::string record = RandomString(static_cast(RandomInt(1, 3000))); + csvData += counter + "," + record + "\n"; + jsonData += "{\"_1\":\"" + counter + "\",\"_2\":\"" + record + "\"}\n"; + } + + client.UploadFrom(reinterpret_cast(csvData.data()), csvData.size()); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.InputTextConfiguration + = Files::DataLake::FileQueryInputTextOptions::CreateCsvTextOptions(); + queryOptions.OutputTextConfiguration + = Files::DataLake::FileQueryOutputTextOptions::CreateJsonTextOptions(); + auto queryResponse = client.Query("SELECT * FROM BlobStorage;", queryOptions); + + size_t comparePos = 0; + std::vector readBuffer(4096); + while (true) + { + auto s = queryResponse.Value.BodyStream->Read(readBuffer.data(), readBuffer.size()); + if (s == 0) + { + break; + } + ASSERT_TRUE(comparePos + s <= jsonData.size()); + ASSERT_EQ( + std::string(readBuffer.begin(), readBuffer.begin() + s), jsonData.substr(comparePos, s)); + comparePos += s; + } + } + + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLeaseId_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + client.UploadFrom(nullptr, 0); + + Files::DataLake::DataLakeLeaseClient leaseClient( + client, Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId()); + leaseClient.Acquire(Files::DataLake::DataLakeLeaseClient::InfiniteLeaseDuration); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.AccessConditions.LeaseId + = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions.AccessConditions.LeaseId = leaseClient.GetLeaseId(); + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionLastModifiedTime_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + client.UploadFrom(nullptr, 0); + + auto lastModifiedTime = client.GetProperties().Value.LastModified; + auto timeBeforeStr = lastModifiedTime - std::chrono::seconds(2); + auto timeAfterStr = lastModifiedTime + std::chrono::seconds(2); + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.AccessConditions.IfModifiedSince = timeBeforeStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfModifiedSince = timeAfterStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Files::DataLake::QueryFileOptions(); + queryOptions.AccessConditions.IfUnmodifiedSince = timeBeforeStr; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + queryOptions.AccessConditions.IfUnmodifiedSince = timeAfterStr; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + } + + TEST_F(DataLakeFileClientTest, QueryBlobAccessConditionETag_LIVEONLY_) + { + auto const testName(GetTestName()); + auto client = m_fileSystemClient->GetFileClient(testName); + client.UploadFrom(nullptr, 0); + + auto etag = client.GetProperties().Value.ETag; + + Files::DataLake::QueryFileOptions queryOptions; + queryOptions.AccessConditions.IfMatch = etag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfMatch = DummyETag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + + queryOptions = Files::DataLake::QueryFileOptions(); + queryOptions.AccessConditions.IfNoneMatch = DummyETag; + EXPECT_NO_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions)); + queryOptions.AccessConditions.IfNoneMatch = etag; + EXPECT_THROW(client.Query("SELECT * FROM BlobStorage;", queryOptions), StorageException); + } +}}} // namespace Azure::Storage::Test From 97d92fc13c7ace990331ac7c4a06a54b49cbcfd0 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Thu, 22 Sep 2022 10:47:40 +0800 Subject: [PATCH 05/11] merge main to feature/datalakestg82 (#3958) --- .github/workflows/event.yml | 11 ++- CONTRIBUTING.md | 6 +- .../templates/steps/sparse-checkout.yml | 6 +- .../Helpers/DevOps-WorkItem-Helpers.ps1 | 26 ++++++- .../Update-DevOps-Release-WorkItem.ps1 | 4 ++ eng/common/testproxy/target_version.txt | 2 +- eng/common/testproxy/test-proxy-docker.yml | 6 ++ eng/common/testproxy/test-proxy-tool.yml | 10 ++- .../stages/archetype-cpp-release.yml | 1 + eng/pipelines/templates/steps/vcpkg-clone.yml | 8 +++ .../azure-security-attestation/CHANGELOG.md | 5 ++ .../cgmanifest.json | 1 + .../azure/attestation/attestation_client.hpp | 20 ++---- .../attestation/attestation_client_models.hpp | 5 +- .../attestation_client_options.hpp | 6 -- .../src/attestation_client.cpp | 7 +- .../attestation_deserializers_private.cpp | 12 ++-- .../attestation_deserializers_private.hpp | 7 +- .../test/ut/tpmattestation_test.cpp | 3 +- sdk/core/azure-core-test/cgmanifest.json | 45 +++++++------ .../cgmanifest.json | 1 + sdk/core/azure-core/cgmanifest.json | 3 +- .../azure-core/inc/azure/core/datetime.hpp | 47 ++++++++++++- sdk/core/azure-core/test/ut/datetime_test.cpp | 41 ++++++++++++ sdk/core/perf/cgmanifest.json | 1 + sdk/identity/azure-identity/cgmanifest.json | 4 +- .../cgmanifest.json | 67 ++++++++++--------- .../cgmanifest.json | 67 ++++++++++--------- .../cgmanifest.json | 67 ++++++++++--------- sdk/storage/azure-storage-blobs/CHANGELOG.md | 6 ++ .../azure-storage-blobs/cgmanifest.json | 1 + .../azure-storage-blobs/src/rest_client.cpp | 21 ++++-- .../azure-storage-blobs/swagger/README.md | 20 ++++++ sdk/storage/azure-storage-blobs/vcpkg.json | 2 +- .../azure-storage-common/cgmanifest.json | 1 + .../cgmanifest.json | 1 + .../cgmanifest.json | 1 + .../azure-storage-queues/cgmanifest.json | 1 + sdk/template/azure-template/cgmanifest.json | 67 ++++++++++--------- 39 files changed, 395 insertions(+), 215 deletions(-) diff --git a/.github/workflows/event.yml b/.github/workflows/event.yml index 3a88082782..76c184c71c 100644 --- a/.github/workflows/event.yml +++ b/.github/workflows/event.yml @@ -1,4 +1,7 @@ -name: GitHub Event Handler +# NOTE: currently azure-sdk-actions only hosts check enforcer code. +# If further functionality is added, this name should be updated to reflect +# the more generic behavior +name: Check Enforcer on: check_suite: @@ -6,8 +9,14 @@ on: issue_comment: types: [created] +permissions: {} + jobs: event-handler: + permissions: + statuses: write # to set status (azure/azure-sdk-actions) + pull-requests: read # to read pull requests (azure/azure-sdk-actions) + checks: read # to read check status (azure/azure-sdk-actions) name: Handle ${{ github.event_name }} ${{ github.event.action }} event runs-on: ubuntu-latest steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6eab9f5cab..e35adf304c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Thank you for your interest in contributing to Azure SDK for C++. - **DO** submit all code changes via pull requests (PRs) rather than through a direct commit. PRs will be reviewed and potentially merged by the repo maintainers after a peer review that includes at least one maintainer. - **DO** review your own PR to make sure there aren't any unintended changes or commits before submitting it. - **DO NOT** submit "work in progress" PRs. A PR should only be submitted when it is considered ready for review and subsequent merging by the contributor. - - If the change is work-in-progress or an experiment, **DO** start if off as a temporary draft PR. + - If the change is work-in-progress or an experiment, **DO** start it off as a temporary draft PR. - **DO** give PRs short-but-descriptive names (e.g. "Improve code coverage for Azure.Core by 10%", not "Fix #1234") and add a description which explains why the change is being made. - **DO** refer to any relevant issues, and include [keywords](https://docs.github.com/articles/closing-issues-via-commit-messages/) that automatically close issues when the PR is merged. - **DO** tag any users that should know about and/or review the change. @@ -48,13 +48,13 @@ Codespaces is new technology that allows you to use a container as your developm ### GitHub Codespaces 1. From the Azure SDK GitHub repo, click on the "Code -> Open with Codespaces" button. -1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing). +1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project). ### VS Code Codespaces 1. Install the [VS Code Remote Extension Pack](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) 1. When you open the Azure SDK for C++ repo in VS Code, it will prompt you to open the project in the Dev Container. If it does not prompt you, then hit CTRL+P, and select "Remote-Containers: Open Folder in Container..." -1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing). +1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project). ## Full Local Setup diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 39d5a7bff1..cee1fbb2d5 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -8,16 +8,12 @@ parameters: - Name: $(Build.Repository.Name) Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory) - # NOTE: SkipDefaultCheckout is being deprecated in favor of SkipCheckoutNone - - name: SkipDefaultCheckout - type: boolean - default: false - name: SkipCheckoutNone type: boolean default: false steps: - - ${{ if and(not(parameters.SkipDefaultCheckout), not(parameters.SkipCheckoutNone)) }}: + - ${{ if not(parameters.SkipCheckoutNone) }}: - checkout: none - task: PowerShell@2 diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 index 07bd6f8848..ba299856b2 100644 --- a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 +++ b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 @@ -108,8 +108,8 @@ function LoginToAzureDevops([string]$devops_pat) if (!$devops_pat) { return } - $azCmdStr = "'$devops_pat' | az devops login $($ReleaseDevOpsOrgParameters -join ' ')" - Invoke-Expression $azCmdStr + # based on the docs at https://aka.ms/azure-devops-cli-auth the recommendation is to set this env variable to login + $env:AZURE_DEVOPS_EXT_PAT = $devops_pat } function BuildHashKeyNoNull() @@ -234,6 +234,7 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr $fields += "Custom.PackagePatchVersions" $fields += "Custom.Generated" $fields += "Custom.RoadmapState" + $fields += "Microsoft.VSTS.Common.StateChangeDate" $fieldList = ($fields | ForEach-Object { "[$_]"}) -join ", " $query = "SELECT ${fieldList} FROM WorkItems WHERE [Work Item Type] = 'Package'" @@ -466,7 +467,7 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte if ($pkgName -ne $existingItem.fields["Custom.Package"]) { $changedField = "Custom.Package" } if ($verMajorMinor -ne $existingItem.fields["Custom.PackageVersionMajorMinor"]) { $changedField = "Custom.PackageVersionMajorMinor" } if ($pkgDisplayName -ne $existingItem.fields["Custom.PackageDisplayName"]) { $changedField = "Custom.PackageDisplayName" } - if ($pkgType -ne $existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" } + if ($pkgType -ne [string]$existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" } if ($pkgNewLibrary -ne $existingItem.fields["Custom.PackageTypeNewLibrary"]) { $changedField = "Custom.PackageTypeNewLibrary" } if ($pkgRepoPath -ne $existingItem.fields["Custom.PackageRepoPath"]) { $changedField = "Custom.PackageRepoPath" } if ($serviceName -ne $existingItem.fields["Custom.ServiceName"]) { $changedField = "Custom.ServiceName" } @@ -884,6 +885,25 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions) "value": "$shippedPackages" } "@ + + # If we shipped a version after we set "In Release" state then reset the state to "Next Release Unknown" + if ($pkgWorkItem.fields["System.State"] -eq "In Release") + { + $lastShippedDate = [DateTime]$newShippedVersions[0].Date + $markedInReleaseDate = ([DateTime]$pkgWorkItem.fields["Microsoft.VSTS.Common.StateChangeDate"]) + + # We just shipped so lets set the state to "Next Release Unknown" + if ($markedInReleaseDate -le $lastShippedDate) + { + $fieldUpdates += @' +{ + "op": "replace", + "path": "/fields/State", + "value": "Next Release Unknown" +} +'@ + } + } } # Full merged version set diff --git a/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 b/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 index 34c7540062..cb96691b83 100644 --- a/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 +++ b/eng/common/scripts/Update-DevOps-Release-WorkItem.ps1 @@ -33,6 +33,10 @@ az extension show -n azure-devops *> $null if (!$?){ Write-Host 'Installing azure-devops extension' az extension add --name azure-devops +} else { + # Force update the extension to the latest version if it was already installed + # this is needed to ensure we have the authentication issue fixed from earlier versions + az extension update -n azure-devops *> $null } . (Join-Path $PSScriptRoot SemVer.ps1) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index 0ea1143fba..aef19a2487 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220810.2 +1.0.0-dev.20220921.2 diff --git a/eng/common/testproxy/test-proxy-docker.yml b/eng/common/testproxy/test-proxy-docker.yml index 52a7c807a3..71a03d1be3 100644 --- a/eng/common/testproxy/test-proxy-docker.yml +++ b/eng/common/testproxy/test-proxy-docker.yml @@ -2,24 +2,30 @@ parameters: rootFolder: '$(Build.SourcesDirectory)' targetVersion: '' templateRoot: '$(Build.SourcesDirectory)' + condition: true steps: - pwsh: | ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | docker info displayName: 'Dump active docker information' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | ${{ parameters.templateRoot }}/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" -VersionOverride="${{ parameters.targetVersion }}" displayName: 'Run the docker container' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | docker container ls -a displayName: Check running container + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" displayName: 'Set PROXY_MANUAL_START' + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/common/testproxy/test-proxy-tool.yml b/eng/common/testproxy/test-proxy-tool.yml index 679ad2108d..293848da12 100644 --- a/eng/common/testproxy/test-proxy-tool.yml +++ b/eng/common/testproxy/test-proxy-tool.yml @@ -3,11 +3,13 @@ parameters: runProxy: true targetVersion: '' templateRoot: '$(Build.SourcesDirectory)' + condition: true steps: - pwsh: | ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1 displayName: 'Language Specific Certificate Trust' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | $version = $(Get-Content "${{ parameters.templateRoot }}/eng/common/testproxy/target_version.txt" -Raw).Trim() @@ -23,6 +25,7 @@ steps: --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json ` --version $version displayName: "Install test-proxy" + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy" @@ -34,19 +37,20 @@ steps: Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password" Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true" displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables' + condition: and(succeeded(), ${{ parameters.condition }}) - pwsh: | Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe ` -ArgumentList "--storage-location ${{ parameters.rootFolder }}" ` -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.templateRoot }}/test-proxy.log displayName: 'Run the testproxy - windows' - condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT')) + condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) # nohup does NOT continue beyond the current session if you use it within powershell - bash: | nohup $(Build.BinariesDirectory)/test-proxy/test-proxy > ${{ parameters.templateRoot }}/test-proxy.log & displayName: "Run the testproxy - linux/mac" - condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT')) + condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }}) workingDirectory: "${{ parameters.rootFolder }}" - pwsh: | @@ -62,4 +66,4 @@ steps: Write-Error "Could not connect to test proxy." exit 1 displayName: Test Proxy IsAlive - + condition: and(succeeded(), ${{ parameters.condition }}) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index e4070400a6..aaf9eeb9f1 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -144,6 +144,7 @@ stages: parameters: RepoOwner: Azure RepoName: azure-sdk-vcpkg-betas + SetVcpkgRoot: false - template: /eng/pipelines/templates/steps/vcpkg-publish.yml parameters: diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index b374bee15d..eb0f1dc8ab 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -3,6 +3,7 @@ parameters: RepoOwner: not-set RepoName: vcpkg PRBranchName: not-set + SetVcpkgRoot: true steps: - pwsh: | @@ -14,6 +15,13 @@ steps: } displayName: Clone vcpkg (${{ parameters.RepoOwner }}/${{ parameters.RepoName }}) + - ${{ if eq(parameters.SetVcpkgRoot, true) }}: + - pwsh: | + $vcpkgRoot = Resolve-Path "${{ parameters.Workspace }}/${{ parameters.RepoName }}" + Write-Host "Set VCPKG_ROOT: $vcpkgRoot" + Write-Host "##vso[task.setvariable variable=VCPKG_ROOT]$vcpkgRoot" + displayName: Set VCPKG_ROOT + # Check out the PR branch if it's already in remote. Ignore failures. - pwsh: | $ErrorActionPreference = "Continue" diff --git a/sdk/attestation/azure-security-attestation/CHANGELOG.md b/sdk/attestation/azure-security-attestation/CHANGELOG.md index b032e5a8d2..e5c537269e 100644 --- a/sdk/attestation/azure-security-attestation/CHANGELOG.md +++ b/sdk/attestation/azure-security-attestation/CHANGELOG.md @@ -6,6 +6,11 @@ ### Breaking Changes +- Changed `AttestationClient::AttestTpm` to match `AttestOpenEnclave` and `AttestSgxEnclave` + - Added `std::vector` dataToAttest parameter. + - Removed `PayLoad` in `TpmAttestationOptions` + - Changed `TpmResult` in `TpmAttestationResult` to type `std::vector` + ### Bugs Fixed ### Other Changes diff --git a/sdk/attestation/azure-security-attestation/cgmanifest.json b/sdk/attestation/azure-security-attestation/cgmanifest.json index d4b0875a5c..648f67628c 100644 --- a/sdk/attestation/azure-security-attestation/cgmanifest.json +++ b/sdk/attestation/azure-security-attestation/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp index 4aa60061ed..d7c44d4941 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp @@ -223,26 +223,20 @@ namespace Azure { namespace Security { namespace Attestation { Azure::Core::Context const& context = Azure::Core::Context{}) const; /** - * @brief Perform a single leg - * - * Processes attestation evidence from a VBS enclave, producing an attestation result. - * + * @brief Sends TPM-based attestation data to the service. * The TPM attestation protocol is defined * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') * - * Unlike OpenEnclave reports and SGX enclave quotes, TPM attestation is implemented using - * JSON encoded strings. * - * The client formats a string serialized JSON request to the - * service, which responds with a JSON response. The serialized JSON object exchange continues - * until the service responds with a JSON string with a property named {@code "report"}, whose - * value will be an attestation result token. + * @param dataToAttest - Attestation request data. + * @param options - Options to the attestation request. + * @param context - Context for the operation. * - * @param options sent to the service for Trusted Platform Module (TPM) attestation. - * @return attestation response for Trusted Platform Module (TPM) attestation. + * @return Response - The result of the attestation operation */ Response AttestTpm( - AttestTpmOptions const& options, + std::vector const& dataToAttest, + AttestTpmOptions const& options = AttestTpmOptions{}, Azure::Core::Context const& context = Azure::Core::Context{}) const; private: diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp index 2fb15bcc51..c4db47e2ae 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp @@ -448,12 +448,13 @@ namespace Azure { namespace Security { namespace Attestation { namespace Models */ struct TpmAttestationResult final { - /** @brief The JSON encoded value returned from TPM attestation. + /** @brief Attestation response data. + * * The TPM attestation protocol is defined * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') * */ - std::string TpmResult; + std::vector TpmResult; }; /** diff --git a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp index a94cf6092e..b9f1a2450a 100644 --- a/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp +++ b/sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp @@ -261,12 +261,6 @@ namespace Azure { namespace Security { namespace Attestation { */ struct AttestTpmOptions final { - /** - * @brief JSON Data to send to the attestation service for TPM attestation. - * @details The TPM attestation protocol is defined - * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol') - */ - std::string Payload; }; /** @brief The AttestationSigningKey represents a tuple of asymmetric private cryptographic key diff --git a/sdk/attestation/azure-security-attestation/src/attestation_client.cpp b/sdk/attestation/azure-security-attestation/src/attestation_client.cpp index 1e0077adc3..8786466e1d 100644 --- a/sdk/attestation/azure-security-attestation/src/attestation_client.cpp +++ b/sdk/attestation/azure-security-attestation/src/attestation_client.cpp @@ -196,13 +196,14 @@ Azure::Response> AttestationClient::AttestOp } Azure::Response AttestationClient::AttestTpm( - AttestTpmOptions const& attestTpmOptions, + std::vector const& dataToAttest, + AttestTpmOptions const&, Azure::Core::Context const& context) const { auto tracingContext(m_tracingFactory.CreateTracingContext("AttestTpm", context)); try { - std::string jsonToSend = TpmDataSerializer::Serialize(attestTpmOptions.Payload); + std::string jsonToSend = TpmDataSerializer::Serialize(dataToAttest); auto encodedVector = std::vector(jsonToSend.begin(), jsonToSend.end()); Azure::Core::IO::MemoryBodyStream stream(encodedVector); @@ -212,7 +213,7 @@ Azure::Response AttestationClient::AttestTpm( // Send the request to the service. auto response = AttestationCommonRequest::SendRequest(*m_pipeline, request, tracingContext.Context); - std::string returnedBody(TpmDataSerializer::Deserialize(response)); + std::vector returnedBody{TpmDataSerializer::Deserialize(response)}; return Response(TpmAttestationResult{returnedBody}, std::move(response)); } catch (std::runtime_error const& ex) diff --git a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp index 580fcdf503..684cec9432 100644 --- a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp +++ b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp @@ -397,21 +397,21 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail returnValue.CertificateThumbprint, jsonResult, "x-ms-certificate-thumbprint"); return returnValue; } - std::string TpmDataSerializer::Serialize(std::string const& tpmData) + std::string TpmDataSerializer::Serialize(std::vector const& tpmData) { Azure::Core::Json::_internal::json jsonData; - jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode( - std::vector(tpmData.begin(), tpmData.end())); + jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode(tpmData); return jsonData.dump(); } - std::string TpmDataSerializer::Deserialize(Azure::Core::Json::_internal::json const& jsonData) + std::vector TpmDataSerializer::Deserialize( + Azure::Core::Json::_internal::json const& jsonData) { std::vector returnValue; JsonOptional::SetIfExists>( returnValue, jsonData, "data", Azure::Core::_internal::Base64Url::Base64UrlDecode); - return std::string(returnValue.begin(), returnValue.end()); + return returnValue; } - std::string TpmDataSerializer::Deserialize( + std::vector TpmDataSerializer::Deserialize( std::unique_ptr const& response) { return TpmDataSerializer::Deserialize( diff --git a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp index 6cc7d1279b..9b72f35fac 100644 --- a/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp @@ -136,9 +136,10 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail struct TpmDataSerializer { - static std::string Serialize(std::string const& tpmData); - static std::string Deserialize(Azure::Core::Json::_internal::json const& jsonData); - static std::string Deserialize(std::unique_ptr const& response); + static std::string Serialize(std::vector const& tpmData); + static std::vector Deserialize(Azure::Core::Json::_internal::json const& jsonData); + static std::vector Deserialize( + std::unique_ptr const& response); }; }}}} // namespace Azure::Security::Attestation::_detail diff --git a/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp b/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp index dad0c78f55..2d0737e12c 100644 --- a/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp +++ b/sdk/attestation/azure-security-attestation/test/ut/tpmattestation_test.cpp @@ -118,7 +118,8 @@ namespace Azure { namespace Security { namespace Attestation { namespace Test { { auto client(CreateClient(InstanceType::AAD)); - auto response(client.AttestTpm(AttestTpmOptions{R"({"payload": { "type": "aikcert" } })"})); + std::string tpmQuote = R"({"payload": { "type": "aikcert" } })"; + auto response(client.AttestTpm(std::vector(tpmQuote.begin(), tpmQuote.end()))); Azure::Core::Json::_internal::json parsedResponse( Azure::Core::Json::_internal::json::parse(response.Value.TpmResult)); diff --git a/sdk/core/azure-core-test/cgmanifest.json b/sdk/core/azure-core-test/cgmanifest.json index 7348d592f0..00dd915e65 100644 --- a/sdk/core/azure-core-test/cgmanifest.json +++ b/sdk/core/azure-core-test/cgmanifest.json @@ -1,25 +1,26 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json b/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json index 698c2a9558..1e4219e5c9 100644 --- a/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json +++ b/sdk/core/azure-core-tracing-opentelemetry/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/core/azure-core/cgmanifest.json b/sdk/core/azure-core/cgmanifest.json index 5e418d475b..a84431c793 100644 --- a/sdk/core/azure-core/cgmanifest.json +++ b/sdk/core/azure-core/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { @@ -94,4 +95,4 @@ "DevelopmentDependency": true } ] -} \ No newline at end of file +} diff --git a/sdk/core/azure-core/inc/azure/core/datetime.hpp b/sdk/core/azure-core/inc/azure/core/datetime.hpp index 4ed531473c..53eecb5f9f 100644 --- a/sdk/core/azure-core/inc/azure/core/datetime.hpp +++ b/sdk/core/azure-core/inc/azure/core/datetime.hpp @@ -269,7 +269,7 @@ inline bool operator>=(std::chrono::system_clock::time_point const& tp, DateTime namespace Core { namespace _internal { /** - * @brief Provides convertion methods for POSIX time to an #Azure::DateTime. + * @brief Provides conversion methods for POSIX time to an #Azure::DateTime. * */ class PosixTimeConverter final { @@ -311,6 +311,51 @@ namespace Core { namespace _internal { */ ~PosixTimeConverter() = delete; }; + + /** + * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. + * + */ + class Win32FileTimeConverter final { + public: + /** + * @brief Converts Win32 FILETIME to an #Azure::DateTime. + * + * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. + * @return Calculated #Azure::DateTime. + */ + static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) + { + auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); + return DateTime(t); + } + + /** + * @brief Converts a DateTime to Win32 FILETIME. + * + * @param dateTime The `%DateTime` to convert. + * @return The number of 100-nanoseconds since 1601-01-01. + */ + static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) + { + return std::chrono::duration_cast(dateTime - DateTime(1601)) + .count(); + } + + private: + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be created. + * + */ + Win32FileTimeConverter() = delete; + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no + * instance can be created. + * + */ + ~Win32FileTimeConverter() = delete; + }; }} // namespace Core::_internal } // namespace Azure diff --git a/sdk/core/azure-core/test/ut/datetime_test.cpp b/sdk/core/azure-core/test/ut/datetime_test.cpp index 2496819925..02534134f2 100644 --- a/sdk/core/azure-core/test/ut/datetime_test.cpp +++ b/sdk/core/azure-core/test/ut/datetime_test.cpp @@ -895,3 +895,44 @@ TEST(DateTime, LeapYear) EXPECT_NO_THROW(static_cast(DateTime(2021, 2, 28))); EXPECT_THROW(static_cast(DateTime(2021, 2, 29)), std::invalid_argument); } + +TEST(DateTime, Win32FileTimeConverter) +{ + using namespace Azure::Core::_internal; + + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(0)), + 0); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(1)), + 1); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412)), + 133080315699856412); + EXPECT_EQ( + Win32FileTimeConverter::DateTimeToWin32FileTime( + Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569)), + 2650467743995784569); + + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(0).ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "1601-01-01T00:00:00.0000000Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(1).ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "1601-01-01T00:00:00.0000001Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412) + .ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "2022-09-19T03:26:09.9856412Z"); + EXPECT_EQ( + Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569) + .ToString( + Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), + "9999-12-31T23:59:59.5784569Z"); +} diff --git a/sdk/core/perf/cgmanifest.json b/sdk/core/perf/cgmanifest.json index 244dcec184..7b765a74ad 100644 --- a/sdk/core/perf/cgmanifest.json +++ b/sdk/core/perf/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/identity/azure-identity/cgmanifest.json b/sdk/identity/azure-identity/cgmanifest.json index 819cdc37e6..f16d4d85ac 100644 --- a/sdk/identity/azure-identity/cgmanifest.json +++ b/sdk/identity/azure-identity/cgmanifest.json @@ -1,4 +1,4 @@ { - "Registrations": [ - ] + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [] } diff --git a/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-certificates/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-keys/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json b/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json +++ b/sdk/keyvault/azure-security-keyvault-secrets/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } diff --git a/sdk/storage/azure-storage-blobs/CHANGELOG.md b/sdk/storage/azure-storage-blobs/CHANGELOG.md index 218ae12165..bbc3e87f8a 100644 --- a/sdk/storage/azure-storage-blobs/CHANGELOG.md +++ b/sdk/storage/azure-storage-blobs/CHANGELOG.md @@ -10,6 +10,12 @@ ### Other Changes +## 12.6.1 (2022-09-16) + +### Other changes + +- No public changes in this release. + ## 12.6.0 (2022-09-06) ### Features Added diff --git a/sdk/storage/azure-storage-blobs/cgmanifest.json b/sdk/storage/azure-storage-blobs/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-blobs/cgmanifest.json +++ b/sdk/storage/azure-storage-blobs/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 7742fd6087..29eb40f9f3 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -6029,8 +6029,11 @@ namespace Azure { namespace Storage { namespace Blobs { = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64")); response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } response.IsServerEncrypted = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) @@ -6138,8 +6141,11 @@ namespace Azure { namespace Storage { namespace Blobs { response.LastModified = DateTime::Parse( pRawResponse->GetHeaders().at("Last-Modified"), Azure::DateTime::DateFormat::Rfc1123); } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } return Response(std::move(response), std::move(pRawResponse)); } Response PageBlobClient::UploadPagesFromUri( @@ -6298,8 +6304,11 @@ namespace Azure { namespace Storage { namespace Blobs { = Core::Convert::Base64Decode(pRawResponse->GetHeaders().at("x-ms-content-crc64")); response.TransactionalContentHash.Value().Algorithm = HashAlgorithm::Crc64; } - response.SequenceNumber - = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + if (pRawResponse->GetHeaders().count("x-ms-blob-sequence-number") != 0) + { + response.SequenceNumber + = std::stoll(pRawResponse->GetHeaders().at("x-ms-blob-sequence-number")); + } response.IsServerEncrypted = pRawResponse->GetHeaders().at("x-ms-request-server-encrypted") == std::string("true"); if (pRawResponse->GetHeaders().count("x-ms-encryption-key-sha256") != 0) diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 493c96ad29..494f44eafb 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -515,6 +515,26 @@ directive: }); } } + - from: swagger-document + where: $ + transform: > + const operations = [ + "PageBlob_UploadPages", + "PageBlob_ClearPages", + "PageBlob_UploadPagesFromUri", + ]; + for (const url in $["x-ms-paths"]) { + for (const verb in $["x-ms-paths"][url]) { + if (!operations.includes($["x-ms-paths"][url][verb].operationId)) continue; + const operation = $["x-ms-paths"][url][verb]; + + const status_codes = Object.keys(operation.responses).filter(s => s !== "default"); + status_codes.forEach((status_code, i) => { + operation.responses[status_code].headers["x-ms-blob-sequence-number"]["x-ms-client-default"] = "int64_t()"; + operation.responses[status_code].headers["x-ms-blob-sequence-number"]["x-nullable"] = true; + }); + } + } ``` ### GetBlobServiceProperties diff --git a/sdk/storage/azure-storage-blobs/vcpkg.json b/sdk/storage/azure-storage-blobs/vcpkg.json index 6d375116db..ae37d635b5 100644 --- a/sdk/storage/azure-storage-blobs/vcpkg.json +++ b/sdk/storage/azure-storage-blobs/vcpkg.json @@ -1,6 +1,6 @@ { "name": "azure-storage-blobs-cpp", - "version-semver": "12.6.0", + "version-semver": "12.6.1", "description": [ "Microsoft Azure Storage Blobs SDK for C++", "This library provides Azure Storage Blobs SDK." diff --git a/sdk/storage/azure-storage-common/cgmanifest.json b/sdk/storage/azure-storage-common/cgmanifest.json index 568ceec412..5a5f148d9c 100644 --- a/sdk/storage/azure-storage-common/cgmanifest.json +++ b/sdk/storage/azure-storage-common/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-files-datalake/cgmanifest.json b/sdk/storage/azure-storage-files-datalake/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-files-datalake/cgmanifest.json +++ b/sdk/storage/azure-storage-files-datalake/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-files-shares/cgmanifest.json b/sdk/storage/azure-storage-files-shares/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-files-shares/cgmanifest.json +++ b/sdk/storage/azure-storage-files-shares/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/storage/azure-storage-queues/cgmanifest.json b/sdk/storage/azure-storage-queues/cgmanifest.json index 728e7b7085..4ac5c618a3 100644 --- a/sdk/storage/azure-storage-queues/cgmanifest.json +++ b/sdk/storage/azure-storage-queues/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { diff --git a/sdk/template/azure-template/cgmanifest.json b/sdk/template/azure-template/cgmanifest.json index 7f2901e706..1b872bc72d 100644 --- a/sdk/template/azure-template/cgmanifest.json +++ b/sdk/template/azure-template/cgmanifest.json @@ -1,36 +1,37 @@ { - "Registrations": [ - { - "Component": { - "Type": "git", - "git": { - "RepositoryUrl": "https://github.com/google/googletest", - "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "clang-format", - "Version": "9.0.0-2", - "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" - } - }, - "DevelopmentDependency": true - }, - { - "Component": { - "Type": "other", - "Other": { - "Name": "doxygen", - "Version": "1.8.20", - "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" - } - }, - "DevelopmentDependency": true + "$schema": "https://json.schemastore.org/component-detection-manifest.json", + "Registrations": [ + { + "Component": { + "Type": "git", + "git": { + "RepositoryUrl": "https://github.com/google/googletest", + "CommitHash": "703bd9caab50b139428cea1aaff9974ebee5742e" } - ] + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "clang-format", + "Version": "9.0.0-2", + "DownloadUrl": "https://ubuntu.pkgs.org/18.04/ubuntu-updates-universe-amd64/clang-format-9_9-2~ubuntu18.04.2_amd64.deb.html" + } + }, + "DevelopmentDependency": true + }, + { + "Component": { + "Type": "other", + "Other": { + "Name": "doxygen", + "Version": "1.8.20", + "DownloadUrl": "http://doxygen.nl/files/doxygen-1.8.20-setup.exe" + } + }, + "DevelopmentDependency": true + } + ] } From 43dd2083ad2e8514367ebe211508ca870c2642a1 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Fri, 23 Sep 2022 10:17:06 +0800 Subject: [PATCH 06/11] cpk for datalake (#3955) --- .../datalake/datalake_directory_client.hpp | 9 +- .../files/datalake/datalake_file_client.hpp | 9 +- .../datalake/datalake_file_system_client.hpp | 7 +- .../files/datalake/datalake_options.hpp | 7 + .../files/datalake/datalake_path_client.hpp | 6 +- .../datalake/datalake_service_client.hpp | 1 + .../storage/files/datalake/rest_client.hpp | 3 + .../src/datalake_directory_client.cpp | 24 +- .../src/datalake_file_client.cpp | 12 + .../src/datalake_file_system_client.cpp | 35 ++- .../src/datalake_path_client.cpp | 15 +- .../src/datalake_service_client.cpp | 14 +- .../src/datalake_utilities.cpp | 1 + .../src/rest_client.cpp | 15 +- .../swagger/README.md | 2 + .../ut/datalake_file_system_client_test.cpp | 226 ++++++++++++++++++ 16 files changed, 349 insertions(+), 37 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp index 05aaf8de74..b6b74c4d5d 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp @@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeDirectoryClient( Azure::Core::Url directoryUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(directoryUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index a4ade4cc68..f9aa309ef3 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -282,8 +282,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { explicit DataLakeFileClient( Azure::Core::Url fileUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) - : DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) + : DataLakePathClient( + std::move(fileUrl), + std::move(blobClient), + pipeline, + customerProvidedKey) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index a1caa22248..9a7430e1c1 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -243,13 +243,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_fileSystemUrl; Blobs::BlobContainerClient m_blobContainerClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakeFileSystemClient( Azure::Core::Url fileSystemUrl, Blobs::BlobContainerClient blobContainerClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_fileSystemUrl(std::move(fileSystemUrl)), - m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)) + m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)), + m_customerProvidedKey(std::move(customerProvidedKey)) { } friend class DataLakeLeaseClient; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index 4d73a3fe15..fc8e69a015 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -23,10 +23,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using SignedIdentifier = Blobs::Models::SignedIdentifier; using FileQueryArrowField = Blobs::Models::BlobQueryArrowField; using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType; + using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType; } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; + using EncryptionKey = Blobs::EncryptionKey; /** * @brief Client options used to initialize all DataLake clients. @@ -46,6 +48,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * API version used by this client. */ std::string ApiVersion; + + /** + * @brief Holds the customer provided key used when making requests. + */ + Azure::Nullable CustomerProvidedKey; }; /** diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp index f6183a3e70..497b36ac0a 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_path_client.hpp @@ -295,13 +295,15 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_pathUrl; Blobs::BlobClient m_blobClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; explicit DataLakePathClient( Azure::Core::Url pathUrl, Blobs::BlobClient blobClient, - std::shared_ptr pipeline) + std::shared_ptr pipeline, + Azure::Nullable customerProvidedKey = Azure::Nullable()) : m_pathUrl(std::move(pathUrl)), m_blobClient(std::move(blobClient)), - m_pipeline(std::move(pipeline)) + m_pipeline(std::move(pipeline)), m_customerProvidedKey(std::move(customerProvidedKey)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp index 633ddf57cc..564c971d02 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp @@ -113,5 +113,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Core::Url m_serviceUrl; Blobs::BlobServiceClient m_blobServiceClient; std::shared_ptr m_pipeline; + Azure::Nullable m_customerProvidedKey; }; }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index 0a58b8e565..c6e755f35c 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -374,6 +374,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable SourceIfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Owner; Nullable Group; Nullable Acl; @@ -478,6 +479,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable IfUnmodifiedSince; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; }; static Response Flush( Core::Http::_internal::HttpPipeline& pipeline, @@ -492,6 +494,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable LeaseId; Nullable EncryptionKey; Nullable> EncryptionKeySha256; + Nullable EncryptionAlgorithm; Nullable Flush; }; static Response Append( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 1ac31d8f52..7e81aa9f0b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -66,7 +66,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(fileName)); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeDirectoryClient::GetSubdirectoryClient( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { builder.AppendPath(_internal::UrlEncodePath(subdirectoryName)); auto blobClient = m_blobClient; blobClient.m_blobUrl.AppendPath(_internal::UrlEncodePath(subdirectoryName)); - return DataLakeDirectoryClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeDirectoryClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } Azure::Response DataLakeDirectoryClient::RenameFile( @@ -119,10 +121,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(response.RawResponse)); } @@ -167,10 +172,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto response = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(response.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 407dbaaaa5..43606b0376 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -81,6 +81,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Append( *m_pipeline, m_pathUrl, content, protocolLayerOptions, context); } @@ -111,6 +117,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch; protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince; protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::FileClient::Flush(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c478c0cf50..38c6668b2b 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -50,7 +50,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -78,7 +79,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -107,7 +109,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_fileSystemUrl(fileSystemUrl), m_blobContainerClient( _detail::GetBlobUrlFromUrl(fileSystemUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -129,7 +132,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(fileName)); auto blobClient = m_blobContainerClient.GetBlobClient(fileName); - return DataLakeFileClient(std::move(builder), std::move(blobClient), m_pipeline); + return DataLakeFileClient( + std::move(builder), std::move(blobClient), m_pipeline, m_customerProvidedKey); } DataLakeDirectoryClient DataLakeFileSystemClient::GetDirectoryClient( @@ -138,7 +142,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_fileSystemUrl; builder.AppendPath(_internal::UrlEncodePath(directoryName)); return DataLakeDirectoryClient( - builder, m_blobContainerClient.GetBlobClient(directoryName), m_pipeline); + builder, + m_blobContainerClient.GetBlobClient(directoryName), + m_pipeline, + m_customerProvidedKey); } Azure::Response DataLakeFileSystemClient::Create( @@ -356,10 +363,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedFileClient = DataLakeFileClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedFileClient), std::move(result.RawResponse)); } @@ -404,10 +414,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto result = _detail::PathClient::Create( *m_pipeline, destinationDfsUrl, protocolLayerOptions, context); - auto renamedBlobClient - = Blobs::BlobClient(_detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline); + auto renamedBlobClient = Blobs::BlobClient( + _detail::GetBlobUrlFromUrl(destinationDfsUrl), m_pipeline, m_customerProvidedKey); auto renamedDirectoryClient = DataLakeDirectoryClient( - std::move(destinationDfsUrl), std::move(renamedBlobClient), m_pipeline); + std::move(destinationDfsUrl), + std::move(renamedBlobClient), + m_pipeline, + m_customerProvidedKey); return Azure::Response( std::move(renamedDirectoryClient), std::move(result.RawResponse)); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 0d6990d864..9226372e12 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -48,7 +48,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -76,7 +77,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_pathUrl(pathUrl), m_blobClient( _detail::GetBlobUrlFromUrl(pathUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -104,7 +106,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const std::string& pathUrl, const DataLakeClientOptions& options) : m_pathUrl(pathUrl), - m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)) + m_blobClient(_detail::GetBlobUrlFromUrl(pathUrl), _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -202,6 +205,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata); protocolLayerOptions.Umask = options.Umask; protocolLayerOptions.Permissions = options.Permissions; + if (m_customerProvidedKey.HasValue()) + { + protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; + protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash; + protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString(); + } return _detail::PathClient::Create(*m_pipeline, m_pathUrl, protocolLayerOptions, context); } diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp index 4633d299ec..240abb8b65 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_service_client.cpp @@ -44,7 +44,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { DataLakeClientOptions newOptions = options; newOptions.PerRetryPolicies.emplace_back( @@ -72,7 +73,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), credential, - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -101,7 +103,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const DataLakeClientOptions& options) : m_serviceUrl(serviceUrl), m_blobServiceClient( _detail::GetBlobUrlFromUrl(serviceUrl), - _detail::GetBlobClientOptions(options)) + _detail::GetBlobClientOptions(options)), + m_customerProvidedKey(options.CustomerProvidedKey) { std::vector> perRetryPolicies; std::vector> perOperationPolicies; @@ -124,7 +127,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { auto builder = m_serviceUrl; builder.AppendPath(_internal::UrlEncodePath(fileSystemName)); return DataLakeFileSystemClient( - builder, m_blobServiceClient.GetBlobContainerClient(fileSystemName), m_pipeline); + builder, + m_blobServiceClient.GetBlobContainerClient(fileSystemName), + m_pipeline, + m_customerProvidedKey); } ListFileSystemsPagedResponse DataLakeServiceClient::ListFileSystems( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp index 0d7c662f88..76a96546bb 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_utilities.cpp @@ -96,6 +96,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam blobOptions.SecondaryHostForRetryReads = _detail::GetBlobUrlFromUrl(options.SecondaryHostForRetryReads); blobOptions.ApiVersion = options.ApiVersion; + blobOptions.CustomerProvidedKey = options.CustomerProvidedKey; return blobOptions; } diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 2d519a9487..f29440d19c 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -253,7 +253,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Owner.HasValue() && !options.Owner.Value().empty()) { request.SetHeader("x-ms-owner", options.Owner.Value()); @@ -661,7 +664,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -728,7 +734,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { "x-ms-encryption-key-sha256", Core::Convert::Base64Encode(options.EncryptionKeySha256.Value())); } - request.SetHeader("x-ms-encryption-algorithm", "AES256"); + if (options.EncryptionAlgorithm.HasValue() && !options.EncryptionAlgorithm.Value().empty()) + { + request.SetHeader("x-ms-encryption-algorithm", options.EncryptionAlgorithm.Value()); + } if (options.Flush.HasValue()) { request.GetUrl().AppendQueryParameter("flush", options.Flush.Value() ? "true" : "false"); diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 5344d267a8..751130bbde 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -181,6 +181,8 @@ directive: transform: > $.Continuation["x-ms-client-name"] = "ContinuationToken"; $.EncryptionKeySha256["format"] = "byte"; + delete $.EncryptionAlgorithm["enum"]; + delete $.EncryptionAlgorithm["x-ms-enum"]; - from: swagger-document where: $.definitions transform: > diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 615e1096d1..d299436ebc 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -397,6 +398,231 @@ namespace Azure { namespace Storage { namespace Test { } } + TEST_F(DataLakeFileSystemClientTest, CustomerProvidedKey_LIVEONLY_) + { + + auto getRandomCustomerProvidedKey = [&]() { + Files::DataLake::EncryptionKey key; + std::vector aes256Key; + aes256Key.resize(32); + RandomBuffer(&aes256Key[0], aes256Key.size()); + key.Key = Azure::Core::Convert::Base64Encode(aes256Key); + key.KeyHash = Azure::Core::Cryptography::_internal::Sha256Hash().Final( + aes256Key.data(), aes256Key.size()); + key.Algorithm = Blobs::Models::EncryptionAlgorithmType::Aes256; + return key; + }; + + const int32_t bufferSize = 1024; // 1KB data size + auto buffer = std::make_shared>(bufferSize, 'x'); + Azure::Core::IO::MemoryBodyStream bodyStream(buffer->data(), buffer->size()); + + auto customerProvidedKey + = std::make_shared(getRandomCustomerProvidedKey()); + Files::DataLake::DataLakeClientOptions options; + options.CustomerProvidedKey = *customerProvidedKey; + auto fileServiceClient = std::make_shared( + Files::DataLake::DataLakeServiceClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), options)); + auto fileSystemClient = std::make_shared( + fileServiceClient->GetFileSystemClient(m_fileSystemName)); + + // fileSystem works + { + auto fileSystemClientWithoutEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName); + // Rename File + const std::string filename1 = GetTestName() + "file1"; + const std::string filename2 = GetTestName() + "file2"; + const std::string filename3 = GetTestName() + "file3"; + const std::string filename4 = GetTestName() + "file4"; + + auto oldFileClient = fileSystemClient->GetFileClient(filename1); + oldFileClient.Create(); + auto newFileClient = fileSystemClient->RenameFile(filename1, filename2).Value; + auto properties = std::make_shared( + newFileClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, filename2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameFile(filename2, filename3)); + + // Rename Directory + const std::string testName(GetTestName()); + const std::string oldDirectoryName = testName + "dir1"; + const std::string newDirectoryName = testName + "dir2"; + const std::string newDirectoryName2 = testName + "dir3"; + + auto oldDirectoryClient = fileSystemClient->GetDirectoryClient(oldDirectoryName); + oldDirectoryClient.Create(); + oldDirectoryClient.GetFileClient(testName + "file3").Create(); + oldDirectoryClient.GetSubdirectoryClient(testName + "dir4").Create(); + + auto newDirectoryClient + = fileSystemClient->RenameDirectory(oldDirectoryName, newDirectoryName).Value; + properties = std::make_shared( + newDirectoryClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto newDirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, newDirectoryName); + EXPECT_THROW(newDirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(fileSystemClientWithoutEncryptionKey.RenameDirectory( + newDirectoryName, newDirectoryName2)); + + auto fileSystemClientWithEncryptionKey + = Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, options); + auto created = std::make_shared( + fileSystemClientWithEncryptionKey.GetFileClient(filename4).Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + + // path works + { + const std::string pathName = "path"; + const std::string pathName2 = "path2"; + + auto pathClient = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName, options); + EXPECT_NO_THROW(pathClient.Create(Files::DataLake::Models::PathResourceType::File)); + EXPECT_NO_THROW(pathClient.SetMetadata(GetMetadata())); + auto properties = std::make_shared( + pathClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + auto pathClientWithoutEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName); + EXPECT_THROW(pathClientWithoutEncryptionKey.SetMetadata(GetMetadata()), StorageException); + EXPECT_THROW(pathClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.GetAccessControlList()); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetHttpHeaders( + Files::DataLake::Models::PathHttpHeaders())); + EXPECT_NO_THROW(pathClientWithoutEncryptionKey.SetPermissions("rwxrw-rw-")); + + auto pathClientWithEncryptionKey + = Files::DataLake::DataLakePathClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, pathName2, options); + auto created + = pathClientWithEncryptionKey.Create(Files::DataLake::Models::PathResourceType::File) + .Value; + EXPECT_TRUE(created.EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created.EncryptionKeySha256.Value()); + } + + // file works + { + const std::string fileName = "file"; + const std::string fileName2 = "file2"; + auto fileClient = fileSystemClient->GetFileClient(fileName); + auto fileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName); + // upload test + EXPECT_NO_THROW(fileClient.Create()); + EXPECT_NO_THROW(fileClient.UploadFrom(buffer->data(), bufferSize)); + auto result = fileClient.Download(); + auto downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); + EXPECT_NO_THROW(fileClient.Delete()); + // append test + EXPECT_NO_THROW(fileClient.Create()); + bodyStream.Rewind(); + EXPECT_NO_THROW(fileClient.Append(bodyStream, 0)); + bodyStream.Rewind(); + EXPECT_THROW(fileClientWithoutEncryptionKey.Append(bodyStream, bufferSize), StorageException); + EXPECT_NO_THROW(fileClient.Flush(bufferSize)); + result = fileClient.Download(); + downloaded = std::make_shared>(ReadBodyStream(result.Value.Body)); + EXPECT_EQ(*buffer, *downloaded); + EXPECT_NO_THROW(fileClient.SetMetadata(GetMetadata())); + auto properties = std::make_shared( + fileClient.GetProperties().Value); + EXPECT_TRUE(properties->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, properties->EncryptionKeySha256.Value()); + EXPECT_THROW(fileClientWithoutEncryptionKey.Flush(bufferSize), StorageException); + EXPECT_THROW(fileClientWithoutEncryptionKey.Download(), StorageException); + + auto fileClientWithEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, fileName2, options); + auto created = std::make_shared( + fileClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + // directory works + { + const std::string directoryName = "directory"; + const std::string directoryName2 = "directory2"; + const std::string subdirectoryName1 = "subdirectory1"; + const std::string subdirectoryName2 = "subdirectory2"; + const std::string subdirectoryName3 = "subdirectory3"; + const std::string fileName1 = "file1"; + const std::string fileName2 = "file2"; + const std::string fileName3 = "file3"; + auto directoryClient = fileSystemClient->GetDirectoryClient(directoryName); + auto directoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName); + // create subdirectory/file + EXPECT_NO_THROW(directoryClient.Create()); + auto subdirectoryClient = directoryClient.GetSubdirectoryClient(subdirectoryName1); + EXPECT_NO_THROW(subdirectoryClient.Create()); + auto fileClient = directoryClient.GetFileClient(fileName1); + EXPECT_NO_THROW(fileClient.Create()); + auto subdirectoryProperties = std::make_shared( + subdirectoryClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, subdirectoryProperties->EncryptionKeySha256.Value()); + auto fileProperties = fileClient.GetProperties(); + EXPECT_EQ(customerProvidedKey->KeyHash, fileProperties.Value.EncryptionKeySha256.Value()); + + // rename file + auto newFileClient + = directoryClient.RenameFile(fileName1, directoryName + "/" + fileName2).Value; + auto newFileProperties = std::make_shared( + newFileClient.GetProperties().Value); + EXPECT_EQ(customerProvidedKey->KeyHash, newFileProperties->EncryptionKeySha256.Value()); + auto newFileClientWithoutEncryptionKey + = Files::DataLake::DataLakeFileClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName + "/" + fileName2); + EXPECT_THROW(newFileClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameFile( + fileName2, directoryName + "/" + fileName3)); + + auto newSubdirectoryClient + = directoryClient + .RenameSubdirectory(subdirectoryName1, directoryName + "/" + subdirectoryName2) + .Value; + auto newSubdirectoryProperties = std::make_shared( + newSubdirectoryClient.GetProperties().Value); + EXPECT_EQ( + customerProvidedKey->KeyHash, newSubdirectoryProperties->EncryptionKeySha256.Value()); + auto newsubdirectoryClientWithoutEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), + m_fileSystemName, + directoryName + "/" + subdirectoryName2); + EXPECT_THROW(newsubdirectoryClientWithoutEncryptionKey.GetProperties(), StorageException); + EXPECT_NO_THROW(directoryClientWithoutEncryptionKey.RenameSubdirectory( + subdirectoryName2, directoryName + "/" + subdirectoryName3)); + + auto directoryClientWithEncryptionKey + = Files::DataLake::DataLakeDirectoryClient::CreateFromConnectionString( + AdlsGen2ConnectionString(), m_fileSystemName, directoryName2, options); + auto created = std::make_shared( + directoryClientWithEncryptionKey.Create().Value); + EXPECT_TRUE(created->EncryptionKeySha256.HasValue()); + EXPECT_EQ(customerProvidedKey->KeyHash, created->EncryptionKeySha256.Value()); + } + } + TEST_F(DataLakeFileSystemClientTest, GetSetAccessPolicy_LIVEONLY_) { CHECK_SKIP_TEST(); From 552d5033a8635f144da880fe891b59c3a47b2564 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Tue, 27 Sep 2022 15:05:56 +0800 Subject: [PATCH 07/11] Add creation expiry time (#3961) --- .../files/datalake/datalake_options.hpp | 140 ++++++- .../files/datalake/datalake_responses.hpp | 54 --- .../storage/files/datalake/rest_client.hpp | 3 + .../src/datalake_file_client.cpp | 1 + .../src/datalake_file_system_client.cpp | 10 + .../src/datalake_path_client.cpp | 29 ++ .../src/rest_client.cpp | 16 + .../swagger/README.md | 12 +- .../test/ut/datalake_file_client_test.cpp | 34 ++ .../ut/datalake_file_system_client_test.cpp | 21 + .../test/ut/datalake_path_client_test.cpp | 81 ++++ ...akeFileClientTest.AppendFileWithFlush.json | 321 +++++++++++++++ ...ataLakeFileSystemClientTest.ListPaths.json | 306 ++++++++------- ...aLakePathClientTest.CreateWithOptions.json | 364 ++++++++++++++++++ 14 files changed, 1179 insertions(+), 213 deletions(-) create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileClientTest.AppendFileWithFlush.json create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakePathClientTest.CreateWithOptions.json diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index fc8e69a015..db134508cd 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -24,6 +24,60 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using FileQueryArrowField = Blobs::Models::BlobQueryArrowField; using FileQueryArrowFieldType = Blobs::Models::BlobQueryArrowFieldType; using EncryptionAlgorithmType = Blobs::Models::EncryptionAlgorithmType; + + /** + * @brief An access control object. + */ + struct Acl final + { + /** + * The scope of the ACL. + */ + std::string Scope; + + /** + * The type of the ACL. + */ + std::string Type; + + /** + * The ID of the ACL. + */ + std::string Id; + + /** + * The permissions of the ACL. + */ + std::string Permissions; + + /** + * @brief Creates an Acl based on acl input string. + * @param aclString the string to be parsed to Acl. + * @return Acl + */ + static Acl FromString(const std::string& aclString); + + /** + * @brief Creates a string from an Acl. + * @param acl the acl object to be serialized to a string. + * @return std::string + */ + static std::string ToString(const Acl& acl); + + /** + * @brief Creates a vector of Acl from a string that indicates multiple acls. + * @param aclsString the string that contains multiple acls. + * @return std::vector + */ + static std::vector DeserializeAcls(const std::string& aclsString); + + /** + * @brief Creates a string that contains several Acls. + * @param aclsArray the acls to be serialized into a string. + * @return std::string + */ + static std::string SerializeAcls(const std::vector& aclsArray); + }; } // namespace Models using DownloadFileToOptions = Blobs::DownloadBlobToOptions; @@ -266,6 +320,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * Specify the lease access conditions. */ LeaseAccessConditions AccessConditions; + + /** + * If true, the file will be flushed after the append. + */ + Azure::Nullable Flush; }; /** @@ -380,6 +439,33 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { PathAccessConditions AccessConditions; }; + using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType; + + /** + * @brief Options for scheduling the deletion of a path. + */ + struct ScheduleFileDeletionOptions final + { + /** + * The expiry time from the specified origin. Only work if ExpiryOrigin is + * ScheduleFileExpiryOriginType::RelativeToCreation or + * ScheduleFileExpiryOriginType::RelativeToNow. + * Does not apply to directories. + * TimeToExpire and ExpiresOn cannot both be set. + */ + Azure::Nullable TimeToExpire; + + /** + * The expiry time in RFC1123 format. Only works if ExpiryOrigin is + * ScheduleFileExpiryOriginType::Absolute. + * Does not apply to directories. + * ExpiresOn and TimeToExpire cannot both be set. + */ + Azure::Nullable ExpiresOn; + }; + + using SchedulePathDeletionOptions = ScheduleFileDeletionOptions; + /** * @brief Optional parameters for #Azure::Storage::Files::DataLake::PathClient::Create. * @remark Some optional parameter is mandatory in certain combination. @@ -421,6 +507,39 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { */ Azure::Nullable Permissions; + /** + * The owner of the file or directory. + */ + Azure::Nullable Owner; + + /** + * The owning group of the file or directory. + */ + Azure::Nullable Group; + + /** + * Sets POSIX access control rights on files and directories. Each access control entry (ACE) + * consists of a scope, a type, a user or group identifier, and permissions. + */ + Azure::Nullable> Acls; + + /** + * Proposed LeaseId. + */ + Azure::Nullable LeaseId; + + /** + * Specifies the duration of the lease, in seconds, or InfiniteLeaseDuration for a lease that + * never expires. A non-infinite lease can be between 15 and 60 seconds. A lease duration cannot + * be changed using renew or change. Does not apply to directories. + */ + Azure::Nullable LeaseDuration; + + /** + * Optional parameters to schedule the file for deletion. + */ + SchedulePathDeletionOptions ScheduleDeletionOptions; + /** * Specify the access condition for the path. */ @@ -620,27 +739,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } TransferOptions; }; - using ScheduleFileExpiryOriginType = Blobs::Models::ScheduleBlobExpiryOriginType; - - /** - * @brief Optional parameters for #Azure::Storage::Files::DataLake::FileClient::UploadFrom. - */ - struct ScheduleFileDeletionOptions final - { - /** - * The expiry time from the specified origin. Only work if ExpiryOrigin is - * ScheduleFileExpiryOriginType::RelativeToCreation or - * ScheduleFileExpiryOriginType::RelativeToNow. - */ - Azure::Nullable TimeToExpire; - - /** - * The expiry time in RFC1123 format. Only work if ExpiryOrigin is - * ScheduleFileExpiryOriginType::Absolute. - */ - Azure::Nullable ExpiresOn; - }; - using AcquireLeaseOptions = Blobs::AcquireLeaseOptions; using BreakLeaseOptions = Blobs::BreakLeaseOptions; using RenewLeaseOptions = Blobs::RenewLeaseOptions; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index bc076a4cae..ef15e55df3 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -193,60 +193,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using RehydratePriority = Blobs::Models::RehydratePriority; using ArchiveStatus = Blobs::Models::ArchiveStatus; - /** - * @brief An access control object. - */ - struct Acl final - { - /** - * The scope of the ACL. - */ - std::string Scope; - - /** - * The type of the ACL. - */ - std::string Type; - - /** - * The ID of the ACL. - */ - std::string Id; - - /** - * The permissions of the ACL. - */ - std::string Permissions; - - /** - * @brief Creates an Acl based on acl input string. - * @param aclString the string to be parsed to Acl. - * @return Acl - */ - static Acl FromString(const std::string& aclString); - - /** - * @brief Creates a string from an Acl. - * @param acl the acl object to be serialized to a string. - * @return std::string - */ - static std::string ToString(const Acl& acl); - - /** - * @brief Creates a vector of Acl from a string that indicates multiple acls. - * @param aclsString the string that contains multiple acls. - * @return std::vector - */ - static std::vector DeserializeAcls(const std::string& aclsString); - - /** - * @brief Creates a string that contains several Acls. - * @param aclsArray the acls to be serialized into a string. - * @return std::string - */ - static std::string SerializeAcls(const std::vector& aclsArray); - }; - /** * @brief The properties of the path. */ diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index c6e755f35c..d4c0afcb5c 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -96,6 +96,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { * The name of the encryption scope under which the blob is encrypted. */ Nullable EncryptionScope; + Nullable CreatedOn; + Nullable ExpiresOn; std::string ETag; }; namespace _detail { @@ -380,6 +382,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Nullable Acl; Nullable ProposedLeaseId; Nullable LeaseDuration; + Nullable ExpiryOptions; Nullable ExpiresOn; }; static Response Create( diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp index 43606b0376..b40d4b6c05 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp @@ -81,6 +81,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } } protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId; + protocolLayerOptions.Flush = options.Flush; if (m_customerProvidedKey.HasValue()) { protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index 38c6668b2b..c4e292127f 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -276,6 +276,16 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptionsCopy, _internal::WithReplicaStatus(context)); + const auto emptyExpiresOn + = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(0); + for (auto& path : response.Value.Paths) + { + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() == emptyExpiresOn) + { + path.ExpiresOn.Reset(); + } + } + ListPathsPagedResponse pagedResponse; pagedResponse.Paths = std::move(response.Value.Paths); pagedResponse.m_onNextPageFunc = func; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp index 9226372e12..38084fc34d 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp @@ -205,6 +205,35 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptions.Properties = _detail::SerializeMetadata(options.Metadata); protocolLayerOptions.Umask = options.Umask; protocolLayerOptions.Permissions = options.Permissions; + protocolLayerOptions.Owner = options.Owner; + protocolLayerOptions.Group = options.Group; + protocolLayerOptions.ProposedLeaseId = options.LeaseId; + if (options.Acls.HasValue()) + { + protocolLayerOptions.Acl = Models::Acl::SerializeAcls(options.Acls.Value()); + } + if (options.LeaseDuration.HasValue()) + { + protocolLayerOptions.LeaseDuration = static_cast(options.LeaseDuration->count()); + } + AZURE_ASSERT_MSG( + !(options.ScheduleDeletionOptions.ExpiresOn.HasValue() + && options.ScheduleDeletionOptions.TimeToExpire.HasValue()), + "ExpiresOn and TimeToExpire are mutually exclusive."); + if (options.ScheduleDeletionOptions.ExpiresOn.HasValue()) + { + protocolLayerOptions.ExpiryOptions + = Files::DataLake::ScheduleFileExpiryOriginType::Absolute.ToString(); + protocolLayerOptions.ExpiresOn = options.ScheduleDeletionOptions.ExpiresOn.Value().ToString( + Azure::DateTime::DateFormat::Rfc1123); + } + else if (options.ScheduleDeletionOptions.TimeToExpire.HasValue()) + { + protocolLayerOptions.ExpiryOptions + = Files::DataLake::ScheduleFileExpiryOriginType::RelativeToNow.ToString(); + protocolLayerOptions.ExpiresOn + = std::to_string(options.ScheduleDeletionOptions.TimeToExpire.Value().count()); + } if (m_customerProvidedKey.HasValue()) { protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key; diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index f29440d19c..47c7d56686 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -114,6 +114,18 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { vectorElement2.EncryptionScope = var0["EncryptionScope"].get(); } + if (var0.count("creationTime") != 0) + { + vectorElement2.CreatedOn + = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(var0["creationTime"].get())); + } + if (var0.count("expiryTime") != 0) + { + vectorElement2.ExpiresOn + = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(var0["expiryTime"].get())); + } if (var0.count("etag") != 0) { vectorElement2.ETag = var0["etag"].get(); @@ -277,6 +289,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { request.SetHeader("x-ms-lease-duration", std::to_string(options.LeaseDuration.Value())); } + if (options.ExpiryOptions.HasValue() && !options.ExpiryOptions.Value().empty()) + { + request.SetHeader("x-ms-expiry-option", options.ExpiryOptions.Value()); + } if (options.ExpiresOn.HasValue() && !options.ExpiresOn.Value().empty()) { request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value()); diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 751130bbde..9626429b02 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -233,8 +233,8 @@ directive: $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize"; $.Path.properties["isDirectory"]["x-ms-client-default"] = false; $.Path.properties["EncryptionScope"]["x-nullable"] = true; - delete $.Path.properties["creationTime"]; - delete $.Path.properties["expiryTime"]; + $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true}; + $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"}; delete $.Path.properties["eTag"]; $.PathList["x-namespace"] = "_detail"; @@ -252,10 +252,6 @@ directive: ```yaml directive: - - from: swagger-document - where: $["x-ms-paths"]["/{filesystem}/{path}"].put.parameters - transform: > - $ = $.filter(p => !(p["$ref"] && (p["$ref"].endsWith("#/parameters/PathExpiryOptionsOptional")))); - from: swagger-document where: $["x-ms-paths"]["/{filesystem}/{path}"].put.responses transform: > @@ -283,7 +279,9 @@ directive: where: $.parameters transform: > delete $["PathExpiryOptions"]; - delete $["PathExpiryOptionsOptional"]; + delete $.PathExpiryOptionsOptional["enum"]; + delete $.PathExpiryOptionsOptional["x-ms-enum"]; + ``` ### DeletePath diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp index 327b6f1245..a253e807bf 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp @@ -245,6 +245,40 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_EQ(buffer, downloaded); } + TEST_F(DataLakeFileClientTest, AppendFileWithFlush) + { + const int32_t bufferSize = 4 * 1024; // 4KB data size + std::vector buffer(bufferSize, 'x'); + auto bufferStream = std::make_unique( + Azure::Core::IO::MemoryBodyStream(buffer)); + + // Append with flush option + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_flush_true"); + client.Create(); + auto properties1 = client.GetProperties(); + Files::DataLake::AppendFileOptions options; + options.Flush = true; + bufferStream->Rewind(); + client.Append(*bufferStream, 0, options); + auto properties2 = client.GetProperties(); + EXPECT_NE(properties1.Value.ETag, properties2.Value.ETag); + EXPECT_EQ(bufferSize, properties2.Value.FileSize); + } + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_flush_false"); + client.Create(); + auto properties1 = client.GetProperties(); + Files::DataLake::AppendFileOptions options; + options.Flush = false; + bufferStream->Rewind(); + client.Append(*bufferStream, 0, options); + auto properties2 = client.GetProperties(); + EXPECT_EQ(properties1.Value.ETag, properties2.Value.ETag); + EXPECT_EQ(0ll, properties2.Value.FileSize); + } + } + TEST_F(DataLakeFileClientTest, FileReadReturns) { const int32_t bufferSize = 4 * 1024; // 4KB data size diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index d299436ebc..2cbf43567a 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -290,6 +290,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(result.end(), iter); EXPECT_EQ(iter->Name, name); EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); + EXPECT_TRUE(iter->CreatedOn.HasValue()); + EXPECT_FALSE(iter->ExpiresOn.HasValue()); } for (const auto& name : m_pathNameSetB) { @@ -300,6 +302,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(result.end(), iter); EXPECT_EQ(iter->Name, name); EXPECT_EQ(iter->Name.substr(0U, m_directoryB.size()), m_directoryB); + EXPECT_TRUE(iter->CreatedOn.HasValue()); + EXPECT_FALSE(iter->ExpiresOn.HasValue()); } } { @@ -331,6 +335,23 @@ namespace Azure { namespace Storage { namespace Test { auto response = m_fileSystemClient->ListPaths(true, options); EXPECT_LE(2U, response.Paths.size()); } + { + // check expiry time + const std::string filename = GetTestNameLowerCase() + "check_expiry"; + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "check_expiry"); + Files::DataLake::CreateFileOptions createOptions; + createOptions.ScheduleDeletionOptions.ExpiresOn = Azure::DateTime::Parse( + "Wed, 29 Sep 2100 09:53:03 GMT", Azure::DateTime::DateFormat::Rfc1123); + client.Create(createOptions); + + auto result = ListAllPaths(false); + auto iter = std::find_if( + result.begin(), result.end(), [&filename](const Files::DataLake::Models::PathItem& path) { + return path.Name == filename; + }); + EXPECT_TRUE(iter->ExpiresOn.HasValue()); + EXPECT_EQ(createOptions.ScheduleDeletionOptions.ExpiresOn.Value(), iter->ExpiresOn.Value()); + } } TEST_F(DataLakeFileSystemClientTest, UnencodedPathDirectoryFileNameWorks) diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp index fd434af7c1..efdb41e9e0 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_path_client_test.cpp @@ -57,6 +57,87 @@ namespace Azure { namespace Storage { namespace Test { return result; } + TEST_F(DataLakePathClientTest, CreateWithOptions) + { + // owner&group + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "owner_group"); + Files::DataLake::CreateFileOptions options; + options.Group = "$superuser"; + options.Owner = "$superuser"; + EXPECT_NO_THROW(client.Create(options)); + auto response = client.GetAccessControlList(); + EXPECT_EQ(options.Group.Value(), response.Value.Group); + EXPECT_EQ(options.Owner.Value(), response.Value.Owner); + } + // acl + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_acl"); + Files::DataLake::CreateFileOptions options; + auto acls = GetValidAcls(); + options.Acls = acls; + EXPECT_NO_THROW(client.Create(options)); + auto resultAcls = client.GetAccessControlList().Value.Acls; + EXPECT_EQ(resultAcls.size(), acls.size() + 1); // Always append mask::rwx + for (const auto& acl : acls) + { + auto iter = std::find_if( + resultAcls.begin(), + resultAcls.end(), + [&acl](const Files::DataLake::Models::Acl& targetAcl) { + return (targetAcl.Type == acl.Type) && (targetAcl.Id == acl.Id) + && (targetAcl.Scope == acl.Scope); + }); + EXPECT_NE(iter, resultAcls.end()); + EXPECT_EQ(iter->Permissions, acl.Permissions); + } + } + // lease + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_lease"); + Files::DataLake::CreateFileOptions options; + options.LeaseId = Files::DataLake::DataLakeLeaseClient::CreateUniqueLeaseId(); + options.LeaseDuration = std::chrono::seconds(20); + EXPECT_NO_THROW(client.Create(options)); + auto response = client.GetProperties(); + EXPECT_TRUE(response.Value.LeaseStatus.HasValue()); + EXPECT_EQ(Files::DataLake::Models::LeaseStatus::Locked, response.Value.LeaseStatus.Value()); + EXPECT_TRUE(response.Value.LeaseState.HasValue()); + EXPECT_EQ(Files::DataLake::Models::LeaseState::Leased, response.Value.LeaseState.Value()); + EXPECT_TRUE(response.Value.LeaseDuration.HasValue()); + EXPECT_EQ( + Files::DataLake::Models::LeaseDurationType::Fixed, response.Value.LeaseDuration.Value()); + } + // relative expiry + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_relative_expiry"); + Files::DataLake::CreateFileOptions options; + options.ScheduleDeletionOptions.TimeToExpire = std::chrono::hours(1); + EXPECT_NO_THROW(client.Create(options)); + auto response = client.GetProperties(); + DateTime leftExpiryTime = response.Value.CreatedOn + + options.ScheduleDeletionOptions.TimeToExpire.Value() - std::chrono::seconds(5); + DateTime rightExpiryTime = response.Value.CreatedOn + + options.ScheduleDeletionOptions.TimeToExpire.Value() + std::chrono::seconds(5); + EXPECT_TRUE(response.Value.ExpiresOn.HasValue()); + EXPECT_TRUE( + response.Value.ExpiresOn.Value() > leftExpiryTime + && response.Value.ExpiresOn.Value() < rightExpiryTime); + } + // absolute expiry + { + auto client = m_fileSystemClient->GetFileClient(GetTestNameLowerCase() + "_absolute_expiry"); + Files::DataLake::CreateFileOptions options; + options.ScheduleDeletionOptions.ExpiresOn = Azure::DateTime::Parse( + "Wed, 29 Sep 2100 09:53:03 GMT", Azure::DateTime::DateFormat::Rfc1123); + EXPECT_NO_THROW(client.Create(options)); + auto response = client.GetProperties(); + EXPECT_TRUE(response.Value.ExpiresOn.HasValue()); + EXPECT_EQ( + options.ScheduleDeletionOptions.ExpiresOn.Value(), response.Value.ExpiresOn.Value()); + } + } + TEST_F(DataLakePathClientTest, PathMetadata) { auto metadata1 = GetMetadata(); diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileClientTest.AppendFileWithFlush.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileClientTest.AppendFileWithFlush.json new file mode 100644 index 0000000000..d002712baa --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileClientTest.AppendFileWithFlush.json @@ -0,0 +1,321 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "589a9f12-0965-4803-646e-270620ce1db2", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:44 GMT", + "etag": "\"0x8DA9D1F08D9DF38\"", + "last-modified": "Fri, 23 Sep 2022 04:49:44 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "589a9f12-0965-4803-646e-270620ce1db2", + "x-ms-request-id": "82088cba-b01e-0077-1107-cf073b000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "3e908b63-8bfa-4038-6608-e9c7bffacdb7", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:45 GMT", + "etag": "\"0x8DA9D1F09AC46D3\"", + "last-modified": "Fri, 23 Sep 2022 04:49:45 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3e908b63-8bfa-4038-6608-e9c7bffacdb7", + "x-ms-request-id": "1306a2f5-101f-008a-4c07-cf891e000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/datalakefileclienttestappendfilewithflush?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "e61cd103-cc06-4583-62aa-5fc6489f6b59", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:46 GMT", + "etag": "\"0x8DA9D1F09DCBE9A\"", + "last-modified": "Fri, 23 Sep 2022 04:49:46 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "e61cd103-cc06-4583-62aa-5fc6489f6b59", + "x-ms-request-id": "1306a34a-101f-008a-2007-cf891e000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_true?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "15780334-3cf6-442d-5482-655fe1c6a124", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Fri, 23 Sep 2022 04:49:45 GMT", + "etag": "\"0x8DA9D1F09DCBE9A\"", + "last-modified": "Fri, 23 Sep 2022 04:49:46 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "15780334-3cf6-442d-5482-655fe1c6a124", + "x-ms-creation-time": "Fri, 23 Sep 2022 04:49:46 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "82089521-b01e-0077-3b07-cf073b000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_true" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "4f31d3bb-e6d5-4228-407b-edd51ad82213", + "x-ms-version": "2021-06-08" + }, + "Method": "PATCH", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:46 GMT", + "etag": "\"0x8DA9D1F0A3AE267\"", + "last-modified": "Fri, 23 Sep 2022 04:49:46 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "4f31d3bb-e6d5-4228-407b-edd51ad82213", + "x-ms-request-id": "1306a3a5-101f-008a-7b07-cf891e000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_true?action=append&flush=true&position=0" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "bbc64830-132f-4f9b-7d3e-1f3bedb0e340", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "4096", + "content-type": "application/octet-stream", + "date": "Fri, 23 Sep 2022 04:49:46 GMT", + "etag": "\"0x8DA9D1F0A3AE267\"", + "last-modified": "Fri, 23 Sep 2022 04:49:46 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "bbc64830-132f-4f9b-7d3e-1f3bedb0e340", + "x-ms-creation-time": "Fri, 23 Sep 2022 04:49:46 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "820896e4-b01e-0077-6607-cf073b000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_true" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "3be6d8cf-026b-4e1c-7201-c48d16b91314", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:47 GMT", + "etag": "\"0x8DA9D1F0A97A92A\"", + "last-modified": "Fri, 23 Sep 2022 04:49:47 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3be6d8cf-026b-4e1c-7201-c48d16b91314", + "x-ms-request-id": "1306a3d9-101f-008a-2f07-cf891e000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_false?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "0c923618-0455-4b10-625f-9bf0496e952d", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Fri, 23 Sep 2022 04:49:47 GMT", + "etag": "\"0x8DA9D1F0A97A92A\"", + "last-modified": "Fri, 23 Sep 2022 04:49:47 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "0c923618-0455-4b10-625f-9bf0496e952d", + "x-ms-creation-time": "Fri, 23 Sep 2022 04:49:47 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "8208990c-b01e-0077-6307-cf073b000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_false" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "feaa3dc2-ef0f-45b2-7f22-32491891657f", + "x-ms-version": "2021-06-08" + }, + "Method": "PATCH", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:47 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "feaa3dc2-ef0f-45b2-7f22-32491891657f", + "x-ms-request-id": "1306a41b-101f-008a-6e07-cf891e000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_false?action=append&flush=false&position=0" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "4bb822e2-f6d1-496f-582b-682a1c30b14a", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Fri, 23 Sep 2022 04:49:47 GMT", + "etag": "\"0x8DA9D1F0A97A92A\"", + "last-modified": "Fri, 23 Sep 2022 04:49:47 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "4bb822e2-f6d1-496f-582b-682a1c30b14a", + "x-ms-creation-time": "Fri, 23 Sep 2022 04:49:47 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "82089b11-b01e-0077-4e07-cf073b000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush/appendfilewithflush_flush_false" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "fb2ca437-05e9-4d64-660a-a843060785e0", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:48 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "fb2ca437-05e9-4d64-660a-a843060785e0", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133083821885911938", + "x-ms-request-id": "1306a44f-101f-008a-2207-cf891e000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefileclienttestappendfilewithflush/datalakefileclienttestappendfilewithflush" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "73126a8f-adad-40d5-69f6-6bc16adf36af", + "x-ms-version": "2021-04-10" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Fri, 23 Sep 2022 04:49:48 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "73126a8f-adad-40d5-69f6-6bc16adf36af", + "x-ms-request-id": "82089dfb-b01e-0077-1007-cf073b000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefileclienttestappendfilewithflush?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListPaths.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListPaths.json index 7148243f33..c9a66a8140 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListPaths.json +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListPaths.json @@ -2,9 +2,9 @@ "networkCallRecords": [ { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "8f08d654-ff1e-4ddd-442d-664d7041f178", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6de26491-c99e-4400-52f6-83e5a7769efd", + "x-ms-version": "2021-04-10" }, "Method": "PUT", "Response": { @@ -12,21 +12,21 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:43 GMT", - "etag": "\"0x8D9EC9127C0EEAB\"", - "last-modified": "Thu, 10 Feb 2022 12:30:43 GMT", + "date": "Fri, 23 Sep 2022 05:04:04 GMT", + "etag": "\"0x8DA9D2109B6B4D5\"", + "last-modified": "Fri, 23 Sep 2022 05:04:04 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "8f08d654-ff1e-4ddd-442d-664d7041f178", - "x-ms-request-id": "b2045056-d01e-0044-327a-1e1884000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "6de26491-c99e-4400-52f6-83e5a7769efd", + "x-ms-request-id": "167a5741-d01e-0085-4309-cfff72000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistpaths?restype=container" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "894906f1-b0ec-4eb3-790f-e26a5d88e144", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6154d696-cf93-4d9b-54db-9f3ca23732fb", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -34,22 +34,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:43 GMT", - "etag": "\"0x8D9EC91287C7FD3\"", - "last-modified": "Thu, 10 Feb 2022 12:30:44 GMT", + "date": "Fri, 23 Sep 2022 05:04:05 GMT", + "etag": "\"0x8DA9D210A80C6F7\"", + "last-modified": "Fri, 23 Sep 2022 05:04:06 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "894906f1-b0ec-4eb3-790f-e26a5d88e144", - "x-ms-request-id": "e84af67f-d01f-004f-4c7a-1e00f0000000", + "x-ms-client-request-id": "6154d696-cf93-4d9b-54db-9f3ca23732fb", + "x-ms-request-id": "959586e8-901f-0070-5009-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpata/0?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "07971491-f58a-43cc-4327-0d5ea89be5ca", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6daa4718-03f6-4081-4d01-1dc9f08b1c73", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -57,22 +57,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:43 GMT", - "etag": "\"0x8D9EC9128A6F7D9\"", - "last-modified": "Thu, 10 Feb 2022 12:30:44 GMT", + "date": "Fri, 23 Sep 2022 05:04:05 GMT", + "etag": "\"0x8DA9D210AB3EC4B\"", + "last-modified": "Fri, 23 Sep 2022 05:04:06 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "07971491-f58a-43cc-4327-0d5ea89be5ca", - "x-ms-request-id": "e84af680-d01f-004f-4d7a-1e00f0000000", + "x-ms-client-request-id": "6daa4718-03f6-4081-4d01-1dc9f08b1c73", + "x-ms-request-id": "9595873b-901f-0070-2109-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpatb/0?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "29ad9060-ef81-4ceb-69db-e4841d557801", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "16d6a7ee-680f-467f-6f8e-549adfc3306b", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -80,22 +80,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:44 GMT", - "etag": "\"0x8D9EC9128CD98A4\"", - "last-modified": "Thu, 10 Feb 2022 12:30:44 GMT", + "date": "Fri, 23 Sep 2022 05:04:05 GMT", + "etag": "\"0x8DA9D210AE31D0D\"", + "last-modified": "Fri, 23 Sep 2022 05:04:06 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "29ad9060-ef81-4ceb-69db-e4841d557801", - "x-ms-request-id": "e84af681-d01f-004f-4e7a-1e00f0000000", + "x-ms-client-request-id": "16d6a7ee-680f-467f-6f8e-549adfc3306b", + "x-ms-request-id": "9595876e-901f-0070-5409-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpata/1?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "cc31b893-142d-4d02-4948-522e34578dd9", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "9d468ae7-c6c1-496d-4e07-08259994e452", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -103,22 +103,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:44 GMT", - "etag": "\"0x8D9EC9128F5FE1B\"", - "last-modified": "Thu, 10 Feb 2022 12:30:45 GMT", + "date": "Fri, 23 Sep 2022 05:04:06 GMT", + "etag": "\"0x8DA9D210B1149EB\"", + "last-modified": "Fri, 23 Sep 2022 05:04:07 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "cc31b893-142d-4d02-4948-522e34578dd9", - "x-ms-request-id": "e84af682-d01f-004f-4f7a-1e00f0000000", + "x-ms-client-request-id": "9d468ae7-c6c1-496d-4e07-08259994e452", + "x-ms-request-id": "95958797-901f-0070-7d09-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpatb/1?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "396fe3bc-c524-4a71-52e3-af3eb0e52156", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5689db32-34d4-430c-584f-5c2720b4b946", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -126,22 +126,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:44 GMT", - "etag": "\"0x8D9EC91291D2661\"", - "last-modified": "Thu, 10 Feb 2022 12:30:45 GMT", + "date": "Fri, 23 Sep 2022 05:04:06 GMT", + "etag": "\"0x8DA9D210B3F8FB2\"", + "last-modified": "Fri, 23 Sep 2022 05:04:07 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "396fe3bc-c524-4a71-52e3-af3eb0e52156", - "x-ms-request-id": "e84af685-d01f-004f-507a-1e00f0000000", + "x-ms-client-request-id": "5689db32-34d4-430c-584f-5c2720b4b946", + "x-ms-request-id": "959587cc-901f-0070-3109-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpata/2?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "b0141a42-0b54-49c5-6f0e-a383ebc4c53b", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "10386e25-e34e-43d9-442b-75acee9d11cc", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -149,22 +149,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:44 GMT", - "etag": "\"0x8D9EC912944F0EB\"", - "last-modified": "Thu, 10 Feb 2022 12:30:45 GMT", + "date": "Fri, 23 Sep 2022 05:04:06 GMT", + "etag": "\"0x8DA9D210B6DF840\"", + "last-modified": "Fri, 23 Sep 2022 05:04:07 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "b0141a42-0b54-49c5-6f0e-a383ebc4c53b", - "x-ms-request-id": "e84af686-d01f-004f-517a-1e00f0000000", + "x-ms-client-request-id": "10386e25-e34e-43d9-442b-75acee9d11cc", + "x-ms-request-id": "95958802-901f-0070-6709-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpatb/2?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "d452f0c5-674d-4c59-763d-91c1913e9779", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "04e95cf9-dbc2-4d96-6234-89d9515458ca", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -172,22 +172,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:45 GMT", - "etag": "\"0x8D9EC91296C18D8\"", - "last-modified": "Thu, 10 Feb 2022 12:30:45 GMT", + "date": "Fri, 23 Sep 2022 05:04:07 GMT", + "etag": "\"0x8DA9D210B9C6B59\"", + "last-modified": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "d452f0c5-674d-4c59-763d-91c1913e9779", - "x-ms-request-id": "e84af687-d01f-004f-527a-1e00f0000000", + "x-ms-client-request-id": "04e95cf9-dbc2-4d96-6234-89d9515458ca", + "x-ms-request-id": "9595882e-901f-0070-1309-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpata/3?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "aa41924d-4b0f-4fc4-4087-5fa5949d765b", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5fc7addd-af4d-408b-48b0-2a77365da9fd", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -195,22 +195,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:45 GMT", - "etag": "\"0x8D9EC9129925C55\"", - "last-modified": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:07 GMT", + "etag": "\"0x8DA9D210BCAF016\"", + "last-modified": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "aa41924d-4b0f-4fc4-4087-5fa5949d765b", - "x-ms-request-id": "e84af688-d01f-004f-537a-1e00f0000000", + "x-ms-client-request-id": "5fc7addd-af4d-408b-48b0-2a77365da9fd", + "x-ms-request-id": "95958867-901f-0070-4c09-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpatb/3?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "79e178cc-8486-4158-6a8f-e918d55b6e0a", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "90b9c96c-ac13-4f96-67ae-17be2b2adc3e", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -218,22 +218,22 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:45 GMT", - "etag": "\"0x8D9EC9129B81E52\"", - "last-modified": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:07 GMT", + "etag": "\"0x8DA9D210BF9C7D5\"", + "last-modified": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "79e178cc-8486-4158-6a8f-e918d55b6e0a", - "x-ms-request-id": "e84af689-d01f-004f-547a-1e00f0000000", + "x-ms-client-request-id": "90b9c96c-ac13-4f96-67ae-17be2b2adc3e", + "x-ms-request-id": "9595889e-901f-0070-0309-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpata/4?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "772483af-f632-46d7-416e-05dc3a83985e", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "ce57591b-39c8-443b-6eef-bc459b75d192", + "x-ms-version": "2021-06-08" }, "Method": "PUT", "Response": { @@ -241,86 +241,130 @@ "REASON_PHRASE": "Created", "STATUS_CODE": "201", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:45 GMT", - "etag": "\"0x8D9EC9129DFAE38\"", - "last-modified": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:08 GMT", + "etag": "\"0x8DA9D210C28D121\"", + "last-modified": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "772483af-f632-46d7-416e-05dc3a83985e", - "x-ms-request-id": "e84af68a-d01f-004f-557a-1e00f0000000", + "x-ms-client-request-id": "ce57591b-39c8-443b-6eef-bc459b75d192", + "x-ms-request-id": "959588d6-901f-0070-3b09-cf6b58000000", "x-ms-request-server-encrypted": "true", - "x-ms-version": "2020-02-10" + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/datalakefilesystemclienttestlistpatb/4?resource=file" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "67f00dbb-6eb6-4fc7-4ea1-0c0a700a2b64", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5288db61-9c12-4d3a-6f0e-c805e391af33", + "x-ms-version": "2021-06-08" }, "Method": "GET", "Response": { - "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"132889698443120578\",\"etag\":\"0x8D9EC912879D279\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698443296723\",\"etag\":\"0x8D9EC91287C7FD3\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698448611492\",\"etag\":\"0x8D9EC9128CD98A4\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698453825121\",\"etag\":\"0x8D9EC91291D2661\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698458999000\",\"etag\":\"0x8D9EC91296C18D8\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698463981138\",\"etag\":\"0x8D9EC9129B81E52\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:46 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698445865410\",\"etag\":\"0x8D9EC9128A3B3C1\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698446079961\",\"etag\":\"0x8D9EC9128A6F7D9\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698451258907\",\"etag\":\"0x8D9EC9128F5FE1B\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698456432875\",\"etag\":\"0x8D9EC912944F0EB\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698461506645\",\"etag\":\"0x8D9EC9129925C55\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:46 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698466573880\",\"etag\":\"0x8D9EC9129DFAE38\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:46 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", + "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"133083830461752560\",\"etag\":\"0x8DA9D210A7C66C1\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830462039799\",\"etag\":\"0x8DA9D210A80C6F7\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830468484365\",\"etag\":\"0x8DA9D210AE31D0D\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830474543026\",\"etag\":\"0x8DA9D210B3F8FB2\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:07 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830480628569\",\"etag\":\"0x8DA9D210B9C6B59\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830486747093\",\"etag\":\"0x8DA9D210BF9C7D5\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830465204460\",\"etag\":\"0x8DA9D210AB110ED\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830465391691\",\"etag\":\"0x8DA9D210AB3EC4B\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830471510507\",\"etag\":\"0x8DA9D210B1149EB\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:07 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830477584448\",\"etag\":\"0x8DA9D210B6DF840\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:07 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830483677206\",\"etag\":\"0x8DA9D210BCAF016\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830489829665\",\"etag\":\"0x8DA9D210C28D121\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/json;charset=utf-8", - "date": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "67f00dbb-6eb6-4fc7-4ea1-0c0a700a2b64", - "x-ms-request-id": "e84af68b-d01f-004f-567a-1e00f0000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "5288db61-9c12-4d3a-6f0e-c805e391af33", + "x-ms-request-id": "95958903-901f-0070-6809-cf6b58000000", + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths?recursive=true&resource=filesystem" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "f14b09c8-57ac-423e-6c51-d7755f4d6e60", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "cbffe896-0fd4-42fe-4d48-61767cb1ae2c", + "x-ms-version": "2021-06-08" }, "Method": "GET", "Response": { - "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"132889698443296723\",\"etag\":\"0x8D9EC91287C7FD3\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698448611492\",\"etag\":\"0x8D9EC9128CD98A4\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698453825121\",\"etag\":\"0x8D9EC91291D2661\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698458999000\",\"etag\":\"0x8D9EC91296C18D8\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:45 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698463981138\",\"etag\":\"0x8D9EC9129B81E52\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:46 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", + "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"133083830462039799\",\"etag\":\"0x8DA9D210A80C6F7\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830468484365\",\"etag\":\"0x8DA9D210AE31D0D\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/1\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830474543026\",\"etag\":\"0x8DA9D210B3F8FB2\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:07 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/2\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830480628569\",\"etag\":\"0x8DA9D210B9C6B59\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/3\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830486747093\",\"etag\":\"0x8DA9D210BF9C7D5\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:08 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/4\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/json;charset=utf-8", - "date": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:08 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "f14b09c8-57ac-423e-6c51-d7755f4d6e60", - "x-ms-request-id": "e84af68c-d01f-004f-577a-1e00f0000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "cbffe896-0fd4-42fe-4d48-61767cb1ae2c", + "x-ms-request-id": "95958928-901f-0070-0d09-cf6b58000000", + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths?directory=datalakefilesystemclienttestlistpata&recursive=true&resource=filesystem" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-files-datalake/12.3.0-beta.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "78793360-1ec9-408b-41ec-981f22bc060e", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f5be34a1-afd9-4ad9-4576-bcd875fe5daa", + "x-ms-version": "2021-06-08" }, "Method": "GET", "Response": { - "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"132889698443120578\",\"etag\":\"0x8D9EC912879D279\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"132889698443296723\",\"etag\":\"0x8D9EC91287C7FD3\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Thu, 10 Feb 2022 12:30:44 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", + "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"133083830461752560\",\"etag\":\"0x8DA9D210A7C66C1\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830462039799\",\"etag\":\"0x8DA9D210A80C6F7\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata/0\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", "REASON_PHRASE": "OK", "STATUS_CODE": "200", "content-type": "application/json;charset=utf-8", - "date": "Thu, 10 Feb 2022 12:30:46 GMT", + "date": "Fri, 23 Sep 2022 05:04:09 GMT", "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", "transfer-encoding": "chunked", - "x-ms-client-request-id": "78793360-1ec9-408b-41ec-981f22bc060e", - "x-ms-continuation": "VBap0Y+oqtbCh+8BGH8Yei9hZGxzZ2VuMnV0ATAxRDU5NDU4NDQ4MTM5NzAvZGF0YWxha2VmaWxlc3lzdGVtY2xpZW50dGVzdGxpc3RwYXRocwEwMUQ4MUU3QTA1NEEwQzc0L2RhdGFsYWtlZmlsZXN5c3RlbWNsaWVudHRlc3RsaXN0cGF0YS8xFgAAAA==", - "x-ms-request-id": "e84af68d-d01f-004f-587a-1e00f0000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "f5be34a1-afd9-4ad9-4576-bcd875fe5daa", + "x-ms-continuation": "VBb46dPpnZTanOsBGIMBGH4vemNoYW5nZGF0YWxha2UBMDFEOEIwODM1QUYzNUY2My9kYXRhbGFrZWZpbGVzeXN0ZW1jbGllbnR0ZXN0bGlzdHBhdGhzATAxRDhDRjA5RTc0MEFFNDEvZGF0YWxha2VmaWxlc3lzdGVtY2xpZW50dGVzdGxpc3RwYXRhLzEWAAAA", + "x-ms-request-id": "95958955-901f-0070-3a09-cf6b58000000", + "x-ms-version": "2021-06-08" }, "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths?maxResults=2&recursive=true&resource=filesystem" }, { "Headers": { - "user-agent": "azsdk-cpp-storage-blobs/12.2.1 (Windows 10 Pro 6.3 19041 19041.1.amd64fre.vb_release.191206-1406)", - "x-ms-client-request-id": "db058798-a0d5-444e-6664-6eb6c21f71ea", - "x-ms-version": "2020-02-10" + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "728f981e-3bcc-485d-54b2-5d06bb4832db", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Fri, 23 Sep 2022 05:04:09 GMT", + "etag": "\"0x8DA9D210CE3AF64\"", + "last-modified": "Fri, 23 Sep 2022 05:04:10 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "728f981e-3bcc-485d-54b2-5d06bb4832db", + "x-ms-request-id": "9595897d-901f-0070-6209-cf6b58000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths/listpathscheck_expiry?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "b751c4eb-73d2-49ae-6f5d-a8286181c316", + "x-ms-version": "2021-06-08" + }, + "Method": "GET", + "Response": { + "BODY": "{\"paths\":[{\"contentLength\":\"0\",\"creationTime\":\"133083830461752560\",\"etag\":\"0x8DA9D210A7C66C1\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpata\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830465204460\",\"etag\":\"0x8DA9D210AB110ED\",\"expiryTime\":\"0\",\"group\":\"$superuser\",\"isDirectory\":\"true\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:06 GMT\",\"name\":\"datalakefilesystemclienttestlistpatb\",\"owner\":\"$superuser\",\"permissions\":\"rwxr-x---\"},{\"contentLength\":\"0\",\"creationTime\":\"133083830502076260\",\"etag\":\"0x8DA9D210CE3AF64\",\"expiryTime\":\"157703683830000000\",\"group\":\"$superuser\",\"lastModified\":\"Fri, 23 Sep 2022 05:04:10 GMT\",\"name\":\"listpathscheck_expiry\",\"owner\":\"$superuser\",\"permissions\":\"rw-r-----\"}]}\n", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/json;charset=utf-8", + "date": "Fri, 23 Sep 2022 05:04:09 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "b751c4eb-73d2-49ae-6f5d-a8286181c316", + "x-ms-request-id": "959589a9-901f-0070-0e09-cf6b58000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistpaths?recursive=false&resource=filesystem" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "8ad5c2bc-6aa8-4434-43ef-b4526c2270aa", + "x-ms-version": "2021-04-10" }, "Method": "DELETE", "Response": { @@ -328,11 +372,11 @@ "REASON_PHRASE": "Accepted", "STATUS_CODE": "202", "content-length": "0", - "date": "Thu, 10 Feb 2022 12:30:47 GMT", + "date": "Fri, 23 Sep 2022 05:04:09 GMT", "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", - "x-ms-client-request-id": "db058798-a0d5-444e-6664-6eb6c21f71ea", - "x-ms-request-id": "b2045ad2-d01e-0044-4d7a-1e1884000000", - "x-ms-version": "2020-02-10" + "x-ms-client-request-id": "8ad5c2bc-6aa8-4434-43ef-b4526c2270aa", + "x-ms-request-id": "167a652a-d01e-0085-0109-cfff72000000", + "x-ms-version": "2021-04-10" }, "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistpaths?restype=container" } diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakePathClientTest.CreateWithOptions.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakePathClientTest.CreateWithOptions.json new file mode 100644 index 0000000000..6f2c3eb842 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakePathClientTest.CreateWithOptions.json @@ -0,0 +1,364 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "e3fe567c-0f47-4060-60f8-ed314816a807", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:21 GMT", + "etag": "\"0x8DA9C7298ACCD16\"", + "last-modified": "Thu, 22 Sep 2022 08:15:22 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "e3fe567c-0f47-4060-60f8-ed314816a807", + "x-ms-request-id": "1b7c2128-801e-0098-7f5b-cef2ce000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakepathclienttestcreatewithoptions?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "06f95515-d0dd-4600-5f1a-deefe5600fd1", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:24 GMT", + "etag": "\"0x8DA9C7299841A56\"", + "last-modified": "Thu, 22 Sep 2022 08:15:24 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "06f95515-d0dd-4600-5f1a-deefe5600fd1", + "x-ms-request-id": "1b286777-f01f-003b-705b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/datalakepathclienttestcreatewithoptions?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "819b4146-7e5a-4444-63cc-8821a68f324a", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:24 GMT", + "etag": "\"0x8DA9C7299B43EFD\"", + "last-modified": "Thu, 22 Sep 2022 08:15:24 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "819b4146-7e5a-4444-63cc-8821a68f324a", + "x-ms-request-id": "1b2867c3-f01f-003b-3c5b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptionsowner_group?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a92a43db-ac5d-4ed4-5c59-57eff2e80175", + "x-ms-version": "2021-06-08" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "date": "Thu, 22 Sep 2022 08:15:24 GMT", + "etag": "\"0x8DA9C7299B43EFD\"", + "last-modified": "Thu, 22 Sep 2022 08:15:24 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-acl": "user::rw-,group::r--,other::---", + "x-ms-client-request-id": "a92a43db-ac5d-4ed4-5c59-57eff2e80175", + "x-ms-group": "$superuser", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "1b2867f0-f01f-003b-685b-ce970b000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptionsowner_group?action=getAccessControl" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1175eaa4-02db-48fa-622f-54a4af42eaff", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:25 GMT", + "etag": "\"0x8DA9C729A128834\"", + "last-modified": "Thu, 22 Sep 2022 08:15:25 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "1175eaa4-02db-48fa-622f-54a4af42eaff", + "x-ms-request-id": "1b286825-f01f-003b-1d5b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_acl?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "4c3e31ae-4d33-4257-732d-7853423d7a11", + "x-ms-version": "2021-06-08" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "date": "Thu, 22 Sep 2022 08:15:25 GMT", + "etag": "\"0x8DA9C729A128834\"", + "last-modified": "Thu, 22 Sep 2022 08:15:25 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-acl": "user::rwx,user:72a3f86f-271f-439e-b031-25678907d381:rwx,group::r--,mask::rwx,other::---", + "x-ms-client-request-id": "4c3e31ae-4d33-4257-732d-7853423d7a11", + "x-ms-group": "$superuser", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rwxrwx---+", + "x-ms-request-id": "1b28684f-f01f-003b-455b-ce970b000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_acl?action=getAccessControl" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "340169e3-a8c6-4f26-5c83-a59959fd1668", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:25 GMT", + "etag": "\"0x8DA9C729A7003AB\"", + "last-modified": "Thu, 22 Sep 2022 08:15:25 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "340169e3-a8c6-4f26-5c83-a59959fd1668", + "x-ms-request-id": "1b28688d-f01f-003b-765b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_lease?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a2b37160-8f93-4f8b-75a3-e87d599f420b", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Thu, 22 Sep 2022 08:15:25 GMT", + "etag": "\"0x8DA9C729A7003AB\"", + "last-modified": "Thu, 22 Sep 2022 08:15:25 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "a2b37160-8f93-4f8b-75a3-e87d599f420b", + "x-ms-creation-time": "Thu, 22 Sep 2022 08:15:25 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-duration": "fixed", + "x-ms-lease-state": "leased", + "x-ms-lease-status": "locked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "1b7c29b3-801e-0098-4c5b-cef2ce000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_lease" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "ef662d54-a94f-460c-6b31-ca23dfcc1c2c", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:26 GMT", + "etag": "\"0x8DA9C729ACE097A\"", + "last-modified": "Thu, 22 Sep 2022 08:15:26 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "ef662d54-a94f-460c-6b31-ca23dfcc1c2c", + "x-ms-request-id": "1b2868d7-f01f-003b-405b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_relative_expiry?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "506f83b9-2b74-4a4a-42a2-fc2aa7c3b476", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Thu, 22 Sep 2022 08:15:25 GMT", + "etag": "\"0x8DA9C729ACE097A\"", + "last-modified": "Thu, 22 Sep 2022 08:15:26 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "506f83b9-2b74-4a4a-42a2-fc2aa7c3b476", + "x-ms-creation-time": "Thu, 22 Sep 2022 08:15:26 GMT", + "x-ms-expiry-time": "Thu, 22 Sep 2022 09:15:26 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "1b7c2bb7-801e-0098-285b-cef2ce000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_relative_expiry" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "7c3cfe20-69f0-4b10-6b5a-f1bd0c8d3fba", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:26 GMT", + "etag": "\"0x8DA9C729B2A21E1\"", + "last-modified": "Thu, 22 Sep 2022 08:15:26 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "7c3cfe20-69f0-4b10-6b5a-f1bd0c8d3fba", + "x-ms-request-id": "1b28691f-f01f-003b-085b-ce970b000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_absolute_expiry?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "0002140c-80d8-403c-57d9-5ddab7c8ef3b", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Thu, 22 Sep 2022 08:15:26 GMT", + "etag": "\"0x8DA9C729B2A21E1\"", + "last-modified": "Thu, 22 Sep 2022 08:15:26 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "0002140c-80d8-403c-57d9-5ddab7c8ef3b", + "x-ms-creation-time": "Thu, 22 Sep 2022 08:15:26 GMT", + "x-ms-expiry-time": "Wed, 29 Sep 2100 09:53:03 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "1b7c2e26-801e-0098-6d5b-cef2ce000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakepathclienttestcreatewithoptions/createwithoptions_absolute_expiry" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "e8a2a362-5cf5-4d44-478c-7767eed9625e", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:27 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "e8a2a362-5cf5-4d44-478c-7767eed9625e", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133083081276164640", + "x-ms-request-id": "1b28696c-f01f-003b-545b-ce970b000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakepathclienttestcreatewithoptions/datalakepathclienttestcreatewithoptions" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "0f3d271c-2eb5-4ae9-5892-1823d52a5c2e", + "x-ms-version": "2021-04-10" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Thu, 22 Sep 2022 08:15:27 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "0f3d271c-2eb5-4ae9-5892-1823d52a5c2e", + "x-ms-request-id": "1b7c3069-801e-0098-1a5b-cef2ce000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakepathclienttestcreatewithoptions?restype=container" + } + ] +} From c277d59325137fbe15ae82e43d67d90c2762ab2d Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Wed, 28 Sep 2022 14:27:27 +0800 Subject: [PATCH 08/11] Change win32converter (#3968) --- eng/common/testproxy/target_version.txt | 2 +- eng/pipelines/templates/jobs/perf.yml | 44 +++++++++++++ eng/pipelines/templates/variables/globals.yml | 22 +++++++ sdk/core/perf.yml | 32 ++++++++++ .../test/ut/blob_query_test.cpp | 2 +- .../files/datalake/datalake_responses.hpp | 61 +++++++++++++++++++ .../storage/files/datalake/rest_client.hpp | 34 +++++------ .../src/datalake_directory_client.cpp | 27 +++++++- .../src/datalake_file_system_client.cpp | 28 ++++++--- .../src/private/datalake_utilities.hpp | 45 ++++++++++++++ .../src/rest_client.cpp | 10 +-- .../swagger/README.md | 5 +- .../ut/datalake_file_system_client_test.cpp | 2 + 13 files changed, 277 insertions(+), 37 deletions(-) create mode 100644 eng/pipelines/templates/jobs/perf.yml create mode 100644 eng/pipelines/templates/variables/globals.yml create mode 100644 sdk/core/perf.yml diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index aef19a2487..dedc530f84 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220921.2 +1.0.0-dev.20220922.1 diff --git a/eng/pipelines/templates/jobs/perf.yml b/eng/pipelines/templates/jobs/perf.yml new file mode 100644 index 0000000000..4e1d328449 --- /dev/null +++ b/eng/pipelines/templates/jobs/perf.yml @@ -0,0 +1,44 @@ +parameters: +- name: ServiceDirectory + type: string + default: '' +- name: Services + type: string + default: '' +- name: PackageVersions + type: string + default: '.*' +- name: Tests + type: string + default: '.*' +- name: Arguments + type: string + default: '.*' +- name: Iterations + type: number + default: '5' +- name: AdditionalArguments + type: string + default: '' +- name: EnvVars + type: object + default: [] + +extends: + template: /eng/common/pipelines/templates/jobs/perf.yml + parameters: + Variables: + - template: /eng/pipelines/templates/variables/globals.yml + Language: Cpp + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Services: ${{ parameters.Services }} + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} + EnvVars: ${{ parameters.EnvVars}} + InstallLanguageSteps: + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: Microsoft diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml new file mode 100644 index 0000000000..b782b88239 --- /dev/null +++ b/eng/pipelines/templates/variables/globals.yml @@ -0,0 +1,22 @@ +variables: + # True if 'Enable system diagnostics' is checked when running a pipeline manually + IsDebug: $[coalesce(variables['System.Debug'], 'false')] + + AdditionalOptions: '' + + # Exists if needed in coalesce situations. + DefaultTestGoals: 'surefire:test' + # This will be overwritten by the test matrix, if configured. + TestGoals: $(DefaultTestGoals) + + # This will be overwritten by the test matrix, if configured. + TestOptions: '' + # TestFromSource is one of the cache keys but isn't set until the test matrix + # has been processed. Without a default value it'll be treated as a string literal + # "$(TestFromSource)" instead of true/false. It'll be overwritten when the test + # matrix has been processed + TestFromSource: false + + skipComponentGovernanceDetection: true + DisableDockerDetector: true + Package.EnableSBOMSigning: true diff --git a/sdk/core/perf.yml b/sdk/core/perf.yml new file mode 100644 index 0000000000..a8494272b1 --- /dev/null +++ b/sdk/core/perf.yml @@ -0,0 +1,32 @@ +parameters: +- name: PackageVersions + displayName: PackageVersions (regex of package versions to run) + type: string + default: 'source' +- name: Tests + displayName: Tests (regex of tests to run) + type: string + default: '^(extendedOptions)$' +- name: Arguments + displayName: Arguments (regex of arguments to run) + type: string + default: '.*' +- name: Iterations + displayName: Iterations (times to run each test) + type: number + default: '5' +- name: AdditionalArguments + displayName: AdditionalArguments (passed to PerfAutomation) + type: string + default: ' ' + +extends: + template: /eng/pipelines/templates/jobs/perf.yml + parameters: + ServiceDirectory: core + Services: "^core$" + PackageVersions: ${{ parameters.PackageVersions }} + Tests: ${{ parameters.Tests }} + Arguments: ${{ parameters.Arguments }} + Iterations: ${{ parameters.Iterations }} + AdditionalArguments: ${{ parameters.AdditionalArguments }} diff --git a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp index 7a139f8093..cab73b98ef 100644 --- a/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp +++ b/sdk/storage/azure-storage-blobs/test/ut/blob_query_test.cpp @@ -130,7 +130,7 @@ id,name,price R"json({"id":"103","name":"apples","price":"99"}|{"id":"106","name":"lemons","price":"69"}|{"id":"110","name":"bananas","price":"39"}|{"id":"112","name":"sapote,mamey","price":"50"}|)json"); } - TEST_F(BlockBlobClientTest, DISABLED_QueryCsvInputArrowOutput_LIVEONLY_) + TEST_F(BlockBlobClientTest, QueryCsvInputArrowOutput_LIVEONLY_) { auto const testName(GetTestName()); auto client = GetBlockBlobClient(testName); diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index ef15e55df3..36a75bcd69 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -193,6 +193,67 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using RehydratePriority = Blobs::Models::RehydratePriority; using ArchiveStatus = Blobs::Models::ArchiveStatus; + /** + * @brief The path item returned when listing the paths. + */ + struct PathItem final + { + /** + * The name of the path. + */ + std::string Name; + + /** + * Indicates whether this path is a directory. + */ + bool IsDirectory = false; + + /** + * The data and time the path was last modified. + */ + DateTime LastModified; + + /** + * The size of the file. + */ + int64_t FileSize = int64_t(); + + /** + * The owner of the path. + */ + std::string Owner; + + /** + * The group of the path. + */ + std::string Group; + + /** + * The permission of the path. + */ + std::string Permissions; + + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + + /** + * The creation time of the path. + */ + Nullable CreatedOn; + + /** + * The expiry time of the path. + */ + Nullable ExpiresOn; + + /** + * An HTTP entity tag associated with the path. + */ + std::string ETag; + }; + /** * @brief The properties of the path. */ diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp index d4c0afcb5c..a2d73be699 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp @@ -83,24 +83,24 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { private: std::string m_value; }; - struct PathItem final - { - std::string Name; - bool IsDirectory = false; - DateTime LastModified; - int64_t FileSize = int64_t(); - std::string Owner; - std::string Group; - std::string Permissions; - /** - * The name of the encryption scope under which the blob is encrypted. - */ - Nullable EncryptionScope; - Nullable CreatedOn; - Nullable ExpiresOn; - std::string ETag; - }; namespace _detail { + struct PathItem final + { + std::string Name; + bool IsDirectory = false; + DateTime LastModified; + int64_t FileSize = int64_t(); + std::string Owner; + std::string Group; + std::string Permissions; + /** + * The name of the encryption scope under which the blob is encrypted. + */ + Nullable EncryptionScope; + Nullable CreatedOn; + Nullable ExpiresOn; + std::string ETag; + }; /** * @brief Response type for #Azure::Storage::Files::DataLake::FileSystemClient::ListPaths. */ diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 7e81aa9f0b..3023758b84 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -247,8 +247,31 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { _internal::WithReplicaStatus(context)); ListPathsPagedResponse pagedResponse; - - pagedResponse.Paths = std::move(response.Value.Paths); + const std::string emptyExpiresOnString = "0"; + for (auto& path : response.Value.Paths) + { + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) + { + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); + } + pagedResponse.Paths.push_back(std::move(item)); + } pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index c4e292127f..a6dddbfc15 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -276,18 +276,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { protocolLayerOptionsCopy, _internal::WithReplicaStatus(context)); - const auto emptyExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime(0); + ListPathsPagedResponse pagedResponse; + const std::string emptyExpiresOnString = "0"; for (auto& path : response.Value.Paths) { - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() == emptyExpiresOn) + Models::PathItem item; + item.Name = std::move(path.Name); + item.IsDirectory = path.IsDirectory; + item.LastModified = std::move(path.LastModified); + item.FileSize = path.FileSize; + item.Owner = std::move(path.Owner); + item.Group = std::move(path.Group); + item.Permissions = std::move(path.Permissions); + item.EncryptionScope = path.EncryptionScope; + item.ETag = std::move(path.ETag); + if (path.CreatedOn.HasValue()) + { + item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.CreatedOn.Value())); + } + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) { - path.ExpiresOn.Reset(); + item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( + std::stoll(path.ExpiresOn.Value())); } + pagedResponse.Paths.push_back(std::move(item)); } - - ListPathsPagedResponse pagedResponse; - pagedResponse.Paths = std::move(response.Value.Paths); pagedResponse.m_onNextPageFunc = func; pagedResponse.CurrentPageToken = continuationToken; pagedResponse.NextPageToken = response.Value.ContinuationToken; diff --git a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp index a8e4e3adf6..d0d006882d 100644 --- a/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp +++ b/sdk/storage/azure-storage-files-datalake/src/private/datalake_utilities.hpp @@ -5,6 +5,7 @@ #include +#include #include #include @@ -28,4 +29,48 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam Blobs::BlobClientOptions GetBlobClientOptions(const DataLakeClientOptions& options); + /** + * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. + * + */ + class Win32FileTimeConverter final { + public: + /** + * @brief Converts Win32 FILETIME to an #Azure::DateTime. + * + * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. + * @return Calculated #Azure::DateTime. + */ + static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) + { + auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); + return DateTime(t); + } + + /** + * @brief Converts a DateTime to Win32 FILETIME. + * + * @param dateTime The `%DateTime` to convert. + * @return The number of 100-nanoseconds since 1601-01-01. + */ + static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) + { + return std::chrono::duration_cast(dateTime - DateTime(1601)) + .count(); + } + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be created. + * + */ + Win32FileTimeConverter() = delete; + + /** + * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no + * instance can be created. + * + */ + ~Win32FileTimeConverter() = delete; + }; + }}}}} // namespace Azure::Storage::Files::DataLake::_detail diff --git a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp index 47c7d56686..c105e1428a 100644 --- a/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/rest_client.cpp @@ -94,7 +94,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { ? jsonRoot["paths"] : Core::Json::_internal::json::array()) { - Models::PathItem vectorElement2; + Models::_detail::PathItem vectorElement2; vectorElement2.Name = var0["name"].get(); if (var0.count("isDirectory") != 0) { @@ -116,15 +116,11 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { } if (var0.count("creationTime") != 0) { - vectorElement2.CreatedOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["creationTime"].get())); + vectorElement2.CreatedOn = var0["creationTime"].get(); } if (var0.count("expiryTime") != 0) { - vectorElement2.ExpiresOn - = Core::_internal::Win32FileTimeConverter::Win32FileTimeToDateTime( - std::stoll(var0["expiryTime"].get())); + vectorElement2.ExpiresOn = var0["expiryTime"].get(); } if (var0.count("etag") != 0) { diff --git a/sdk/storage/azure-storage-files-datalake/swagger/README.md b/sdk/storage/azure-storage-files-datalake/swagger/README.md index 9626429b02..b7c915b212 100644 --- a/sdk/storage/azure-storage-files-datalake/swagger/README.md +++ b/sdk/storage/azure-storage-files-datalake/swagger/README.md @@ -229,12 +229,13 @@ directive: where: $.definitions transform: > $.Path["x-ms-client-name"] = "PathItem"; + $.Path["x-namespace"] = "_detail"; $.Path.properties["lastModified"]["format"] = "date-time-rfc1123"; $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize"; $.Path.properties["isDirectory"]["x-ms-client-default"] = false; $.Path.properties["EncryptionScope"]["x-nullable"] = true; - $.Path.properties["creationTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "CreatedOn", "x-nullable": true}; - $.Path.properties["expiryTime"] = {"type": "string", "x-ms-format": "win32filetime", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; + $.Path.properties["creationTime"] = {"type": "string", "x-ms-client-name": "CreatedOn", "x-nullable": true}; + $.Path.properties["expiryTime"] = {"type": "string", "x-ms-client-name": "ExpiresOn", "x-nullable": true}; $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"}; delete $.Path.properties["eTag"]; $.PathList["x-namespace"] = "_detail"; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 2cbf43567a..24f88449a1 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -318,6 +318,8 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NE(result.end(), iter); EXPECT_EQ(iter->Name, name); EXPECT_EQ(iter->Name.substr(0U, m_directoryA.size()), m_directoryA); + EXPECT_TRUE(iter->CreatedOn.HasValue()); + EXPECT_FALSE(iter->ExpiresOn.HasValue()); } for (const auto& name : m_pathNameSetB) { From 77c6f59e2bc6b364d4b12b964021a9349f5a37b4 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Thu, 29 Sep 2022 15:51:37 +0800 Subject: [PATCH 09/11] hns soft delete (#3963) --- .../storage/blobs/blob_container_client.hpp | 1 + .../inc/azure/storage/blobs/rest_client.hpp | 5 + .../azure-storage-blobs/src/rest_client.cpp | 23 + .../azure-storage-blobs/swagger/README.md | 15 + .../datalake/datalake_file_system_client.hpp | 26 + .../files/datalake/datalake_options.hpp | 35 ++ .../files/datalake/datalake_responses.hpp | 56 ++ .../datalake/datalake_service_client.hpp | 34 ++ .../src/datalake_file_system_client.cpp | 71 +++ .../src/datalake_responses.cpp | 6 + .../ut/datalake_file_system_client_test.cpp | 123 ++++ .../test/ut/datalake_service_client_test.cpp | 212 +++++++ ...FileSystemClientTest.ListDeletedPaths.json | 334 +++++++++++ ...DataLakeFileSystemClientTest.Undelete.json | 563 ++++++++++++++++++ ...taLakeServiceClientTest.GetProperties.json | 25 + ...taLakeServiceClientTest.SetProperties.json | 88 +++ sdk/storage/test-resources.json | 4 + 17 files changed, 1621 insertions(+) create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListDeletedPaths.json create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.Undelete.json create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.GetProperties.json create mode 100644 sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.SetProperties.json diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp index 74a4de2dc5..f77b34b94c 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp @@ -338,6 +338,7 @@ namespace Azure { namespace Storage { namespace Blobs { friend class BlobServiceClient; friend class BlobLeaseClient; friend class BlobContainerBatch; + friend class Files::DataLake::DataLakeFileSystemClient; }; }}} // namespace Azure::Storage::Blobs diff --git a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp index 2cb58be985..1619af2717 100644 --- a/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp +++ b/sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp @@ -1365,6 +1365,10 @@ namespace Azure { namespace Storage { namespace Blobs { * Type of the blob. */ Models::BlobType BlobType; + /** + * The deletion ID associated with the deleted path. + */ + Nullable DeletionId; }; } // namespace _detail /** @@ -3447,6 +3451,7 @@ namespace Azure { namespace Storage { namespace Blobs { Nullable Marker; Nullable MaxResults; Nullable Include; + Nullable ShowOnly; }; static Response ListBlobsByHierarchy( Core::Http::_internal::HttpPipeline& pipeline, diff --git a/sdk/storage/azure-storage-blobs/src/rest_client.cpp b/sdk/storage/azure-storage-blobs/src/rest_client.cpp index 29eb40f9f3..f49fb800c5 100644 --- a/sdk/storage/azure-storage-blobs/src/rest_client.cpp +++ b/sdk/storage/azure-storage-blobs/src/rest_client.cpp @@ -2310,6 +2310,7 @@ namespace Azure { namespace Storage { namespace Blobs { kHasVersionsOnly, kContentLength, kBlobType, + kDeletionId, }; const std::unordered_map XmlTagEnumMap{ {"EnumerationResults", XmlTagEnum::kEnumerationResults}, @@ -2370,6 +2371,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, + {"DeletionId", XmlTagEnum::kDeletionId}, }; std::vector xmlPath; Models::_detail::BlobItem vectorElement1; @@ -2807,6 +2809,13 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement1.BlobType = Models::BlobType(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kDeletionId) + { + vectorElement1.DeletionId = node.Value; + } } else if (node.Type == _internal::XmlNodeType::Attribute) { @@ -2908,6 +2917,11 @@ namespace Azure { namespace Storage { namespace Blobs { ListBlobsIncludeFlagsToString(options.Include.Value()))); } request.SetHeader("x-ms-version", "2021-04-10"); + if (options.ShowOnly.HasValue() && !options.ShowOnly.Value().empty()) + { + request.GetUrl().AppendQueryParameter( + "showonly", _internal::UrlEncodeQueryParameter(options.ShowOnly.Value())); + } auto pRawResponse = pipeline.Send(request, context); auto httpStatusCode = pRawResponse->GetStatusCode(); if (httpStatusCode != Core::Http::HttpStatusCode::Ok) @@ -2981,6 +2995,7 @@ namespace Azure { namespace Storage { namespace Blobs { kHasVersionsOnly, kContentLength, kBlobType, + kDeletionId, kBlobPrefix, }; const std::unordered_map XmlTagEnumMap{ @@ -3043,6 +3058,7 @@ namespace Azure { namespace Storage { namespace Blobs { {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly}, {"Content-Length", XmlTagEnum::kContentLength}, {"BlobType", XmlTagEnum::kBlobType}, + {"DeletionId", XmlTagEnum::kDeletionId}, {"BlobPrefix", XmlTagEnum::kBlobPrefix}, }; std::vector xmlPath; @@ -3488,6 +3504,13 @@ namespace Azure { namespace Storage { namespace Blobs { { vectorElement1.BlobType = Models::BlobType(node.Value); } + else if ( + xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults + && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob + && xmlPath[3] == XmlTagEnum::kDeletionId) + { + vectorElement1.DeletionId = node.Value; + } else if ( xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlobPrefix diff --git a/sdk/storage/azure-storage-blobs/swagger/README.md b/sdk/storage/azure-storage-blobs/swagger/README.md index 494f44eafb..efb40c7ee0 100644 --- a/sdk/storage/azure-storage-blobs/swagger/README.md +++ b/sdk/storage/azure-storage-blobs/swagger/README.md @@ -273,6 +273,15 @@ directive: {"value": "legalhold", "name": "LegalHold"}, {"value": "deletedwithversions", "name": "DeletedWithVersions"} ]; + $["ListBlobsShowOnly"]= { + "name": "showonly", + "x-ms-client-name": "ShowOnly", + "in": "query", + "required": false, + "type": "string", + "x-ms-parameter-location": "method", + "description": "Include this parameter to specify one or more datasets to include in the response." + }; $.DeleteSnapshots["x-ms-enum"]["name"] = "DeleteSnapshotsOption"; $.DeleteSnapshots["x-ms-enum"]["values"] = [{"value": "include", "name": "IncludeSnapshots"},{"value":"only", "name": "OnlySnapshots"}]; $.BlobExpiryOptions["x-ms-enum"]["name"] = "ScheduleBlobExpiryOriginType"; @@ -829,6 +838,7 @@ directive: $.BlobItemInternal.properties["BlobType"] = $.BlobPropertiesInternal.properties["BlobType"]; $.BlobItemInternal.properties["BlobType"]["x-ms-xml"] = {"name": "Properties/BlobType"}; delete $.BlobPropertiesInternal.properties["BlobType"]; + $.BlobItemInternal.properties["DeletionId"] = {"type": "string"}; $.BlobItemInternal.required.push("BlobType", "BlobSize"); $.BlobItemInternal.properties["Name"].description = "Blob name."; $.BlobItemInternal.properties["Deleted"].description = "Indicates whether this blob was deleted."; @@ -837,6 +847,7 @@ directive: $.BlobItemInternal.properties["IsCurrentVersion"].description = "Indicates if this is the current version of the blob."; $.BlobItemInternal.properties["BlobType"].description = "Type of the blob."; $.BlobItemInternal.properties["HasVersionsOnly"].description = "Indicates that this root blob has been deleted, but it has versions that are active."; + $.BlobItemInternal.properties["DeletionId"].description = "The deletion ID associated with the deleted path."; $.BlobPropertiesInternal.properties["Etag"]["x-ms-client-name"] = "ETag"; $.BlobPropertiesInternal["x-ms-client-name"] = "BlobItemDetails"; @@ -916,6 +927,10 @@ directive: delete $.ListBlobsHierarchySegmentResponse.properties["Segment"]; delete $.ListBlobsHierarchySegmentResponse.required; $.ListBlobsHierarchySegmentResponse.properties["NextMarker"]["x-nullable"] = true; + - from: swagger-document + where: $["x-ms-paths"]["/{containerName}?restype=container&comp=list&hierarchy"].get.parameters + transform: > + $.push({"$ref": "#/parameters/ListBlobsShowOnly"}); ``` ### DownloadBlob diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp index 9a7430e1c1..4e390b13b5 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp @@ -239,6 +239,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { const RenameDirectoryOptions& options = RenameDirectoryOptions(), const Azure::Core::Context& context = Azure::Core::Context()) const; + /** + * @brief Gets the paths that have recently been soft deleted in this file system. + * @param options Optional parameters to list deleted paths. + * @param context Context for cancelling long running operations. + * @return Azure::Response The client targets the restored path. + * @remark This request is sent to Blob endpoint. + */ + ListDeletedPathsPagedResponse ListDeletedPaths( + const ListDeletedPathsOptions& options = ListDeletedPathsOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + + /** + * @brief Restores a soft deleted path. + * @param deletedPath The path of the deleted path. + * @param deletionId The deletion ID associated with the soft deleted path. You can get soft + * deleted paths and their associated deletion IDs with ListDeletedPaths. + * @param context Context for cancelling long running operations. + * @return Azure::Response The client targets the restored path. + * @remark This request is sent to Blob endpoint. + */ + Azure::Response UndeletePath( + const std::string& deletedPath, + const std::string& deletionId, + const UndeletePathOptions& options = UndeletePathOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const; + private: Azure::Core::Url m_fileSystemUrl; Blobs::BlobContainerClient m_blobContainerClient; diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp index db134508cd..591269bb05 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp @@ -82,6 +82,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using DownloadFileToOptions = Blobs::DownloadBlobToOptions; using GetUserDelegationKeyOptions = Blobs::GetUserDelegationKeyOptions; + using GetServicePropertiesOptions = Blobs::GetServicePropertiesOptions; + using SetServicePropertiesOptions = Blobs::SetServicePropertiesOptions; using EncryptionKey = Blobs::EncryptionKey; /** @@ -245,6 +247,39 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { Azure::Nullable PageSizeHint; }; + /** + * @brief Optional parameters for #Azure::Storage::Files::DataLake::FileSystemClient::ListPaths. + */ + struct ListDeletedPathsOptions final + { + /** + * Gets the paths that have recently been soft deleted in this file system. + */ + Azure::Nullable Prefix; + + /** + * The number of paths returned with each invocation is limited. If the number of paths to be + * returned exceeds this limit, a continuation token is returned in the response header + * x-ms-continuation. When a continuation token is returned in the response, it must be + * specified in a subsequent invocation of the list operation to continue listing the paths. + */ + Azure::Nullable ContinuationToken; + + /** + * An optional value that specifies the maximum number of items to return. If omitted or greater + * than 5,000, the response will include up to 5,000 items. + */ + Azure::Nullable PageSizeHint; + }; + + /** + * @brief Optional parameters for + * #Azure::Storage::Files::DataLake::FileSystemClient::UndeletePath. + */ + struct UndeletePathOptions final + { + }; + /** * @brief Optional parameters for * #Azure::Storage::Files::DataLake::FileSystemClient::GetAccessPolicy. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 36a75bcd69..173f274b7f 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -14,6 +14,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { class DataLakeServiceClient; + class DataLakeFileSystemClient; class DataLakePathClient; namespace Models { @@ -26,6 +27,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { // ServiceClient models: using UserDelegationKey = Blobs::Models::UserDelegationKey; + using RetentionPolicy = Blobs::Models::RetentionPolicy; + using AnalyticsLogging = Blobs::Models::AnalyticsLogging; + using Metrics = Blobs::Models::Metrics; + using CorsRule = Blobs::Models::CorsRule; + using StaticWebsite = Blobs::Models::StaticWebsite; + using DataLakeServiceProperties = Blobs::Models::BlobServiceProperties; + using SetServicePropertiesResult = Blobs::Models::SetServicePropertiesResult; /** * @brief The detailed information of a file system. @@ -450,6 +458,32 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { using SetPathPermissionsResult = SetPathAccessControlListResult; + /** + * @brief A path that has been soft deleted. + */ + struct PathDeletedItem final + { + /** + * The name of the path. + */ + std::string Name; + + /** + * The deletion ID associated with the deleted path. + */ + std::string DeletionId; + + /** + * When the path was deleted. + */ + DateTime DeletedOn; + + /** + * The number of days left before the soft deleted path will be permanently deleted. + */ + int64_t RemainingRetentionDays = int64_t(); + }; + // FileClient models: using UploadFileFromResult = Blobs::Models::UploadBlockBlobResult; @@ -719,6 +753,28 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { friend class Azure::Core::PagedResponse; }; + /** + * @brief Response type for + * #Azure::Storage::Files::DataLake::DataLakeFileSystemClient::ListDeletedPaths. + */ + class ListDeletedPathsPagedResponse final + : public Azure::Core::PagedResponse { + public: + /** + * Path items. + */ + std::vector DeletedPaths; + + private: + void OnNextPage(const Azure::Core::Context& context); + + std::shared_ptr m_fileSystemClient; + ListDeletedPathsOptions m_operationOptions; + + friend class DataLakeFileSystemClient; + friend class Azure::Core::PagedResponse; + }; + /** * @brief Response type for * #Azure::Storage::Files::DataLake::DataLakePathClient::SetAccessControlListRecursive. diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp index 564c971d02..1cb927ff3f 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_service_client.hpp @@ -109,6 +109,40 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { return m_blobServiceClient.GetUserDelegationKey(expiresOn, options, context); } + /** + * @brief Sets properties for a storage account's Blob service endpoint, including + * properties for Storage Analytics, CORS (Cross-Origin Resource Sharing) rules and soft delete + * settings. You can also use this operation to set the default request version for all incoming + * requests to the DataLake service that do not have a version specified. + * + * @param properties The DataLake service properties. + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A SetServicePropertiesResult on successfully setting the properties. + */ + Azure::Response SetProperties( + Models::DataLakeServiceProperties properties, + const SetServicePropertiesOptions& options = SetServicePropertiesOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const + { + return m_blobServiceClient.SetProperties(properties, options, context); + } + + /** + * @brief Gets the properties of a storage account's datalake service, including properties + * for Storage Analytics and CORS (Cross-Origin Resource Sharing) rules. + * + * @param options Optional parameters to execute this function. + * @param context Context for cancelling long running operations. + * @return A DataLakeServiceProperties describing the service properties. + */ + Azure::Response GetProperties( + const GetServicePropertiesOptions& options = GetServicePropertiesOptions(), + const Azure::Core::Context& context = Azure::Core::Context()) const + { + return m_blobServiceClient.GetProperties(options, context); + } + private: Azure::Core::Url m_serviceUrl; Blobs::BlobServiceClient m_blobServiceClient; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index a6dddbfc15..3ca209e5f1 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -449,4 +449,75 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(renamedDirectoryClient), std::move(result.RawResponse)); } + ListDeletedPathsPagedResponse DataLakeFileSystemClient::ListDeletedPaths( + const ListDeletedPathsOptions& options, + const Azure::Core::Context& context) const + { + Blobs::_detail::BlobContainerClient::ListBlobContainerBlobsByHierarchyOptions + protocolLayerOptions; + protocolLayerOptions.Prefix = options.Prefix; + protocolLayerOptions.MaxResults = options.PageSizeHint; + protocolLayerOptions.Marker = options.ContinuationToken; + protocolLayerOptions.ShowOnly = "deleted"; + auto result = Blobs::_detail::BlobContainerClient::ListBlobsByHierarchy( + *m_pipeline, m_blobContainerClient.m_blobContainerUrl, protocolLayerOptions, context); + + ListDeletedPathsPagedResponse pagedResponse; + for (auto& item : result.Value.Items) + { + Models::PathDeletedItem pathDeletedItem; + if (item.Name.Encoded) + { + pathDeletedItem.Name = Core::Url::Decode(item.Name.Content); + } + else + { + pathDeletedItem.Name = std::move(item.Name.Content); + } + pathDeletedItem.DeletedOn = item.Details.DeletedOn.Value(); + pathDeletedItem.DeletionId = item.DeletionId.Value(); + pathDeletedItem.RemainingRetentionDays = item.Details.RemainingRetentionDays.Value(); + + pagedResponse.DeletedPaths.push_back(std::move(pathDeletedItem)); + } + pagedResponse.m_operationOptions = options; + pagedResponse.m_fileSystemClient = std::make_shared(*this); + pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); + pagedResponse.NextPageToken = result.Value.ContinuationToken; + pagedResponse.RawResponse = std::move(result.RawResponse); + + return pagedResponse; + } + + Azure::Response DataLakeFileSystemClient::UndeletePath( + const std::string& deletedPath, + const std::string& deletionId, + const UndeletePathOptions& options, + const Azure::Core::Context& context) const + { + (void)options; + /* cspell:disable-next-line */ + std::string undeleteSource = "?deletionid=" + deletionId; + + auto blobUrl = m_blobContainerClient.m_blobContainerUrl; + blobUrl.AppendPath(_internal::UrlEncodePath(deletedPath)); + + _detail::PathClient::UndeletePathOptions protocolLayerOptions; + protocolLayerOptions.UndeleteSource = undeleteSource; + auto result + = _detail::PathClient::Undelete(*m_pipeline, blobUrl, protocolLayerOptions, context); + + if (result.Value.ResourceType.HasValue() + && result.Value.ResourceType.Value() == Models::PathResourceType::Directory.ToString()) + { + return Azure::Response( + std::move(GetDirectoryClient(deletedPath)), std::move(result.RawResponse)); + } + else + { + return Azure::Response( + std::move(GetFileClient(deletedPath)), std::move(result.RawResponse)); + } + } + }}}} // namespace Azure::Storage::Files::DataLake diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp index 7e3291beae..0ef5a9d612 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_responses.cpp @@ -93,6 +93,12 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { *this = m_onNextPageFunc(NextPageToken.Value(), context); } + void ListDeletedPathsPagedResponse::OnNextPage(const Azure::Core::Context& context) + { + m_operationOptions.ContinuationToken = NextPageToken; + *this = m_fileSystemClient->ListDeletedPaths(m_operationOptions, context); + } + void SetPathAccessControlListRecursivePagedResponse::OnNextPage( const Azure::Core::Context& context) { diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 24f88449a1..37446f48b2 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -844,4 +844,127 @@ namespace Azure { namespace Storage { namespace Test { m_fileSystemClient->GetDirectoryClient(destinationDirectoryName).GetProperties()); } + TEST_F(DataLakeFileSystemClientTest, DISABLED_ListDeletedPaths) + { + const std::string deletedFilename = GetTestNameLowerCase() + "_file_deleted"; + const std::string nonDeletedFilename = GetTestNameLowerCase() + "_file"; + const std::string deletedDirectoryName = GetTestNameLowerCase() + "_dir_deleted"; + const std::string nonDeletedDirectoryName = GetTestNameLowerCase() + "_dir"; + + auto deletedFileClient = m_fileSystemClient->GetFileClient(deletedFilename); + auto nonDeletedFileClient = m_fileSystemClient->GetFileClient(nonDeletedFilename); + auto deletedDirectoryClient = m_fileSystemClient->GetDirectoryClient(deletedDirectoryName); + auto nonDeletedDirectoryClient + = m_fileSystemClient->GetDirectoryClient(nonDeletedDirectoryName); + + deletedFileClient.Create(); + deletedFileClient.Delete(); + nonDeletedFileClient.Create(); + deletedDirectoryClient.Create(); + deletedDirectoryClient.DeleteEmpty(); + nonDeletedDirectoryClient.Create(); + + { + std::vector paths; + EXPECT_NO_THROW(paths = std::move(m_fileSystemClient->ListDeletedPaths().DeletedPaths)); + EXPECT_EQ(2, paths.size()); + EXPECT_EQ(deletedDirectoryName, paths[0].Name); + EXPECT_EQ(deletedFilename, paths[1].Name); + } + // List max result + { + Files::DataLake::ListDeletedPathsOptions options; + options.PageSizeHint = 1; + std::vector paths; + for (auto pageResult = m_fileSystemClient->ListDeletedPaths(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); + EXPECT_EQ(1, pageResult.DeletedPaths.size()); + } + EXPECT_EQ(2, paths.size()); + } + // prefix works + { + const std::string directoryName = GetTestNameLowerCase() + "_prefix"; + const std::string filename = "file"; + + auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName); + directoryClient.Create(); + auto fileClient = directoryClient.GetFileClient(filename); + fileClient.Create(); + fileClient.Delete(); + + Files::DataLake::ListDeletedPathsOptions options; + options.Prefix = directoryName; + std::vector paths; + EXPECT_NO_THROW( + paths = std::move(m_fileSystemClient->ListDeletedPaths(options).DeletedPaths)); + EXPECT_EQ(1, paths.size()); + EXPECT_EQ(directoryName + "/" + filename, paths[0].Name); + } + } + + TEST_F(DataLakeFileSystemClientTest, DISABLED_Undelete) + { + const std::string directoryName = GetTestNameLowerCase() + "_dir"; + const std::string subFileName = "sub_file"; + + auto directoryClient = m_fileSystemClient->GetDirectoryClient(directoryName); + directoryClient.Create(); + auto subFileClient = m_fileSystemClient->GetFileClient(directoryName + "/" + subFileName); + subFileClient.Create(); + Files::DataLake::Models::PathProperties properties = directoryClient.GetProperties().Value; + Files::DataLake::Models::PathProperties subFileProperties = subFileClient.GetProperties().Value; + + // recursive works + { + directoryClient.DeleteRecursive(); + + auto paths = m_fileSystemClient->ListDeletedPaths().DeletedPaths; + const std::string deletionId = paths[0].DeletionId; + + auto restoredClient = m_fileSystemClient->UndeletePath(directoryName, deletionId).Value; + + paths = m_fileSystemClient->ListDeletedPaths().DeletedPaths; + EXPECT_EQ(0, paths.size()); + Files::DataLake::Models::PathProperties restoredProperties; + EXPECT_NO_THROW(restoredProperties = restoredClient.GetProperties().Value); + EXPECT_TRUE(restoredProperties.IsDirectory); + EXPECT_EQ(properties.ETag, restoredProperties.ETag); + Files::DataLake::Models::PathProperties restoredSubFileProperties; + EXPECT_NO_THROW(restoredSubFileProperties = subFileClient.GetProperties().Value); + EXPECT_TRUE(!restoredSubFileProperties.IsDirectory); + EXPECT_EQ(subFileProperties.ETag, restoredSubFileProperties.ETag); + } + // not recursive works + { + subFileClient.Delete(); + directoryClient.DeleteEmpty(); + + auto paths = m_fileSystemClient->ListDeletedPaths().DeletedPaths; + std::string deletionId = paths[0].DeletionId; + + // restore directory + auto restoredClient = m_fileSystemClient->UndeletePath(directoryName, deletionId).Value; + paths = m_fileSystemClient->ListDeletedPaths().DeletedPaths; + EXPECT_EQ(1, paths.size()); // not restore subFile + Files::DataLake::Models::PathProperties restoredProperties; + EXPECT_NO_THROW(restoredProperties = restoredClient.GetProperties().Value); + EXPECT_TRUE(restoredProperties.IsDirectory); + EXPECT_EQ(properties.ETag, restoredProperties.ETag); + EXPECT_THROW(subFileClient.GetProperties(), StorageException); + + // restore file + deletionId = paths[0].DeletionId; + restoredClient + = m_fileSystemClient->UndeletePath(directoryName + "/" + subFileName, deletionId).Value; + paths = m_fileSystemClient->ListDeletedPaths().DeletedPaths; + EXPECT_EQ(0, paths.size()); + EXPECT_NO_THROW(restoredProperties = restoredClient.GetProperties().Value); + EXPECT_FALSE(restoredProperties.IsDirectory); + EXPECT_EQ(subFileProperties.ETag, restoredProperties.ETag); + } + } + }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp index ca7458b166..8eb1ada010 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_service_client_test.cpp @@ -6,6 +6,66 @@ #include #include +namespace Azure { namespace Storage { namespace Blobs { namespace Models { + + bool operator==(const RetentionPolicy& lhs, const RetentionPolicy& rhs) + { + if (lhs.IsEnabled != rhs.IsEnabled) + { + return false; + } + if (lhs.Days.HasValue() != rhs.Days.HasValue()) + { + return false; + } + if (lhs.Days.HasValue() && rhs.Days.HasValue() && lhs.Days.Value() != rhs.Days.Value()) + { + return false; + } + return true; + } + + bool operator==(const CorsRule& lhs, const CorsRule& rhs) + { + return lhs.AllowedHeaders == rhs.AllowedHeaders && lhs.AllowedMethods == rhs.AllowedMethods + && lhs.AllowedOrigins == rhs.AllowedOrigins && lhs.ExposedHeaders == rhs.ExposedHeaders + && lhs.MaxAgeInSeconds == rhs.MaxAgeInSeconds; + } + + bool operator==(const StaticWebsite& lhs, const StaticWebsite& rhs) + { + if (lhs.IsEnabled != rhs.IsEnabled) + { + return false; + } + if (lhs.DefaultIndexDocumentPath.HasValue() != rhs.DefaultIndexDocumentPath.HasValue()) + { + return false; + } + if (lhs.DefaultIndexDocumentPath.HasValue() && rhs.DefaultIndexDocumentPath.HasValue() + && lhs.DefaultIndexDocumentPath.Value() != rhs.DefaultIndexDocumentPath.Value()) + { + return false; + } + if (lhs.DefaultIndexDocumentPath.HasValue() != rhs.DefaultIndexDocumentPath.HasValue()) + { + return false; + } + if (lhs.ErrorDocument404Path.HasValue() && rhs.ErrorDocument404Path.HasValue() + && lhs.ErrorDocument404Path.Value() != rhs.ErrorDocument404Path.Value()) + { + return false; + } + if (lhs.IndexDocument.HasValue() && rhs.IndexDocument.HasValue() + && lhs.IndexDocument.Value() != rhs.IndexDocument.Value()) + { + return false; + } + return true; + } + +}}}} // namespace Azure::Storage::Blobs::Models + namespace Azure { namespace Storage { namespace Test { const size_t FileSystemTestSize = 5; @@ -174,4 +234,156 @@ namespace Azure { namespace Storage { namespace Test { EXPECT_NO_THROW(datalakeServiceClient.ListFileSystems()); } + TEST_F(DataLakeServiceClientTest, GetProperties) + { + + auto ret = m_dataLakeServiceClient->GetProperties(); + auto properties = ret.Value; + auto logging = properties.Logging; + EXPECT_FALSE(logging.Version.empty()); + if (logging.RetentionPolicy.IsEnabled) + { + EXPECT_TRUE(logging.RetentionPolicy.Days.HasValue()); + } + auto hourMetrics = properties.HourMetrics; + if (hourMetrics.IsEnabled) + { + EXPECT_FALSE(hourMetrics.Version.empty()); + if (hourMetrics.RetentionPolicy.IsEnabled) + { + EXPECT_TRUE(hourMetrics.RetentionPolicy.Days.HasValue()); + } + } + auto minuteMetrics = properties.HourMetrics; + if (minuteMetrics.IsEnabled) + { + EXPECT_FALSE(minuteMetrics.Version.empty()); + if (minuteMetrics.RetentionPolicy.IsEnabled) + { + EXPECT_TRUE(minuteMetrics.RetentionPolicy.Days.HasValue()); + } + } + auto deleteRetentionPolicy = properties.DeleteRetentionPolicy; + if (deleteRetentionPolicy.IsEnabled) + { + EXPECT_TRUE(deleteRetentionPolicy.Days.HasValue()); + } + } + + TEST_F(DataLakeServiceClientTest, SetProperties) + { + + auto getServicePropertiesResult = m_dataLakeServiceClient->GetProperties().Value; + Files::DataLake::Models::DataLakeServiceProperties properties; + properties.Logging = getServicePropertiesResult.Logging; + properties.HourMetrics = getServicePropertiesResult.HourMetrics; + properties.MinuteMetrics = getServicePropertiesResult.MinuteMetrics; + properties.Cors = getServicePropertiesResult.Cors; + properties.DefaultServiceVersion = getServicePropertiesResult.DefaultServiceVersion; + properties.DeleteRetentionPolicy = getServicePropertiesResult.DeleteRetentionPolicy; + properties.StaticWebsite = getServicePropertiesResult.StaticWebsite; + + auto originalProperties = properties; + + properties.Logging.Delete = !properties.Logging.Delete; + properties.Logging.Read = !properties.Logging.Read; + properties.Logging.Write = !properties.Logging.Write; + properties.Logging.RetentionPolicy.IsEnabled = true; + properties.Logging.RetentionPolicy.Days = 3; + + properties.HourMetrics.IsEnabled = true; + properties.HourMetrics.RetentionPolicy.IsEnabled = true; + properties.HourMetrics.RetentionPolicy.Days = 4; + properties.HourMetrics.IncludeApis = true; + + properties.MinuteMetrics.IsEnabled = true; + properties.MinuteMetrics.RetentionPolicy.IsEnabled = true; + properties.MinuteMetrics.RetentionPolicy.Days = 4; + properties.MinuteMetrics.IncludeApis = true; + + properties.DefaultServiceVersion = Files::DataLake::_detail::ApiVersion; + + properties.StaticWebsite.IsEnabled = true; + properties.StaticWebsite.IndexDocument = "index.html"; + properties.StaticWebsite.ErrorDocument404Path = "404.html"; + properties.StaticWebsite.DefaultIndexDocumentPath.Reset(); + + Files::DataLake::Models::CorsRule corsRule; + corsRule.AllowedOrigins = "http://www.example1.com"; + corsRule.AllowedMethods = "GET,PUT"; + corsRule.AllowedHeaders = "x-ms-header1,x-ms-header2"; + corsRule.ExposedHeaders = "x-ms-header3"; + corsRule.MaxAgeInSeconds = 10; + properties.Cors.emplace_back(corsRule); + + corsRule.AllowedOrigins = "http://www.example2.com"; + corsRule.AllowedMethods = "DELETE"; + corsRule.AllowedHeaders = "x-ms-header1"; + corsRule.ExposedHeaders = "x-ms-header2,x-ms-header3"; + corsRule.MaxAgeInSeconds = 20; + properties.Cors.emplace_back(corsRule); + + properties.DeleteRetentionPolicy.IsEnabled = true; + properties.DeleteRetentionPolicy.Days = 7; + + EXPECT_NO_THROW(m_dataLakeServiceClient->SetProperties(properties)); + + // It takes some time before the new properties comes into effect. + using namespace std::chrono_literals; + TestSleep(10s); + auto downloadedProperties = m_dataLakeServiceClient->GetProperties().Value; + EXPECT_EQ(downloadedProperties.Logging.Version, properties.Logging.Version); + EXPECT_EQ(downloadedProperties.Logging.Delete, properties.Logging.Delete); + EXPECT_EQ(downloadedProperties.Logging.Read, properties.Logging.Read); + EXPECT_EQ(downloadedProperties.Logging.Write, properties.Logging.Write); + EXPECT_EQ(downloadedProperties.Logging.RetentionPolicy, properties.Logging.RetentionPolicy); + + EXPECT_EQ(downloadedProperties.HourMetrics.Version, properties.HourMetrics.Version); + EXPECT_EQ(downloadedProperties.HourMetrics.IsEnabled, properties.HourMetrics.IsEnabled); + EXPECT_EQ( + downloadedProperties.HourMetrics.IncludeApis.HasValue(), + properties.HourMetrics.IncludeApis.HasValue()); + if (downloadedProperties.HourMetrics.IncludeApis.HasValue() + == properties.HourMetrics.IncludeApis.HasValue()) + { + EXPECT_EQ( + downloadedProperties.HourMetrics.IncludeApis.Value(), + properties.HourMetrics.IncludeApis.Value()); + } + EXPECT_EQ( + downloadedProperties.HourMetrics.RetentionPolicy, properties.HourMetrics.RetentionPolicy); + + EXPECT_EQ(downloadedProperties.MinuteMetrics.Version, properties.MinuteMetrics.Version); + EXPECT_EQ(downloadedProperties.MinuteMetrics.IsEnabled, properties.MinuteMetrics.IsEnabled); + EXPECT_EQ( + downloadedProperties.MinuteMetrics.IncludeApis.HasValue(), + properties.MinuteMetrics.IncludeApis.HasValue()); + if (downloadedProperties.MinuteMetrics.IncludeApis.HasValue() + == properties.MinuteMetrics.IncludeApis.HasValue()) + { + EXPECT_EQ( + downloadedProperties.MinuteMetrics.IncludeApis.Value(), + properties.MinuteMetrics.IncludeApis.Value()); + } + EXPECT_EQ( + downloadedProperties.MinuteMetrics.RetentionPolicy, + properties.MinuteMetrics.RetentionPolicy); + + EXPECT_EQ( + downloadedProperties.DefaultServiceVersion.HasValue(), + properties.DefaultServiceVersion.HasValue()); + if (downloadedProperties.DefaultServiceVersion.HasValue()) + { + EXPECT_EQ( + downloadedProperties.DefaultServiceVersion.Value(), + properties.DefaultServiceVersion.Value()); + } + EXPECT_EQ(downloadedProperties.Cors, properties.Cors); + + EXPECT_EQ(downloadedProperties.StaticWebsite, properties.StaticWebsite); + + EXPECT_EQ(downloadedProperties.DeleteRetentionPolicy, properties.DeleteRetentionPolicy); + + auto res = m_dataLakeServiceClient->SetProperties(originalProperties); + } }}} // namespace Azure::Storage::Test diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListDeletedPaths.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListDeletedPaths.json new file mode 100644 index 0000000000..b0764dbbd4 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.ListDeletedPaths.json @@ -0,0 +1,334 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "4852b7d9-3925-4652-42b3-dae71445ca10", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:07 GMT", + "etag": "\"0x8DAA0348687215F\"", + "last-modified": "Tue, 27 Sep 2022 03:01:08 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "4852b7d9-3925-4652-42b3-dae71445ca10", + "x-ms-request-id": "3803bf63-901e-0084-381d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "9597f3b2-8051-4fae-7a42-af04abd14058", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:09 GMT", + "etag": "\"0x8DAA03487668E60\"", + "last-modified": "Tue, 27 Sep 2022 03:01:09 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "9597f3b2-8051-4fae-7a42-af04abd14058", + "x-ms-request-id": "4c4b6f94-201f-0028-7d1d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_file_deleted?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "283a24d9-c672-4a35-469d-38c6ddff5a3a", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:10 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "283a24d9-c672-4a35-469d-38c6ddff5a3a", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212699969504", + "x-ms-request-id": "4c4b6faf-201f-0028-181d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_file_deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "fd92820d-a84c-4005-553c-236f35238574", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:10 GMT", + "etag": "\"0x8DAA03487DB1A28\"", + "last-modified": "Tue, 27 Sep 2022 03:01:10 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "fd92820d-a84c-4005-553c-236f35238574", + "x-ms-request-id": "4c4b6fd9-201f-0028-421d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_file?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "98d837bb-2268-465e-5f0d-dd46d60a6ee6", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:10 GMT", + "etag": "\"0x8DAA0348811B4A5\"", + "last-modified": "Tue, 27 Sep 2022 03:01:10 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "98d837bb-2268-465e-5f0d-dd46d60a6ee6", + "x-ms-request-id": "4c4b6ffb-201f-0028-631d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_dir_deleted?resource=directory" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "3b879536-f06b-4e70-4d1f-fa955c686651", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:11 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3b879536-f06b-4e70-4d1f-fa955c686651", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212712013186", + "x-ms-request-id": "4c4b7029-201f-0028-111d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_dir_deleted?recursive=false" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "b1485ff3-9018-4e1d-7612-d7c6361391a2", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:11 GMT", + "etag": "\"0x8DAA03488877376\"", + "last-modified": "Tue, 27 Sep 2022 03:01:11 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "b1485ff3-9018-4e1d-7612-d7c6361391a2", + "x-ms-request-id": "4c4b704a-201f-0028-321d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_dir?resource=directory" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "01922a3c-dfe2-444c-4ebe-35b4180305dd", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "listdeletedpaths_dir_deleted133087212712013186trueTue, 27 Sep 2022 03:01:10 GMTTue, 27 Sep 2022 03:01:10 GMTTue, 04 Oct 2022 03:01:11 GMT0x8DAA0348811B4A5directory0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:11 GMT6listdeletedpaths_file_deleted133087212699969504trueTue, 27 Sep 2022 03:01:09 GMTTue, 27 Sep 2022 03:01:09 GMTTue, 04 Oct 2022 03:01:09 GMT0x8DAA03487668E60file0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:09 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:11 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "01922a3c-dfe2-444c-4ebe-35b4180305dd", + "x-ms-request-id": "3803c98a-901e-0084-781d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a4a99895-6c61-4928-6900-ab38c0040236", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "1listdeletedpaths_dir_deleted133087212712013186trueTue, 27 Sep 2022 03:01:10 GMTTue, 27 Sep 2022 03:01:10 GMTTue, 04 Oct 2022 03:01:11 GMT0x8DAA0348811B4A5directory0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:11 GMT62!384!MDAwMjQ0IVZCYjYrSnordTlTSjVQd0JHS2NCR0tFQkwzcGphR0Z1WjJSaGRHRnNZV3RsQVRBeFJEaENNRGd6TlVGR016VkdOak12SkhSeVlYTm9MMlJoZEdGc1lXdGxabWxzWlhONWMzUmxiV05zYVdWdWRIUmxjM1JzYVhOMFpHVnNaWFJsWkhCaGRHaHpBVEF4UkRoRU1qRkVOalF4TUVKQ1F6VUNMMnhwYzNSa1pXeGxkR1ZrY0dGMGFITmZabWxzWlY5a1pXeGxkR1ZrQVRJd01qSXRNRGt0TWpkVU1ETTZNREU2TURrdU9UazJPVFV3TkZvV0FBQUEhMDAwMDI4ITIwMjItMDktMjdUMDM6MDE6MTIuMTU4MDU1Mloh", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:11 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "a4a99895-6c61-4928-6900-ab38c0040236", + "x-ms-request-id": "3803c9fc-901e-0084-5c1d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?comp=list&maxresults=1&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "ee16b368-7738-415c-436f-f0c892a836d7", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "2!384!MDAwMjQ0IVZCYjYrSnordTlTSjVQd0JHS2NCR0tFQkwzcGphR0Z1WjJSaGRHRnNZV3RsQVRBeFJEaENNRGd6TlVGR016VkdOak12SkhSeVlYTm9MMlJoZEdGc1lXdGxabWxzWlhONWMzUmxiV05zYVdWdWRIUmxjM1JzYVhOMFpHVnNaWFJsWkhCaGRHaHpBVEF4UkRoRU1qRkVOalF4TUVKQ1F6VUNMMnhwYzNSa1pXeGxkR1ZrY0dGMGFITmZabWxzWlY5a1pXeGxkR1ZrQVRJd01qSXRNRGt0TWpkVU1ETTZNREU2TURrdU9UazJPVFV3TkZvV0FBQUEhMDAwMDI4ITIwMjItMDktMjdUMDM6MDE6MTIuMTU4MDU1Mloh1listdeletedpaths_file_deleted133087212699969504trueTue, 27 Sep 2022 03:01:09 GMTTue, 27 Sep 2022 03:01:09 GMTTue, 04 Oct 2022 03:01:09 GMT0x8DAA03487668E60file0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:09 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:11 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "ee16b368-7738-415c-436f-f0c892a836d7", + "x-ms-request-id": "3803ca64-901e-0084-391d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?comp=list&marker=2!384!MDAwMjQ0IVZCYjYrSnordTlTSjVQd0JHS2NCR0tFQkwzcGphR0Z1WjJSaGRHRnNZV3RsQVRBeFJEaENNRGd6TlVGR016VkdOak12SkhSeVlYTm9MMlJoZEdGc1lXdGxabWxzWlhONWMzUmxiV05zYVdWdWRIUmxjM1JzYVhOMFpHVnNaWFJsWkhCaGRHaHpBVEF4UkRoRU1qRkVOalF4TUVKQ1F6VUNMMnhwYzNSa1pXeGxkR1ZrY0dGMGFITmZabWxzWlY5a1pXeGxkR1ZrQVRJd01qSXRNRGt0TWpkVU1ETTZNREU2TURrdU9UazJPVFV3TkZvV0FBQUEhMDAwMDI4ITIwMjItMDktMjdUMDM6MDE6MTIuMTU4MDU1Mloh&maxresults=1&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "b4b269ee-68df-4159-6b2f-dee9c01573ab", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:12 GMT", + "etag": "\"0x8DAA034893E6CF7\"", + "last-modified": "Tue, 27 Sep 2022 03:01:12 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "b4b269ee-68df-4159-6b2f-dee9c01573ab", + "x-ms-request-id": "4c4b7097-201f-0028-7f1d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_prefix?resource=directory" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6a1ff000-459b-4d1d-7692-e02752422968", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:13 GMT", + "etag": "\"0x8DAA034896F65DB\"", + "last-modified": "Tue, 27 Sep 2022 03:01:13 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "6a1ff000-459b-4d1d-7692-e02752422968", + "x-ms-request-id": "4c4b70b7-201f-0028-1f1d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_prefix/file?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "46f874e5-552f-4866-564e-ad35c1f89ffb", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:13 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "46f874e5-552f-4866-564e-ad35c1f89ffb", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212735027045", + "x-ms-request-id": "4c4b70d2-201f-0028-3a1d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestlistdeletedpaths/listdeletedpaths_prefix/file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "fe187a16-e3e7-4992-5fd9-57cde493b940", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "listdeletedpaths_prefixlistdeletedpaths_prefix/file133087212735027045trueTue, 27 Sep 2022 03:01:13 GMTTue, 27 Sep 2022 03:01:13 GMTTue, 04 Oct 2022 03:01:13 GMT0x8DAA034896F65DBfile0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:13 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:13 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "fe187a16-e3e7-4992-5fd9-57cde493b940", + "x-ms-request-id": "3803cce8-901e-0084-031d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?comp=list&prefix=listdeletedpaths_prefix&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "449930df-a9cb-44a9-5ad9-59eb4cca1288", + "x-ms-version": "2021-04-10" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:13 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "449930df-a9cb-44a9-5ad9-59eb4cca1288", + "x-ms-request-id": "3803cd63-901e-0084-721d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestlistdeletedpaths?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.Undelete.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.Undelete.json new file mode 100644 index 0000000000..bdb126a06b --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeFileSystemClientTest.Undelete.json @@ -0,0 +1,563 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f268529f-3a61-4d97-7a8b-87c1bf7722d6", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:13 GMT", + "etag": "\"0x8DAA0348A4720C7\"", + "last-modified": "Tue, 27 Sep 2022 03:01:14 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f268529f-3a61-4d97-7a8b-87c1bf7722d6", + "x-ms-request-id": "3803ce00-901e-0084-7b1d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?restype=container" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "3ca3565f-4f12-453a-449e-d412b76ecc28", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:14 GMT", + "etag": "\"0x8DAA0348A7A8ACF\"", + "last-modified": "Tue, 27 Sep 2022 03:01:14 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "3ca3565f-4f12-453a-449e-d412b76ecc28", + "x-ms-request-id": "4c4b714d-201f-0028-331d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir?resource=directory" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "80d7247b-af1e-4b67-5ae6-24dcbe1afdb8", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Created", + "STATUS_CODE": "201", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:15 GMT", + "etag": "\"0x8DAA0348AAE3164\"", + "last-modified": "Tue, 27 Sep 2022 03:01:15 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "80d7247b-af1e-4b67-5ae6-24dcbe1afdb8", + "x-ms-request-id": "4c4b7165-201f-0028-4b1d-d2b307000000", + "x-ms-request-server-encrypted": "true", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file?resource=file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6e0c849e-d03a-4470-7e93-230e9967aeb6", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:14 GMT", + "etag": "\"0x8DAA0348A7A8ACF\"", + "last-modified": "Tue, 27 Sep 2022 03:01:14 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "6e0c849e-d03a-4470-7e93-230e9967aeb6", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:14 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-meta-hdi_isfolder": "true", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rwxr-x---", + "x-ms-request-id": "3803cf55-901e-0084-1c1d-d2a0ae000000", + "x-ms-resource-type": "directory", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "c6d3db69-4932-4fbc-654b-d93773ce37ce", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:14 GMT", + "etag": "\"0x8DAA0348AAE3164\"", + "last-modified": "Tue, 27 Sep 2022 03:01:15 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "c6d3db69-4932-4fbc-654b-d93773ce37ce", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:15 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "3803cfc8-901e-0084-051d-d2a0ae000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "9cf7aead-77ab-44ce-5b4f-64f8425e8e07", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:16 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "9cf7aead-77ab-44ce-5b4f-64f8425e8e07", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212760750222", + "x-ms-request-id": "4c4b71b3-201f-0028-171d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir?recursive=true" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "d96f0fc6-feb6-49b1-4a05-077bc9370ee0", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "undelete_dir133087212760750222trueTue, 27 Sep 2022 03:01:14 GMTTue, 27 Sep 2022 03:01:14 GMTTue, 04 Oct 2022 03:01:16 GMT0x8DAA0348A7A8ACFdirectory0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:16 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:15 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "d96f0fc6-feb6-49b1-4a05-077bc9370ee0", + "x-ms-request-id": "3803d1e4-901e-0084-591d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f31e94c6-3605-4f8b-70c9-fb23b35f071a", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:16 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "f31e94c6-3605-4f8b-70c9-fb23b35f071a", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:14 GMT", + "x-ms-request-id": "3803d270-901e-0084-581d-d2a0ae000000", + "x-ms-resource-type": "directory", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir?comp=undelete" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "a5210b6f-e947-4216-7312-c8c1e02da4ec", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:16 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "a5210b6f-e947-4216-7312-c8c1e02da4ec", + "x-ms-request-id": "3803d370-901e-0084-3b1d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "7a2a53d1-5a47-44af-65d8-1f5c9ceb07b8", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:16 GMT", + "etag": "\"0x8DAA0348A7A8ACF\"", + "last-modified": "Tue, 27 Sep 2022 03:01:14 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "7a2a53d1-5a47-44af-65d8-1f5c9ceb07b8", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:14 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-meta-hdi_isfolder": "true", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rwxr-x---", + "x-ms-request-id": "3803d427-901e-0084-6b1d-d2a0ae000000", + "x-ms-resource-type": "directory", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "6db9b566-c773-4f1c-623d-658776e8072e", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:17 GMT", + "etag": "\"0x8DAA0348AAE3164\"", + "last-modified": "Tue, 27 Sep 2022 03:01:15 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "6db9b566-c773-4f1c-623d-658776e8072e", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:15 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "3803d4c8-901e-0084-7d1d-d2a0ae000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "21dc66de-ddfe-448b-47ed-b27f94e8c077", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:18 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "21dc66de-ddfe-448b-47ed-b27f94e8c077", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212781678638", + "x-ms-request-id": "4c4b727f-201f-0028-5e1d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "37df0b10-09dd-4cd4-5a36-b96ba299910a", + "x-ms-version": "2021-06-08" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:18 GMT", + "server": "Windows-Azure-HDFS/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "37df0b10-09dd-4cd4-5a36-b96ba299910a", + "x-ms-delete-type-permanent": "false", + "x-ms-deletion-id": "133087212785510990", + "x-ms-request-id": "4c4b72b1-201f-0028-101d-d2b307000000", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.dfs.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir?recursive=false" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "2a834864-f79e-4b1a-6f9d-6e0d97aa73d2", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "undelete_dir133087212785510990trueTue, 27 Sep 2022 03:01:14 GMTTue, 27 Sep 2022 03:01:14 GMTTue, 04 Oct 2022 03:01:18 GMT0x8DAA0348A7A8ACFdirectory0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:18 GMT6undelete_dir/sub_file133087212781678638trueTue, 27 Sep 2022 03:01:15 GMTTue, 27 Sep 2022 03:01:15 GMTTue, 04 Oct 2022 03:01:18 GMT0x8DAA0348AAE3164file0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:18 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:18 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "2a834864-f79e-4b1a-6f9d-6e0d97aa73d2", + "x-ms-request-id": "3803d788-901e-0084-101d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "e4026214-e9d2-4954-7e54-e044c02c50f4", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:18 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "e4026214-e9d2-4954-7e54-e044c02c50f4", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:14 GMT", + "x-ms-request-id": "3803d86d-901e-0084-671d-d2a0ae000000", + "x-ms-resource-type": "directory", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir?comp=undelete" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "1e70a5c5-5183-432f-7e26-4f833541d032", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "undelete_dir/sub_file133087212781678638trueTue, 27 Sep 2022 03:01:15 GMTTue, 27 Sep 2022 03:01:15 GMTTue, 04 Oct 2022 03:01:18 GMT0x8DAA0348AAE3164file0application/octet-streamAAAAAAAAAAA=BlockBlobHottruetrueTue, 27 Sep 2022 03:01:18 GMT6", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:18 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "1e70a5c5-5183-432f-7e26-4f833541d032", + "x-ms-request-id": "3803d921-901e-0084-0d1d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "012ef39d-9e4b-418e-7d34-0a6ec5334788", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:19 GMT", + "etag": "\"0x8DAA0348A7A8ACF\"", + "last-modified": "Tue, 27 Sep 2022 03:01:14 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "012ef39d-9e4b-418e-7d34-0a6ec5334788", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:14 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-meta-hdi_isfolder": "true", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rwxr-x---", + "x-ms-request-id": "3803d9c6-901e-0084-281d-d2a0ae000000", + "x-ms-resource-type": "directory", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "061ab627-ac84-4754-78d7-b4e2b7d11f70", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "The specified blob does not exist.", + "STATUS_CODE": "404", + "date": "Tue, 27 Sep 2022 03:01:19 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "061ab627-ac84-4754-78d7-b4e2b7d11f70", + "x-ms-error-code": "BlobNotFound", + "x-ms-request-id": "3803da20-901e-0084-7c1d-d2a0ae000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "8655ee98-73bc-4597-63af-a1f5548ac3f9", + "x-ms-version": "2021-06-08" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:21 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "8655ee98-73bc-4597-63af-a1f5548ac3f9", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:15 GMT", + "x-ms-request-id": "03af34b7-601e-0080-121d-d22da9000000", + "x-ms-resource-type": "file", + "x-ms-version": "2021-06-08" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file?comp=undelete" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-files-datalake/12.4.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "f8ccabc0-3e10-499c-4c2c-bce7860b0aae", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:21 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "f8ccabc0-3e10-499c-4c2c-bce7860b0aae", + "x-ms-request-id": "03af374d-601e-0080-691d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?comp=list&restype=container&showonly=deleted" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "5455eddc-71a6-4548-4a9a-519a0e0f0723", + "x-ms-version": "2021-04-10" + }, + "Method": "HEAD", + "Response": { + "BODY": "", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "accept-ranges": "bytes", + "content-length": "0", + "content-type": "application/octet-stream", + "date": "Tue, 27 Sep 2022 03:01:21 GMT", + "etag": "\"0x8DAA0348AAE3164\"", + "last-modified": "Tue, 27 Sep 2022 03:01:15 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-client-request-id": "5455eddc-71a6-4548-4a9a-519a0e0f0723", + "x-ms-creation-time": "Tue, 27 Sep 2022 03:01:15 GMT", + "x-ms-group": "$superuser", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-owner": "$superuser", + "x-ms-permissions": "rw-r-----", + "x-ms-request-id": "03af3852-601e-0080-451d-d22da9000000", + "x-ms-resource-type": "file", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete/undelete_dir/sub_file" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "77c75e1a-136c-4119-7de7-70a21e03e300", + "x-ms-version": "2021-04-10" + }, + "Method": "DELETE", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:22 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "77c75e1a-136c-4119-7de7-70a21e03e300", + "x-ms-request-id": "03af390f-601e-0080-691d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net/datalakefilesystemclienttestundelete?restype=container" + } + ] +} diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.GetProperties.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.GetProperties.json new file mode 100644 index 0000000000..fd3652f422 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.GetProperties.json @@ -0,0 +1,25 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "4ab022bd-238e-47ef-6d84-52a8364d6bf5", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "1.0falsefalsefalsefalse1.0truetruetrue71.0falsefalsetrue7falsefalse2021-06-08", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:22 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "4ab022bd-238e-47ef-6d84-52a8364d6bf5", + "x-ms-request-id": "03af39bd-601e-0080-031d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" + } + ] +} diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.SetProperties.json b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.SetProperties.json new file mode 100644 index 0000000000..2b273d75e3 --- /dev/null +++ b/sdk/storage/azure-storage-files-datalake/test/ut/recordings/DataLakeServiceClientTest.SetProperties.json @@ -0,0 +1,88 @@ +{ + "networkCallRecords": [ + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "c78bf5fa-f301-4939-6993-61a1345f6318", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "1.0falsefalsefalsefalse1.0truetruetrue71.0falsefalsetrue7falsefalse2021-06-08", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:22 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "c78bf5fa-f301-4939-6993-61a1345f6318", + "x-ms-request-id": "03af3a6a-601e-0080-161d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" + }, + { + "Headers": { + "content-type": "application/xml; charset=UTF-8", + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "37898e72-ed62-4cb8-63d8-1f2efd991379", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:23 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "37898e72-ed62-4cb8-63d8-1f2efd991379", + "x-ms-request-id": "03af3b21-601e-0080-331d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" + }, + { + "Headers": { + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "90d05d28-d31a-4083-5684-5a9a119077ef", + "x-ms-version": "2021-04-10" + }, + "Method": "GET", + "Response": { + "BODY": "1.0truetruetruetrue31.0truetruetrue41.0truetruetrue4GET,PUThttp://www.example1.comx-ms-header1,x-ms-header2x-ms-header310DELETEhttp://www.example2.comx-ms-header1x-ms-header2,x-ms-header320true7falsetrueindex.html404.html2021-06-08", + "REASON_PHRASE": "OK", + "STATUS_CODE": "200", + "content-type": "application/xml", + "date": "Tue, 27 Sep 2022 03:01:33 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "transfer-encoding": "chunked", + "x-ms-client-request-id": "90d05d28-d31a-4083-5684-5a9a119077ef", + "x-ms-request-id": "03af5887-601e-0080-551d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" + }, + { + "Headers": { + "content-type": "application/xml; charset=UTF-8", + "user-agent": "azsdk-cpp-storage-blobs/12.7.0-beta.1 (Windows 10 Enterprise 6.3 22000 22000.1.amd64fre.co_release.210604-1628)", + "x-ms-client-request-id": "efa1718f-4308-489a-6e79-306563b7d7c4", + "x-ms-version": "2021-04-10" + }, + "Method": "PUT", + "Response": { + "BODY": "", + "REASON_PHRASE": "Accepted", + "STATUS_CODE": "202", + "content-length": "0", + "date": "Tue, 27 Sep 2022 03:01:34 GMT", + "server": "Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0", + "x-ms-client-request-id": "efa1718f-4308-489a-6e79-306563b7d7c4", + "x-ms-request-id": "03af5926-601e-0080-6b1d-d22da9000000", + "x-ms-version": "2021-04-10" + }, + "Url": "https://REDACTED.blob.core.windows.net?comp=properties&restype=service" + } + ] +} diff --git a/sdk/storage/test-resources.json b/sdk/storage/test-resources.json index b785415ec2..1e4c36bb03 100644 --- a/sdk/storage/test-resources.json +++ b/sdk/storage/test-resources.json @@ -207,6 +207,10 @@ "kind": "StorageV2", "properties": { "isHnsEnabled": true, + "deleteRetentionPolicy": { + "enabled": true, + "days": 7 + }, "cors": { "corsRules": [ { From cb1a62725b062ac24c849d91c7aeb5f3bfcb8fc3 Mon Sep 17 00:00:00 2001 From: microzchang <110015819+microzchang@users.noreply.github.com> Date: Thu, 29 Sep 2022 17:07:01 +0800 Subject: [PATCH 10/11] merge main becasue of conflict (#3972) --- eng/common/testproxy/target_version.txt | 2 +- .../azure-core/inc/azure/core/datetime.hpp | 47 +------------------ sdk/core/azure-core/test/ut/datetime_test.cpp | 41 ---------------- 3 files changed, 2 insertions(+), 88 deletions(-) diff --git a/eng/common/testproxy/target_version.txt b/eng/common/testproxy/target_version.txt index dedc530f84..5cc40cb14c 100644 --- a/eng/common/testproxy/target_version.txt +++ b/eng/common/testproxy/target_version.txt @@ -1 +1 @@ -1.0.0-dev.20220922.1 +1.0.0-dev.20220928.2 diff --git a/sdk/core/azure-core/inc/azure/core/datetime.hpp b/sdk/core/azure-core/inc/azure/core/datetime.hpp index 53eecb5f9f..4ed531473c 100644 --- a/sdk/core/azure-core/inc/azure/core/datetime.hpp +++ b/sdk/core/azure-core/inc/azure/core/datetime.hpp @@ -269,7 +269,7 @@ inline bool operator>=(std::chrono::system_clock::time_point const& tp, DateTime namespace Core { namespace _internal { /** - * @brief Provides conversion methods for POSIX time to an #Azure::DateTime. + * @brief Provides convertion methods for POSIX time to an #Azure::DateTime. * */ class PosixTimeConverter final { @@ -311,51 +311,6 @@ namespace Core { namespace _internal { */ ~PosixTimeConverter() = delete; }; - - /** - * @brief Provides conversion methods for Win32 FILETIME to an #Azure::DateTime. - * - */ - class Win32FileTimeConverter final { - public: - /** - * @brief Converts Win32 FILETIME to an #Azure::DateTime. - * - * @param win32Filetime The number of 100-nanoseconds since 1601-01-01. - * @return Calculated #Azure::DateTime. - */ - static DateTime Win32FileTimeToDateTime(int64_t win32Filetime) - { - auto t = DateTime(1601) + Azure::_detail::Clock::duration(win32Filetime); - return DateTime(t); - } - - /** - * @brief Converts a DateTime to Win32 FILETIME. - * - * @param dateTime The `%DateTime` to convert. - * @return The number of 100-nanoseconds since 1601-01-01. - */ - static int64_t DateTimeToWin32FileTime(DateTime const& dateTime) - { - return std::chrono::duration_cast(dateTime - DateTime(1601)) - .count(); - } - - private: - /** - * @brief An instance of `%Win32FileTimeConverter` class cannot be created. - * - */ - Win32FileTimeConverter() = delete; - - /** - * @brief An instance of `%Win32FileTimeConverter` class cannot be destructed, because no - * instance can be created. - * - */ - ~Win32FileTimeConverter() = delete; - }; }} // namespace Core::_internal } // namespace Azure diff --git a/sdk/core/azure-core/test/ut/datetime_test.cpp b/sdk/core/azure-core/test/ut/datetime_test.cpp index 02534134f2..2496819925 100644 --- a/sdk/core/azure-core/test/ut/datetime_test.cpp +++ b/sdk/core/azure-core/test/ut/datetime_test.cpp @@ -895,44 +895,3 @@ TEST(DateTime, LeapYear) EXPECT_NO_THROW(static_cast(DateTime(2021, 2, 28))); EXPECT_THROW(static_cast(DateTime(2021, 2, 29)), std::invalid_argument); } - -TEST(DateTime, Win32FileTimeConverter) -{ - using namespace Azure::Core::_internal; - - EXPECT_EQ( - Win32FileTimeConverter::DateTimeToWin32FileTime( - Win32FileTimeConverter::Win32FileTimeToDateTime(0)), - 0); - EXPECT_EQ( - Win32FileTimeConverter::DateTimeToWin32FileTime( - Win32FileTimeConverter::Win32FileTimeToDateTime(1)), - 1); - EXPECT_EQ( - Win32FileTimeConverter::DateTimeToWin32FileTime( - Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412)), - 133080315699856412); - EXPECT_EQ( - Win32FileTimeConverter::DateTimeToWin32FileTime( - Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569)), - 2650467743995784569); - - EXPECT_EQ( - Win32FileTimeConverter::Win32FileTimeToDateTime(0).ToString( - Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), - "1601-01-01T00:00:00.0000000Z"); - EXPECT_EQ( - Win32FileTimeConverter::Win32FileTimeToDateTime(1).ToString( - Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), - "1601-01-01T00:00:00.0000001Z"); - EXPECT_EQ( - Win32FileTimeConverter::Win32FileTimeToDateTime(133080315699856412) - .ToString( - Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), - "2022-09-19T03:26:09.9856412Z"); - EXPECT_EQ( - Win32FileTimeConverter::Win32FileTimeToDateTime(2650467743995784569) - .ToString( - Azure::DateTime::DateFormat::Rfc3339, Azure::DateTime::TimeFractionFormat::AllDigits), - "9999-12-31T23:59:59.5784569Z"); -} From 91480d65f4f12465822e96a3118203dbe530c33a Mon Sep 17 00:00:00 2001 From: Jinming Hu Date: Fri, 30 Sep 2022 10:23:09 +0800 Subject: [PATCH 11/11] some changes --- .../files/datalake/datalake_directory_client.hpp | 2 +- .../files/datalake/datalake_file_client.hpp | 2 +- .../storage/files/datalake/datalake_responses.hpp | 2 +- .../src/datalake_directory_client.cpp | 5 ++--- .../src/datalake_file_system_client.cpp | 11 +++++------ .../test/ut/datalake_file_system_client_test.cpp | 15 +++++++++++---- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp index b6b74c4d5d..d8d2f4defd 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp @@ -240,7 +240,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(directoryUrl), std::move(blobClient), pipeline, - customerProvidedKey) + std::move(customerProvidedKey)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp index f9aa309ef3..40dc3856b8 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp @@ -288,7 +288,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { std::move(fileUrl), std::move(blobClient), pipeline, - customerProvidedKey) + std::move(customerProvidedKey)) { } diff --git a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp index 173f274b7f..1b0e3d804a 100644 --- a/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp +++ b/sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp @@ -761,7 +761,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { : public Azure::Core::PagedResponse { public: /** - * Path items. + * Deleted path items. */ std::vector DeletedPaths; diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp index 3023758b84..ec704e1513 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_directory_client.cpp @@ -247,7 +247,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { _internal::WithReplicaStatus(context)); ListPathsPagedResponse pagedResponse; - const std::string emptyExpiresOnString = "0"; for (auto& path : response.Value.Paths) { Models::PathItem item; @@ -258,14 +257,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { item.Owner = std::move(path.Owner); item.Group = std::move(path.Group); item.Permissions = std::move(path.Permissions); - item.EncryptionScope = path.EncryptionScope; + item.EncryptionScope = std::move(path.EncryptionScope); item.ETag = std::move(path.ETag); if (path.CreatedOn.HasValue()) { item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( std::stoll(path.CreatedOn.Value())); } - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") { item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( std::stoll(path.ExpiresOn.Value())); diff --git a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp index 3ca209e5f1..5f438953a6 100644 --- a/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp +++ b/sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp @@ -277,7 +277,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { _internal::WithReplicaStatus(context)); ListPathsPagedResponse pagedResponse; - const std::string emptyExpiresOnString = "0"; for (auto& path : response.Value.Paths) { Models::PathItem item; @@ -288,14 +287,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { item.Owner = std::move(path.Owner); item.Group = std::move(path.Group); item.Permissions = std::move(path.Permissions); - item.EncryptionScope = path.EncryptionScope; + item.EncryptionScope = std::move(path.EncryptionScope); item.ETag = std::move(path.ETag); if (path.CreatedOn.HasValue()) { item.CreatedOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( std::stoll(path.CreatedOn.Value())); } - if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != emptyExpiresOnString) + if (path.ExpiresOn.HasValue() && path.ExpiresOn.Value() != "0") { item.ExpiresOn = _detail::Win32FileTimeConverter::Win32FileTimeToDateTime( std::stoll(path.ExpiresOn.Value())); @@ -474,8 +473,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { { pathDeletedItem.Name = std::move(item.Name.Content); } - pathDeletedItem.DeletedOn = item.Details.DeletedOn.Value(); - pathDeletedItem.DeletionId = item.DeletionId.Value(); + pathDeletedItem.DeletedOn = std::move(item.Details.DeletedOn.Value()); + pathDeletedItem.DeletionId = std::move(item.DeletionId.Value()); pathDeletedItem.RemainingRetentionDays = item.Details.RemainingRetentionDays.Value(); pagedResponse.DeletedPaths.push_back(std::move(pathDeletedItem)); @@ -483,7 +482,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { pagedResponse.m_operationOptions = options; pagedResponse.m_fileSystemClient = std::make_shared(*this); pagedResponse.CurrentPageToken = options.ContinuationToken.ValueOr(std::string()); - pagedResponse.NextPageToken = result.Value.ContinuationToken; + pagedResponse.NextPageToken = std::move(result.Value.ContinuationToken); pagedResponse.RawResponse = std::move(result.RawResponse); return pagedResponse; diff --git a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp index 37446f48b2..5278d72e59 100644 --- a/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp +++ b/sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_system_client_test.cpp @@ -866,7 +866,11 @@ namespace Azure { namespace Storage { namespace Test { { std::vector paths; - EXPECT_NO_THROW(paths = std::move(m_fileSystemClient->ListDeletedPaths().DeletedPaths)); + for (auto pageResult = m_fileSystemClient->ListDeletedPaths(); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); + } EXPECT_EQ(2, paths.size()); EXPECT_EQ(deletedDirectoryName, paths[0].Name); EXPECT_EQ(deletedFilename, paths[1].Name); @@ -880,7 +884,7 @@ namespace Azure { namespace Storage { namespace Test { pageResult.MoveToNextPage()) { paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); - EXPECT_EQ(1, pageResult.DeletedPaths.size()); + EXPECT_LE(pageResult.DeletedPaths.size(), 1); } EXPECT_EQ(2, paths.size()); } @@ -898,8 +902,11 @@ namespace Azure { namespace Storage { namespace Test { Files::DataLake::ListDeletedPathsOptions options; options.Prefix = directoryName; std::vector paths; - EXPECT_NO_THROW( - paths = std::move(m_fileSystemClient->ListDeletedPaths(options).DeletedPaths)); + for (auto pageResult = m_fileSystemClient->ListDeletedPaths(options); pageResult.HasPage(); + pageResult.MoveToNextPage()) + { + paths.insert(paths.end(), pageResult.DeletedPaths.begin(), pageResult.DeletedPaths.end()); + } EXPECT_EQ(1, paths.size()); EXPECT_EQ(directoryName + "/" + filename, paths[0].Name); }