Merge feature/datalakestg82

sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp

@@ -338,6 +338,7 @@ namespace Azure { namespace Storage { namespace Blobs {
     friend class BlobServiceClient;
     friend class BlobLeaseClient;
     friend class BlobContainerBatch;
+    friend class Files::DataLake::DataLakeFileSystemClient;
   };
 
 }}} // namespace Azure::Storage::Blobs

sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp

@@ -1365,6 +1365,10 @@ namespace Azure { namespace Storage { namespace Blobs {
          * Type of the blob.
          */
         Models::BlobType BlobType;
+        /**
+         * The deletion ID associated with the deleted path.
+         */
+        Nullable<std::string> DeletionId;
       };
     } // namespace _detail
     /**
@@ -3447,6 +3451,7 @@ namespace Azure { namespace Storage { namespace Blobs {
         Nullable<std::string> Marker;
         Nullable<int32_t> MaxResults;
         Nullable<Models::ListBlobsIncludeFlags> Include;
+        Nullable<std::string> ShowOnly;
       };
       static Response<Models::_detail::ListBlobsByHierarchyResult> ListBlobsByHierarchy(
           Core::Http::_internal::HttpPipeline& pipeline,

sdk/storage/azure-storage-blobs/src/rest_client.cpp

@@ -2310,6 +2310,7 @@ namespace Azure { namespace Storage { namespace Blobs {
           kHasVersionsOnly,
           kContentLength,
           kBlobType,
+          kDeletionId,
         };
         const std::unordered_map<std::string, XmlTagEnum> XmlTagEnumMap{
             {"EnumerationResults", XmlTagEnum::kEnumerationResults},
@@ -2370,6 +2371,7 @@ namespace Azure { namespace Storage { namespace Blobs {
             {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly},
             {"Content-Length", XmlTagEnum::kContentLength},
             {"BlobType", XmlTagEnum::kBlobType},
+            {"DeletionId", XmlTagEnum::kDeletionId},
         };
         std::vector<XmlTagEnum> xmlPath;
         Models::_detail::BlobItem vectorElement1;
@@ -2807,6 +2809,13 @@ namespace Azure { namespace Storage { namespace Blobs {
             {
               vectorElement1.BlobType = Models::BlobType(node.Value);
             }
+            else if (
+                xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
+                && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob
+                && xmlPath[3] == XmlTagEnum::kDeletionId)
+            {
+              vectorElement1.DeletionId = node.Value;
+            }
           }
           else if (node.Type == _internal::XmlNodeType::Attribute)
           {
@@ -2908,6 +2917,11 @@ namespace Azure { namespace Storage { namespace Blobs {
                 ListBlobsIncludeFlagsToString(options.Include.Value())));
       }
       request.SetHeader("x-ms-version", "2021-04-10");
+      if (options.ShowOnly.HasValue() && !options.ShowOnly.Value().empty())
+      {
+        request.GetUrl().AppendQueryParameter(
+            "showonly", _internal::UrlEncodeQueryParameter(options.ShowOnly.Value()));
+      }
       auto pRawResponse = pipeline.Send(request, context);
       auto httpStatusCode = pRawResponse->GetStatusCode();
       if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@@ -2981,6 +2995,7 @@ namespace Azure { namespace Storage { namespace Blobs {
           kHasVersionsOnly,
           kContentLength,
           kBlobType,
+          kDeletionId,
           kBlobPrefix,
         };
         const std::unordered_map<std::string, XmlTagEnum> XmlTagEnumMap{
@@ -3043,6 +3058,7 @@ namespace Azure { namespace Storage { namespace Blobs {
             {"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly},
             {"Content-Length", XmlTagEnum::kContentLength},
             {"BlobType", XmlTagEnum::kBlobType},
+            {"DeletionId", XmlTagEnum::kDeletionId},
             {"BlobPrefix", XmlTagEnum::kBlobPrefix},
         };
         std::vector<XmlTagEnum> xmlPath;
@@ -3488,6 +3504,13 @@ namespace Azure { namespace Storage { namespace Blobs {
             {
               vectorElement1.BlobType = Models::BlobType(node.Value);
             }
+            else if (
+                xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
+                && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob
+                && xmlPath[3] == XmlTagEnum::kDeletionId)
+            {
+              vectorElement1.DeletionId = node.Value;
+            }
             else if (
                 xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
                 && xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlobPrefix
@@ -5573,9 +5596,15 @@ namespace Azure { namespace Storage { namespace Blobs {
         response.LeaseDuration
             = Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration"));
       }
-      response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state"));
-      response.LeaseStatus
-          = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status"));
+      if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0)
+      {
+        response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state"));
+      }
+      if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0)
+      {
+        response.LeaseStatus
+            = Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status"));
+      }
       response.IsServerEncrypted
           = pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true");
       return Response<Models::QueryBlobResult>(std::move(response), std::move(pRawResponse));

sdk/storage/azure-storage-blobs/swagger/README.md

@@ -273,6 +273,15 @@ directive:
           {"value": "legalhold", "name": "LegalHold"},
           {"value": "deletedwithversions", "name": "DeletedWithVersions"}
       ];
+      $["ListBlobsShowOnly"]= {
+        "name": "showonly",
+        "x-ms-client-name": "ShowOnly",
+        "in": "query",
+        "required": false,
+        "type": "string",
+        "x-ms-parameter-location": "method",
+        "description": "Include this parameter to specify one or more datasets to include in the response."
+      };
       $.DeleteSnapshots["x-ms-enum"]["name"] = "DeleteSnapshotsOption";
       $.DeleteSnapshots["x-ms-enum"]["values"] = [{"value": "include", "name": "IncludeSnapshots"},{"value":"only", "name": "OnlySnapshots"}];
       $.BlobExpiryOptions["x-ms-enum"]["name"] = "ScheduleBlobExpiryOriginType";
@@ -829,6 +838,7 @@ directive:
       $.BlobItemInternal.properties["BlobType"] = $.BlobPropertiesInternal.properties["BlobType"];
       $.BlobItemInternal.properties["BlobType"]["x-ms-xml"] = {"name": "Properties/BlobType"};
       delete $.BlobPropertiesInternal.properties["BlobType"];
+      $.BlobItemInternal.properties["DeletionId"] = {"type": "string"};
       $.BlobItemInternal.required.push("BlobType", "BlobSize");
       $.BlobItemInternal.properties["Name"].description = "Blob name.";
       $.BlobItemInternal.properties["Deleted"].description = "Indicates whether this blob was deleted.";
@@ -837,6 +847,7 @@ directive:
       $.BlobItemInternal.properties["IsCurrentVersion"].description = "Indicates if this is the current version of the blob.";
       $.BlobItemInternal.properties["BlobType"].description = "Type of the blob.";
       $.BlobItemInternal.properties["HasVersionsOnly"].description = "Indicates that this root blob has been deleted, but it has versions that are active.";
+      $.BlobItemInternal.properties["DeletionId"].description = "The deletion ID associated with the deleted path.";
 
       $.BlobPropertiesInternal.properties["Etag"]["x-ms-client-name"] = "ETag";
       $.BlobPropertiesInternal["x-ms-client-name"] = "BlobItemDetails";
@@ -916,6 +927,10 @@ directive:
       delete $.ListBlobsHierarchySegmentResponse.properties["Segment"];
       delete $.ListBlobsHierarchySegmentResponse.required;
       $.ListBlobsHierarchySegmentResponse.properties["NextMarker"]["x-nullable"] = true;
+  - from: swagger-document
+    where: $["x-ms-paths"]["/{containerName}?restype=container&comp=list&hierarchy"].get.parameters
+    transform: >
+      $.push({"$ref": "#/parameters/ListBlobsShowOnly"});
 ```
 
 ### DownloadBlob
@@ -1312,6 +1327,10 @@ directive:
         delete $[status_code].headers["x-ms-blob-content-md5"];
         delete $[status_code].headers["x-ms-content-crc64"];
         $[status_code].headers["x-ms-lease-duration"]["x-nullable"] = true;
+        $[status_code].headers["x-ms-lease-state"]["x-ms-client-default"] = "";
+        $[status_code].headers["x-ms-lease-state"]["x-nullable"] = true;
+        $[status_code].headers["x-ms-lease-status"]["x-ms-client-default"] = "";
+        $[status_code].headers["x-ms-lease-status"]["x-nullable"] = true;
       }
 ```
 

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp

@@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     explicit DataLakeDirectoryClient(
         Azure::Core::Url directoryUrl,
         Blobs::BlobClient blobClient,
-        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
-        : DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline)
+        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
+        Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
+        : DataLakePathClient(
+            std::move(directoryUrl),
+            std::move(blobClient),
+            pipeline,
+            std::move(customerProvidedKey))
     {
     }
 

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp

@@ -265,12 +265,30 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         const ScheduleFileDeletionOptions& options = ScheduleFileDeletionOptions(),
         const Azure::Core::Context& context = Azure::Core::Context()) const;
 
+    /**
+     * @brief Returns the result of a query against the file.
+     *
+     * @param querySqlExpression The query expression in SQL.
+     * @param options Optional parameters to execute this function.
+     * @param context Context for cancelling long running operations.
+     * @return A QueryFileResult describing the query result.
+     */
+    Azure::Response<Models::QueryFileResult> Query(
+        const std::string& querySqlExpression,
+        const QueryFileOptions& options = QueryFileOptions(),
+        const Azure::Core::Context& context = Azure::Core::Context()) const;
+
   private:
     explicit DataLakeFileClient(
         Azure::Core::Url fileUrl,
         Blobs::BlobClient blobClient,
-        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
-        : DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline)
+        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
+        Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
+        : DataLakePathClient(
+            std::move(fileUrl),
+            std::move(blobClient),
+            pipeline,
+            std::move(customerProvidedKey))
     {
     }
 

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp

@@ -239,17 +239,46 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         const RenameDirectoryOptions& options = RenameDirectoryOptions(),
         const Azure::Core::Context& context = Azure::Core::Context()) const;
 
+    /**
+     * @brief Gets the paths that have recently been soft deleted in this file system.
+     * @param options Optional parameters to list deleted paths.
+     * @param context Context for cancelling long running operations.
+     * @return Azure::Response<DataLakePathClient> The client targets the restored path.
+     * @remark This request is sent to Blob endpoint.
+     */
+    ListDeletedPathsPagedResponse ListDeletedPaths(
+        const ListDeletedPathsOptions& options = ListDeletedPathsOptions(),
+        const Azure::Core::Context& context = Azure::Core::Context()) const;
+
+    /**
+     * @brief Restores a soft deleted path.
+     * @param deletedPath The path of the deleted path.
+     * @param deletionId The deletion ID associated with the soft deleted path. You can get soft
+     * deleted paths and their associated deletion IDs with ListDeletedPaths.
+     * @param context Context for cancelling long running operations.
+     * @return Azure::Response<DataLakePathClient> The client targets the restored path.
+     * @remark This request is sent to Blob endpoint.
+     */
+    Azure::Response<DataLakePathClient> UndeletePath(
+        const std::string& deletedPath,
+        const std::string& deletionId,
+        const UndeletePathOptions& options = UndeletePathOptions(),
+        const Azure::Core::Context& context = Azure::Core::Context()) const;
+
   private:
     Azure::Core::Url m_fileSystemUrl;
     Blobs::BlobContainerClient m_blobContainerClient;
     std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
+    Azure::Nullable<EncryptionKey> m_customerProvidedKey;
 
     explicit DataLakeFileSystemClient(
         Azure::Core::Url fileSystemUrl,
         Blobs::BlobContainerClient blobContainerClient,
-        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
+        std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
+        Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
         : m_fileSystemUrl(std::move(fileSystemUrl)),
-          m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline))
+          m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)),
+          m_customerProvidedKey(std::move(customerProvidedKey))
     {
     }
     friend class DataLakeLeaseClient;