Storage April Release (#4513)

* Stg87 protocol layer (#4374)

* storage stg87 protocol layer

* Storage/feature/stg87-hns encryption context (#4383)

* hns_encryption_context

* Storage/feature/stg87-trailing dot (#4389)

 trailing dot

* Storage/feature/stg87-File OAuth (#4413)

* file oauth

* enable some stg87 tests

sdk/storage/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "cpp",
   "TagPrefix": "cpp/storage",
-  "Tag": "cpp/storage_5f1f86318a"
+  "Tag": "cpp/storage_53d9e49e60"
 }

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_options.hpp

@@ -634,6 +634,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
      * Specify the access condition for the path.
      */
     PathAccessConditions AccessConditions;
+
+    /**
+     * Encryption context of the file. Encryption context is metadata that is not encrypted when
+     * stored on the file. The primary application of this field is to store non-encrypted data that
+     * can be used to derive the customer-provided key for a file.
+     * Not applicable for directories.
+     */
+    Azure::Nullable<std::string> EncryptionContext;
   };
 
   /**

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_responses.hpp

@@ -266,6 +266,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        */
       Nullable<std::string> EncryptionScope;
 
+      /**
+       * Encryption context of the file. Encryption context is metadata that is not encrypted when
+       * stored on the file. The primary application of this field is to store non-encrypted data
+       * that can be used to derive the customer-provided key for a file.
+       * Not applicable for directories.
+       */
+      Nullable<std::string> EncryptionContext;
+
       /**
        * The creation time of the path.
        */
@@ -349,6 +357,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        */
       Nullable<std::string> EncryptionScope;
 
+      /**
+       * Encryption context of the file. Encryption context is metadata that is not encrypted when
+       * stored on the file. The primary application of this field is to store non-encrypted data
+       * that can be used to derive the customer-provided key for a file.
+       * Not applicable for directories.
+       */
+      Nullable<std::string> EncryptionContext;
+
       /**
        * The copy ID of the path, if the path is created from a copy operation.
        */
@@ -664,6 +680,14 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
        * The encryption scope.
        */
       Azure::Nullable<std::string> EncryptionScope;
+
+      /*
+       * Encryption context of the file. Encryption context is metadata that is not encrypted when
+       * stored on the file. The primary application of this field is to store non-encrypted data
+       * that can be used to derive the customer-provided key for a file.
+       * Not applicable for directories.
+       */
+      Nullable<std::string> EncryptionContext;
     };
 
     /**

sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/rest_client.hpp

@@ -99,6 +99,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         Nullable<std::string> EncryptionScope;
         Nullable<std::string> CreatedOn;
         Nullable<std::string> ExpiresOn;
+        Nullable<std::string> EncryptionContext;
         std::string ETag;
       };
       /**
@@ -412,6 +413,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         Nullable<int64_t> LeaseDuration;
         Nullable<std::string> ExpiryOptions;
         Nullable<std::string> ExpiresOn;
+        Nullable<std::string> EncryptionContext;
       };
       static Response<Models::CreatePathResult> Create(
           Core::Http::_internal::HttpPipeline& pipeline,

sdk/storage/azure-storage-files-datalake/src/datalake_file_client.cpp

@@ -211,6 +211,9 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     ret.Details.EncryptionKeySha256 = std::move(response.Value.Details.EncryptionKeySha256);
     ret.Details.EncryptionScope = std::move(response.Value.Details.EncryptionScope);
     ret.Details.IsServerEncrypted = response.Value.Details.IsServerEncrypted;
+    ret.Details.EncryptionContext
+        = Azure::Core::Http::_internal::HttpShared::GetHeaderOrEmptyString(
+            response.RawResponse->GetHeaders(), _detail::EncryptionContextHeaderName);
     return Azure::Response<Models::DownloadFileResult>(
         std::move(ret), std::move(response.RawResponse));
   }

sdk/storage/azure-storage-files-datalake/src/datalake_file_system_client.cpp

@@ -292,6 +292,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
         item.Group = std::move(path.Group);
         item.Permissions = std::move(path.Permissions);
         item.EncryptionScope = std::move(path.EncryptionScope);
+        item.EncryptionContext = std::move(path.EncryptionContext);
         item.ETag = std::move(path.ETag);
         if (path.CreatedOn.HasValue())
         {

sdk/storage/azure-storage-files-datalake/src/datalake_path_client.cpp

@@ -208,6 +208,7 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     protocolLayerOptions.Owner = options.Owner;
     protocolLayerOptions.Group = options.Group;
     protocolLayerOptions.ProposedLeaseId = options.LeaseId;
+    protocolLayerOptions.EncryptionContext = options.EncryptionContext;
     if (options.Acls.HasValue())
     {
       protocolLayerOptions.Acl = Models::Acl::SerializeAcls(options.Acls.Value());
@@ -341,6 +342,8 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
     ret.VersionId = std::move(response.Value.VersionId);
     ret.IsCurrentVersion = std::move(response.Value.IsCurrentVersion);
     ret.IsDirectory = _detail::MetadataIncidatesIsDirectory(ret.Metadata);
+    ret.EncryptionContext = Azure::Core::Http::_internal::HttpShared::GetHeaderOrEmptyString(
+        response.RawResponse->GetHeaders(), _detail::EncryptionContextHeaderName);
     return Azure::Response<Models::PathProperties>(std::move(ret), std::move(response.RawResponse));
   }
 

sdk/storage/azure-storage-files-datalake/src/private/datalake_constants.hpp

@@ -11,5 +11,6 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake { nam
   constexpr static const char* DataLakePathNotFound = "PathNotFound";
   constexpr static const char* DataLakePathAlreadyExists = "PathAlreadyExists";
   constexpr static const char* DataLakeIsDirectoryKey = "hdi_isFolder";
+  constexpr static const char* EncryptionContextHeaderName = "x-ms-encryption-context";
 
 }}}}} // namespace Azure::Storage::Files::DataLake::_detail

sdk/storage/azure-storage-files-datalake/src/rest_client.cpp

@@ -126,6 +126,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
           {
             vectorElement2.ExpiresOn = var0["expiryTime"].get<std::string>();
           }
+          if (var0.count("EncryptionContext") != 0)
+          {
+            vectorElement2.EncryptionContext = var0["EncryptionContext"].get<std::string>();
+          }
           if (var0.count("etag") != 0)
           {
             vectorElement2.ETag = var0["etag"].get<std::string>();
@@ -297,6 +301,10 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
       {
         request.SetHeader("x-ms-expiry-time", options.ExpiresOn.Value());
       }
+      if (options.EncryptionContext.HasValue() && !options.EncryptionContext.Value().empty())
+      {
+        request.SetHeader("x-ms-encryption-context", options.EncryptionContext.Value());
+      }
       auto pRawResponse = pipeline.Send(request, context);
       auto httpStatusCode = pRawResponse->GetStatusCode();
       if (httpStatusCode != Core::Http::HttpStatusCode::Created)

sdk/storage/azure-storage-files-datalake/swagger/README.md

@@ -234,6 +234,7 @@ directive:
       $.Path.properties["contentLength"]["x-ms-client-name"] = "FileSize";
       $.Path.properties["isDirectory"]["x-ms-client-default"] = false;
       $.Path.properties["EncryptionScope"]["x-nullable"] = true;
+      $.Path.properties["EncryptionContext"]["x-nullable"] = true;
       $.Path.properties["creationTime"] = {"type": "string", "x-ms-client-name": "CreatedOn", "x-nullable": true};
       $.Path.properties["expiryTime"] = {"type": "string", "x-ms-client-name": "ExpiresOn", "x-nullable": true};
       $.Path.properties["etag"] = {"type": "string", "x-ms-format": "string", "x-ms-client-default": "", "x-ms-client-name": "ETag"};

sdk/storage/azure-storage-files-datalake/test/ut/datalake_file_client_test.cpp

@@ -466,6 +466,34 @@ namespace Azure { namespace Storage { namespace Test {
     }
   }
 
+  TEST_F(DataLakeFileClientTest, CreateWithEncryptionContext)
+  {
+    std::string encryptionContext = "encryptionContext";
+    const std::string fileName = RandomString();
+    auto fileClient = m_fileSystemClient->GetFileClient(fileName);
+    Files::DataLake::CreateFileOptions options;
+    options.EncryptionContext = encryptionContext;
+    // Assert Create
+    EXPECT_NO_THROW(fileClient.Create(options));
+    // Assert GetProperties
+    auto properties = fileClient.GetProperties();
+    EXPECT_TRUE(properties.Value.EncryptionContext.HasValue());
+    EXPECT_EQ(encryptionContext, properties.Value.EncryptionContext.Value());
+    // Assert Download
+    auto downloadResult = fileClient.Download();
+    EXPECT_TRUE(downloadResult.Value.Details.EncryptionContext.HasValue());
+    EXPECT_EQ(encryptionContext, downloadResult.Value.Details.EncryptionContext.Value());
+    // Assert ListPaths
+    auto paths = m_fileSystemClient->ListPaths(false).Paths;
+    auto iter = std::find_if(
+        paths.begin(), paths.end(), [&fileName](const Files::DataLake::Models::PathItem& path) {
+          return path.Name == fileName;
+        });
+    EXPECT_NE(paths.end(), iter);
+    EXPECT_TRUE(iter->EncryptionContext.HasValue());
+    EXPECT_EQ(encryptionContext, iter->EncryptionContext.Value());
+  }
+
   TEST_F(DataLakeFileClientTest, FileReadReturns)
   {
     const int32_t bufferSize = 20;