[Storage] [DataMovement] [Shares] Preserve Permission Key (#44696)

* WIP * Preserve Single Transfer Permissions; Rerecorded failing tests * Cleanup * Preserve permission key defaults to false * PR Comments * Removed unnecessary ResourceProperties in ShareDirectoryResource; Fixed LastWritten values; preserve permissions * Rerecord tests * Revert assests.json to main * Rerecord tests * Rerecord test to hopefully resolve merge conflict assets tag * Added end to end tests for short permission values
Azure · Jul 3, 2024 · b413fa5 · b413fa5
1 parent e8c05c3
commit b413fa5
Show file tree

Hide file tree

Showing 56 changed files with 1,834 additions and 264 deletions.
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json b/sdk/storage/Azure.Storage.DataMovement.Blobs/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Blobs",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_0523c237f1"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Blobs_db329f90cd"
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs
@@ -325,5 +325,18 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
                 metadata:_options?.Metadata,
                 tags: default);
         }
+
+        // no-op for get permissions
+        protected override Task<string> GetPermissionsAsync(
+            StorageResourceItemProperties properties = default,
+            CancellationToken cancellationToken = default)
+            => Task.FromResult((string)default);
+
+        // no-op for set permissions
+        protected override Task SetPermissionsAsync(
+            StorageResourceItem sourceResource,
+            StorageResourceItemProperties sourceProperties,
+            CancellationToken cancellationToken = default)
+            => Task.CompletedTask;
     }
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs
@@ -157,6 +157,7 @@ private StorageResourceItem GetBlobAsStorageResource(
         /// </summary>
         /// <returns>List of the child resources in the storage container.</returns>
         protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAsync(
+            StorageResourceContainer destinationContainer = default,
             [EnumeratorCancellation] CancellationToken cancellationToken = default)
         {
             // Suffix the backwards slash when searching if there's a prefix specified,

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs
@@ -378,5 +378,18 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
                 metadata: _options?.Metadata,
                 tags: default);
         }
+
+        // no-op for get permissions
+        protected override Task<string> GetPermissionsAsync(
+            StorageResourceItemProperties properties = default,
+            CancellationToken cancellationToken = default)
+            => Task.FromResult((string)default);
+
+        // no-op for set permissions
+        protected override Task SetPermissionsAsync(
+            StorageResourceItem sourceResource,
+            StorageResourceItemProperties sourceProperties,
+            CancellationToken cancellationToken = default)
+            => Task.CompletedTask;
     }
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs
@@ -336,5 +336,18 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
                 metadata: _options?.Metadata,
                 tags: default);
         }
+
+        // no-op for get permissions
+        protected override Task<string> GetPermissionsAsync(
+            StorageResourceItemProperties properties = default,
+            CancellationToken cancellationToken = default)
+            => Task.FromResult((string)default);
+
+        // no-op for set permissions
+        protected override Task SetPermissionsAsync(
+            StorageResourceItem sourceResource,
+            StorageResourceItemProperties sourceProperties,
+            CancellationToken cancellationToken = default)
+            => Task.CompletedTask;
     }
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobStartTransferCopyTests.cs
@@ -238,7 +238,6 @@ protected override async Task VerifyPropertiesCopyAsync(
                 BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
 
                 Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
                 Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
                 Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToPageBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/AppendBlobToPageBlobTests.cs
@@ -257,7 +257,6 @@ protected override async Task VerifyPropertiesCopyAsync(
                 BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
 
                 Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
                 Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
                 Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobContainerServiceToServiceJobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobContainerServiceToServiceJobTests.cs
@@ -108,7 +108,9 @@ await checkpointer.AddNewJobAsync(
                 GetMockBlockBlobResource("blob2").Object,
                 GetMockBlockBlobResource("blob3").Object,
             };
-            sourceMock.Setup(r => r.GetStorageResourcesAsync(It.IsAny<CancellationToken>()))
+            sourceMock.Setup(r => r.GetStorageResourcesAsync(
+                It.IsAny<StorageResourceContainer>(),
+                It.IsAny<CancellationToken>()))
                 .Returns(GetStorageResourceItemsAsyncEnumerable(blobItems));
             ServiceToServiceTransferJob transferJob = new ServiceToServiceTransferJob(
                 new DataTransfer(
@@ -165,7 +167,9 @@ await checkpointer.AddNewJobAsync(
                 GetMockAppendBlobResource("appendblob2").Object,
                 GetMockBlockBlobResource("blockblob3").Object,
             };
-            sourceMock.Setup(r => r.GetStorageResourcesAsync(It.IsAny<CancellationToken>()))
+            sourceMock.Setup(r => r.GetStorageResourcesAsync(
+                It.IsAny<StorageResourceContainer>(),
+                It.IsAny<CancellationToken>()))
                 .Returns(GetStorageResourceItemsAsyncEnumerable(blobItems));
             ServiceToServiceTransferJob transferJob = new ServiceToServiceTransferJob(
                 new DataTransfer(

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToAppendBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlockBlobToAppendBlobTests.cs
@@ -245,7 +245,6 @@ protected override async Task VerifyPropertiesCopyAsync(
                 BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
 
                 Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
                 Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
                 Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobStartTransferCopyTests.cs
@@ -240,7 +240,6 @@ protected override async Task VerifyPropertiesCopyAsync(
                 BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
 
                 Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
                 Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
                 Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToAppendBlobTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/PageBlobToAppendBlobTests.cs
@@ -254,7 +254,6 @@ protected override async Task VerifyPropertiesCopyAsync(
                 BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync();
 
                 Assert.That(_defaultMetadata, Is.EqualTo(destinationProperties.Metadata));
-                Assert.AreEqual(_defaultAccessTier.ToString(), destinationProperties.AccessTier);
                 Assert.AreEqual(_defaultContentDisposition, destinationProperties.ContentDisposition);
                 Assert.AreEqual(_defaultContentLanguage, destinationProperties.ContentLanguage);
                 Assert.AreEqual(_defaultCacheControl, destinationProperties.CacheControl);

diff --git a/...taMovement.Files.Shares/BlobToFileSharesTests/AppendBlobDirectoryToShareDirectoryTests.cs b/...taMovement.Files.Shares/BlobToFileSharesTests/AppendBlobDirectoryToShareDirectoryTests.cs
@@ -128,7 +128,10 @@ private async Task UploadAppendBlocksAsync(
         protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestShareAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            ShareClient containerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
         {
             // Authorize with SAS when performing operations
             if (containerClient.CanGenerateSasUri)
@@ -171,7 +174,13 @@ protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient d
             Assert.IsEmpty(items);
         }
 
-        protected override async Task VerifyResultsAsync(BlobContainerClient sourceContainer, string sourcePrefix, ShareClient destinationContainer, string destinationPrefix, CancellationToken cancellationToken = default)
+        protected override async Task VerifyResultsAsync(
+            BlobContainerClient sourceContainer,
+            string sourcePrefix,
+            ShareClient destinationContainer,
+            string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
+            CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
 

diff --git a/...ataMovement.Files.Shares/BlobToFileSharesTests/BlockBlobDirectoryToShareDirectoryTests.cs b/...ataMovement.Files.Shares/BlobToFileSharesTests/BlockBlobDirectoryToShareDirectoryTests.cs
@@ -108,7 +108,10 @@ await blobClient.UploadAsync(
         protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestShareAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            ShareClient containerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
         {
             // Authorize with SAS when performing operations
             if (containerClient.CanGenerateSasUri)
@@ -151,7 +154,13 @@ protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient d
             Assert.IsEmpty(items);
         }
 
-        protected override async Task VerifyResultsAsync(BlobContainerClient sourceContainer, string sourcePrefix, ShareClient destinationContainer, string destinationPrefix, CancellationToken cancellationToken = default)
+        protected override async Task VerifyResultsAsync(
+            BlobContainerClient sourceContainer,
+            string sourcePrefix,
+            ShareClient destinationContainer,
+            string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
+            CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
 

diff --git a/...DataMovement.Files.Shares/BlobToFileSharesTests/PageBlobDirectoryToShareDirectoryTests.cs b/...DataMovement.Files.Shares/BlobToFileSharesTests/PageBlobDirectoryToShareDirectoryTests.cs
@@ -128,7 +128,10 @@ private async Task UploadPagesAsync(
         protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestShareAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            ShareClient containerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
         {
             // Authorize with SAS when performing operations
             if (containerClient.CanGenerateSasUri)
@@ -171,7 +174,13 @@ protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient d
             Assert.IsEmpty(items);
         }
 
-        protected override async Task VerifyResultsAsync(BlobContainerClient sourceContainer, string sourcePrefix, ShareClient destinationContainer, string destinationPrefix, CancellationToken cancellationToken = default)
+        protected override async Task VerifyResultsAsync(
+            BlobContainerClient sourceContainer,
+            string sourcePrefix,
+            ShareClient destinationContainer,
+            string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
+            CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
 

diff --git a/...taMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToAppendBlobDirectoryTests.cs b/...taMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToAppendBlobDirectoryTests.cs
@@ -136,7 +136,10 @@ protected override async Task CreateObjectInSourceAsync(
         protected override async Task<IDisposingContainer<BlobContainerClient>> GetDestinationDisposingContainerAsync(BlobServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestContainerAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(BlobContainerClient sourceContainerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            BlobContainerClient sourceContainerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
             => new BlobStorageResourceContainer(sourceContainerClient, new BlobStorageResourceContainerOptions() { BlobDirectoryPrefix = directoryPath, BlobType = new(BlobType.Append) });
 
         protected override BlobContainerClient GetOAuthDestinationContainerClient(string containerName)
@@ -184,6 +187,7 @@ protected override async Task VerifyResultsAsync(
             string sourcePrefix,
             BlobContainerClient destinationContainer,
             string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

diff --git a/...ataMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToBlockBlobDirectoryTests.cs b/...ataMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToBlockBlobDirectoryTests.cs
@@ -117,7 +117,10 @@ protected override async Task CreateObjectInSourceAsync(
         protected override async Task<IDisposingContainer<BlobContainerClient>> GetDestinationDisposingContainerAsync(BlobServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestContainerAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(BlobContainerClient sourceContainerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            BlobContainerClient sourceContainerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
             => new BlobStorageResourceContainer(sourceContainerClient, new BlobStorageResourceContainerOptions() { BlobDirectoryPrefix = directoryPath, BlobType = new(BlobType.Block) });
 
         protected override BlobContainerClient GetOAuthDestinationContainerClient(string containerName)
@@ -165,6 +168,7 @@ protected override async Task VerifyResultsAsync(
             string sourcePrefix,
             BlobContainerClient destinationContainer,
             string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

diff --git a/...DataMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToPageBlobDirectoryTests.cs b/...DataMovement.Files.Shares/BlobToFileSharesTests/ShareDirectoryToPageBlobDirectoryTests.cs
@@ -137,7 +137,10 @@ protected override async Task CreateObjectInSourceAsync(
         protected override async Task<IDisposingContainer<BlobContainerClient>> GetDestinationDisposingContainerAsync(BlobServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
             => await DestinationClientBuilder.GetTestContainerAsync(service, containerName);
 
-        protected override StorageResourceContainer GetDestinationStorageResourceContainer(BlobContainerClient sourceContainerClient, string directoryPath)
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(
+            BlobContainerClient sourceContainerClient,
+            string directoryPath,
+            TransferPropertiesTestType propertiesTestType = default)
             => new BlobStorageResourceContainer(sourceContainerClient, new BlobStorageResourceContainerOptions() { BlobDirectoryPrefix = directoryPath, BlobType = new(BlobType.Page) });
 
         protected override BlobContainerClient GetOAuthDestinationContainerClient(string containerName)
@@ -185,6 +188,7 @@ protected override async Task VerifyResultsAsync(
             string sourcePrefix,
             BlobContainerClient destinationContainer,
             string destinationPrefix,
+            TransferPropertiesTestType propertiesTestType = default,
             CancellationToken cancellationToken = default)
         {
             CancellationHelper.ThrowIfCancellationRequested(cancellationToken);

diff --git a/...e.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs b/...e.Storage.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.net6.0.cs
@@ -41,8 +41,7 @@ public ShareFileStorageResourceOptions() { }
         public Azure.Storage.DataMovement.DataTransferProperty<System.DateTimeOffset?> FileCreatedOn { get { throw null; } set { } }
         public Azure.Storage.DataMovement.DataTransferProperty<System.DateTimeOffset?> FileLastWrittenOn { get { throw null; } set { } }
         public Azure.Storage.DataMovement.DataTransferProperty<System.Collections.Generic.IDictionary<string, string>> FileMetadata { get { throw null; } set { } }
-        public string FilePermissionKey { get { throw null; } set { } }
-        public string FilePermissions { get { throw null; } set { } }
+        public Azure.Storage.DataMovement.DataTransferProperty FilePermissions { get { throw null; } set { } }
         public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
     }
 }

diff --git a/...e.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs b/...e.DataMovement.Files.Shares/api/Azure.Storage.DataMovement.Files.Shares.netstandard2.0.cs
@@ -41,8 +41,7 @@ public ShareFileStorageResourceOptions() { }
         public Azure.Storage.DataMovement.DataTransferProperty<System.DateTimeOffset?> FileCreatedOn { get { throw null; } set { } }
         public Azure.Storage.DataMovement.DataTransferProperty<System.DateTimeOffset?> FileLastWrittenOn { get { throw null; } set { } }
         public Azure.Storage.DataMovement.DataTransferProperty<System.Collections.Generic.IDictionary<string, string>> FileMetadata { get { throw null; } set { } }
-        public string FilePermissionKey { get { throw null; } set { } }
-        public string FilePermissions { get { throw null; } set { } }
+        public Azure.Storage.DataMovement.DataTransferProperty FilePermissions { get { throw null; } set { } }
         public Azure.Storage.Files.Shares.Models.ShareFileRequestConditions SourceConditions { get { throw null; } set { } }
     }
 }

diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "net",
   "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares",
-  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_2ce88bf195"
+  "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_e86fda7f54"
 }