[Storage] [DataMovement] Service Copy File Share

sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs

@@ -150,5 +150,19 @@ private string ApplyOptionalPrefix(string path)
             => IsDirectory
                 ? string.Join("/", DirectoryPrefix, path)
                 : path;
+
+        // We will require containers to be created before the transfer starts
+        // Since blobs is a flat namespace, we do not need to create directories (as they are virtual).
+        protected override Task CreateIfNotExistsAsync(CancellationToken cancellationToken)
+            => Task.CompletedTask;
+
+        protected override StorageResourceContainer GetChildStorageResourceContainer(string path)
+        {
+            BlobStorageResourceContainerOptions options = _options.DeepCopy();
+            options.BlobDirectoryPrefix = string.Join("/", DirectoryPrefix, path);
+            return new BlobStorageResourceContainer(
+                BlobContainerClient,
+                options);
+        }
     }
 }

sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs

@@ -471,5 +471,23 @@ internal static BlobStorageResourceContainerOptions GetBlobContainerOptions(
                 BlobOptions = baseOptions,
             };
         }
+
+        internal static BlobStorageResourceContainerOptions DeepCopy(this BlobStorageResourceContainerOptions options)
+            => new BlobStorageResourceContainerOptions()
+            {
+                BlobType = options?.BlobType ?? BlobType.Block,
+                BlobDirectoryPrefix = options?.BlobDirectoryPrefix,
+                BlobOptions = new BlobStorageResourceOptions()
+                {
+                    Metadata = options?.BlobOptions?.Metadata,
+                    Tags = options?.BlobOptions?.Tags,
+                    HttpHeaders = options?.BlobOptions?.HttpHeaders,
+                    AccessTier = options?.BlobOptions?.AccessTier,
+                    DestinationImmutabilityPolicy = options?.BlobOptions?.DestinationImmutabilityPolicy,
+                    LegalHold = options?.BlobOptions?.LegalHold,
+                    UploadTransferValidationOptions = options?.BlobOptions?.UploadTransferValidationOptions,
+                    DownloadTransferValidationOptions = options?.BlobOptions?.DownloadTransferValidationOptions,
+                }
+            };
     }
 }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs

@@ -25,10 +25,12 @@ internal static ShareFileUploadRangeOptions ToShareFileUploadRangeOptions(
             };
 
         internal static ShareFileUploadRangeFromUriOptions ToShareFileUploadRangeFromUriOptions(
-            this ShareFileStorageResourceOptions options)
+            this ShareFileStorageResourceOptions options,
+            HttpAuthorization sourceAuthorization)
             => new()
             {
                 Conditions = options?.DestinationConditions,
+                SourceAuthentication = sourceAuthorization
             };
 
         internal static StorageResourceProperties ToStorageResourceProperties(

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs

@@ -43,10 +43,17 @@ protected override StorageResourceItem GetStorageResourceReference(string path)
         protected override async IAsyncEnumerable<StorageResource> GetStorageResourcesAsync(
             [EnumeratorCancellation] CancellationToken cancellationToken = default)
         {
-            await foreach (ShareFileClient client in PathScanner.ScanFilesAsync(
+            await foreach ((ShareDirectoryClient dir, ShareFileClient file) in PathScanner.ScanAsync(
                 ShareDirectoryClient, cancellationToken).ConfigureAwait(false))
             {
-                yield return new ShareFileStorageResource(client, ResourceOptions);
+                if (file != default)
+                {
+                    yield return new ShareFileStorageResource(file, ResourceOptions);
+                }
+                else
+                {
+                    yield return new ShareDirectoryStorageResourceContainer(dir, ResourceOptions);
+                }
             }
         }
 
@@ -59,5 +66,17 @@ protected override StorageResourceCheckpointData GetDestinationCheckpointData()
         {
             return new ShareFileDestinationCheckpointData();
         }
+
+        protected override async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default)
+        {
+            await ShareDirectoryClient.CreateIfNotExistsAsync(
+                metadata: default,
+                smbProperties: default,
+                filePermission: default,
+                cancellationToken: cancellationToken).ConfigureAwait(false);
+        }
+
+        protected override StorageResourceContainer GetChildStorageResourceContainer(string path)
+            => new ShareDirectoryStorageResourceContainer(ShareDirectoryClient.GetSubdirectoryClient(path), ResourceOptions);
     }
 }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs

@@ -116,7 +116,7 @@ await ShareFileClient.UploadRangeFromUriAsync(
                 sourceUri: sourceResource.Uri,
                 range: range,
                 sourceRange: range,
-                options: _options.ToShareFileUploadRangeFromUriOptions(),
+                options: _options.ToShareFileUploadRangeFromUriOptions(options?.SourceAuthentication),
                 cancellationToken: cancellationToken).ConfigureAwait(false);
         }
 
@@ -165,7 +165,7 @@ await ShareFileClient.UploadRangeFromUriAsync(
                     sourceUri: sourceResource.Uri,
                     range: new HttpRange(0, completeLength),
                     sourceRange: new HttpRange(0, completeLength),
-                    options: _options.ToShareFileUploadRangeFromUriOptions(),
+                    options: _options.ToShareFileUploadRangeFromUriOptions(options?.SourceAuthentication),
                     cancellationToken: cancellationToken).ConfigureAwait(false);
             }
         }

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj

@@ -44,6 +44,7 @@
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)TestEventsRaised.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)DisposingLocalDirectory.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferUploadTestBase.cs" LinkBase="Shared\DataMovement" />
+    <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDirectoryCopyTestBase.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferCopyTestBase.cs" LinkBase="Shared\DataMovement" />
     <Compile Include="$(AzureStorageDataMovementTestSharedSources)StartTransferDownloadTestBase.cs" LinkBase="Shared\DataMovement" />
   </ItemGroup>

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs

@@ -10,7 +10,8 @@
 using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder<
     Azure.Storage.Files.Shares.ShareServiceClient,
     Azure.Storage.Files.Shares.ShareClientOptions>;
-using Azure.Storage.Files.Shares.Models;
+using System.Threading;
+using Azure.Core;
 
 namespace Azure.Storage.DataMovement.Files.Shares.Tests
 {
@@ -43,24 +44,15 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu
                 (uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions),
                 () => new ShareClientOptions(serviceVersion));
 
-        public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) =>
-            clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth);
-
-        public static ShareServiceClient GetServiceClient_OAuth(
-            this SharesClientBuilder clientBuilder, ShareClientOptions options = default)
-        {
-            options ??= clientBuilder.GetOptions();
-            options.ShareTokenIntent = ShareTokenIntent.Backup;
-            return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options);
-        }
-
         public static async Task<DisposingShare> GetTestShareAsync(
             this SharesClientBuilder clientBuilder,
             ShareServiceClient service = default,
             string shareName = default,
             IDictionary<string, string> metadata = default,
-            ShareClientOptions options = default)
+            ShareClientOptions options = default,
+            CancellationToken cancellationToken = default)
         {
+            CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
             service ??= clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigDefault, options);
             metadata ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
             shareName ??= clientBuilder.GetNewShareName();

sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs

@@ -0,0 +1,205 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Azure.Core.TestFramework;
+using Azure.Storage.DataMovement.Tests;
+using Azure.Storage.Files.Shares;
+using Azure.Storage.Files.Shares.Models;
+using Azure.Storage.Test.Shared;
+using Azure.Storage.Files.Shares.Tests;
+using NUnit.Framework;
+using System.Security.AccessControl;
+using Microsoft.Extensions.Options;
+using System.Threading;
+using Azure.Core;
+
+namespace Azure.Storage.DataMovement.Files.Shares.Tests
+{
+    [ShareClientTestFixture]
+    public class ShareDirectoryStartTransferCopyTests : StartTransferDirectoryCopyTestBase<
+        ShareServiceClient,
+        ShareClient,
+        ShareClientOptions,
+        ShareServiceClient,
+        ShareClient,
+        ShareClientOptions,
+        StorageTestEnvironment>
+    {
+        private const string _fileResourcePrefix = "test-file-";
+        private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file.";
+
+        public ShareDirectoryStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion)
+            : base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */)
+        {
+            SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
+            DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion);
+        }
+
+        protected override async Task CreateObjectInSourceAsync(
+            ShareClient container,
+            long? objectLength = null,
+            string objectName = null,
+            Stream contents = default,
+            CancellationToken cancellationToken = default)
+            => await CreateShareFileAsync(container, objectLength, objectName, contents, cancellationToken);
+
+        protected override async Task CreateObjectInDestinationAsync(
+            ShareClient container,
+            long? objectLength = null,
+            string objectName = null,
+            Stream contents = null,
+            CancellationToken cancellationToken = default)
+            => await CreateShareFileAsync(container, objectLength, objectName, contents, cancellationToken);
+
+        protected override async Task<IDisposingContainer<ShareClient>> GetDestinationDisposingContainerAsync(
+            ShareServiceClient service = null,
+            string containerName = null,
+            CancellationToken cancellationToken = default)
+            => await DestinationClientBuilder.GetTestShareAsync(service, containerName, cancellationToken: cancellationToken);
+
+        protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string prefix)
+            => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default);
+
+        protected override ShareClient GetOAuthSourceContainerClient(string containerName)
+        {
+            ShareClientOptions options = SourceClientBuilder.GetOptions();
+            options.ShareTokenIntent = ShareTokenIntent.Backup;
+            ShareServiceClient oauthService = SourceClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth, options);
+            return oauthService.GetShareClient(containerName);
+        }
+
+        protected override ShareClient GetOAuthDestinationContainerClient(string containerName)
+        {
+            ShareClientOptions options = DestinationClientBuilder.GetOptions();
+            options.ShareTokenIntent = ShareTokenIntent.Backup;
+            ShareServiceClient oauthService = DestinationClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth, options);
+            return oauthService.GetShareClient(containerName);
+        }
+
+        protected override async Task<IDisposingContainer<ShareClient>> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default)
+        {
+            service ??= SourceClientBuilder.GetServiceClientFromSharedKeyConfig(SourceClientBuilder.Tenants.TestConfigDefault, SourceClientBuilder.GetOptions());
+            ShareServiceClient sasService = new ShareServiceClient(service.GenerateAccountSasUri(
+                Sas.AccountSasPermissions.All,
+                SourceClientBuilder.Recording.UtcNow.AddDays(1),
+                Sas.AccountSasResourceTypes.All),
+                SourceClientBuilder.GetOptions());
+            return await SourceClientBuilder.GetTestShareAsync(sasService, containerName, cancellationToken: cancellationToken);
+        }
+
+        protected override StorageResourceContainer GetSourceStorageResourceContainer(ShareClient containerClient, string prefix = null)
+            => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default);
+
+        protected override async Task CreateDirectoryInSourceAsync(ShareClient sourceContainer, string directoryPath, CancellationToken cancellationToken = default)
+            => await CreateDirectoryTreeAsync(sourceContainer, directoryPath, cancellationToken);
+
+        protected override async Task CreateDirectoryInDestinationAsync(ShareClient destinationContainer, string directoryPath, CancellationToken cancellationToken = default)
+            => await CreateDirectoryTreeAsync(destinationContainer, directoryPath, cancellationToken);
+
+        protected override async Task VerifyEmptyDestinationContainerAsync(
+            ShareClient destinationContainer,
+            string destinationPrefix,
+            CancellationToken cancellationToken = default)
+        {
+            CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
+            ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ?
+                destinationContainer.GetRootDirectoryClient() :
+                destinationContainer.GetDirectoryClient(destinationPrefix);
+            IList<ShareFileItem> items = await destinationDirectory.GetFilesAndDirectoriesAsync(cancellationToken: cancellationToken).ToListAsync();
+            Assert.IsEmpty(items);
+        }
+
+        protected override async Task VerifyResultsAsync(
+            ShareClient sourceContainer,
+            string sourcePrefix,
+            ShareClient destinationContainer,
+            string destinationPrefix,
+            CancellationToken cancellationToken = default)
+        {
+            CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
+
+            // List all files in source blob folder path
+            List<string> sourceFileNames = new List<string>();
+            List<string> sourceDirectoryNames = new List<string>();
+
+            // Get source directory client and list the paths
+            ShareDirectoryClient sourceDirectory = string.IsNullOrEmpty(sourcePrefix) ?
+                sourceContainer.GetRootDirectoryClient() :
+                sourceContainer.GetDirectoryClient(sourcePrefix);
+            await foreach (Page<ShareFileItem> page in sourceDirectory.GetFilesAndDirectoriesAsync().AsPages())
+            {
+                sourceFileNames.AddRange(page.Values.Where((ShareFileItem item) => !item.IsDirectory).Select((ShareFileItem item) => item.Name));
+                sourceDirectoryNames.AddRange(page.Values.Where((ShareFileItem item) => item.IsDirectory).Select((ShareFileItem item) => item.Name));
+            }
+
+            // List all files in the destination blob folder path
+            List<string> destinationFileNames = new List<string>();
+            List<string> destinationDirectoryNames = new List<string>();
+
+            ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ?
+                destinationContainer.GetRootDirectoryClient() :
+                destinationContainer.GetDirectoryClient(destinationPrefix);
+            await foreach (Page<ShareFileItem> page in destinationDirectory.GetFilesAndDirectoriesAsync().AsPages())
+            {
+                destinationFileNames.AddRange(page.Values.Where((ShareFileItem item) => !item.IsDirectory).Select((ShareFileItem item) => item.Name));
+                destinationDirectoryNames.AddRange(page.Values.Where((ShareFileItem item) => item.IsDirectory).Select((ShareFileItem item) => item.Name));
+            }
+
+            // Assert subdirectories
+            Assert.AreEqual(sourceDirectoryNames.Count, destinationDirectoryNames.Count);
+            Assert.AreEqual(sourceDirectoryNames, destinationDirectoryNames);
+
+            // Assert file and file contents
+            Assert.AreEqual(sourceFileNames.Count, destinationFileNames.Count);
+            for (int i = 0; i < sourceFileNames.Count; i++)
+            {
+                Assert.AreEqual(
+                    sourceFileNames[i],
+                    destinationFileNames[i]);
+
+                // Verify Download
+                string sourceFileName = Path.Combine(sourcePrefix, sourceFileNames[i]);
+                using Stream sourceStream = await sourceDirectory.GetFileClient(sourceFileNames[i]).OpenReadAsync(cancellationToken: cancellationToken);
+                using Stream destinationStream = await destinationDirectory.GetFileClient(destinationFileNames[i]).OpenReadAsync(cancellationToken: cancellationToken);
+                Assert.AreEqual(sourceStream, destinationStream);
+            }
+        }
+
+        private async Task CreateShareFileAsync(
+            ShareClient container,
+            long? objectLength = null,
+            string objectName = null,
+            Stream contents = default,
+            CancellationToken cancellationToken = default)
+        {
+            CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
+            objectName ??= GetNewObjectName();
+            if (!objectLength.HasValue)
+            {
+                throw new InvalidOperationException($"Cannot create share file without size specified. Specify {nameof(objectLength)}.");
+            }
+            ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName);
+            await fileClient.CreateAsync(objectLength.Value);
+
+            if (contents != default)
+            {
+                await fileClient.UploadAsync(contents, cancellationToken: cancellationToken);
+            }
+        }
+
+        private async Task CreateDirectoryTreeAsync(ShareClient container, string directoryPath, CancellationToken cancellationToken = default)
+        {
+            CancellationHelper.ThrowIfCancellationRequested(cancellationToken);
+            // Parse for parent directory names and create the parent directory(s).
+            ShareDirectoryClient directory = container.GetRootDirectoryClient().GetSubdirectoryClient(directoryPath);
+            await directory.CreateIfNotExistsAsync(cancellationToken: cancellationToken);
+        }
+    }
+}