From 669b54267a118c81474c90a9f2db5bdacfb69d36 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 12 Oct 2023 16:30:38 -0700 Subject: [PATCH 01/11] WIP --- .../Shared/StartTransferDirectoryTestBase.cs | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs new file mode 100644 index 0000000000000..971efc6dddbc5 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.TestFramework; +using Azure.Storage.Test.Shared; + +namespace Azure.Storage.DataMovement.Tests +{ + public abstract class StartTransferDirectoryTestBase + : StorageTestBase + where TSourceServiceClient : class + where TSourceContainerClient : class + where TSourceObjectClient : class + where TSourceClientOptions : ClientOptions + where TDestinationServiceClient : class + where TDestinationContainerClient : class + where TDestinationObjectClient : class + where TDestinationClientOptions : ClientOptions + where TEnvironment : StorageTestEnvironment, new() + { + private readonly string _generatedResourceNamePrefix; + private readonly string _expectedOverwriteExceptionMessage; + + public ClientBuilder SourceClientBuilder { get; protected set; } + public ClientBuilder DestinationClientBuilder { get; protected set; } + + /// + /// Constructor for TransferManager.StartTransferAsync tests + /// + /// The async is defaulted to true, since we do not have sync StartTransfer methods. + /// + /// + /// + public StartTransferDirectoryTestBase( + bool async, + string expectedOverwriteExceptionMessage, + string generatedResourceNamePrefix = default, + RecordedTestMode? mode = null) : base(async, mode) + { + Argument.CheckNotNullOrEmpty(expectedOverwriteExceptionMessage, nameof(expectedOverwriteExceptionMessage)); + _generatedResourceNamePrefix = generatedResourceNamePrefix ?? "test-resource-"; + _expectedOverwriteExceptionMessage = expectedOverwriteExceptionMessage; + } + + #region Service-Specific Methods + /// + /// Gets a service-specific disposing container for use with tests in this class. + /// + /// Optionally specified service client to get container from. + /// Optional container name specification. + protected abstract Task> GetSourceDisposingContainerAsync( + TSourceServiceClient service = default, + string containerName = default); + + /// + /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a + /// BlobContainerClient or a TDestinationObjectClient from a ShareClient. + /// + /// Container to get resource from. + /// Sets the resource size in bytes, for resources that require this upfront. + /// Whether to call CreateAsync on the resource, if necessary. + /// Optional name for the resource. + /// ClientOptions for the resource client. + /// If specified, the contents will be uploaded to the object client. + protected abstract Task GetSourceObjectClientAsync( + TSourceContainerClient container, + long? objectLength = default, + bool createResource = false, + string objectName = default, + TSourceClientOptions options = default, + Stream contents = default); + + /// + /// Gets the specific storage resource from the given TDestinationObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceItem GetSourceStorageResourceItem(TSourceObjectClient objectClient); + + /// + /// Calls the OpenRead method on the TDestinationObjectClient. + /// + /// This is mainly used to verify the contents of the Object Client. + /// + /// The object client to get the Open Read Stream from. + /// + protected abstract Task SourceOpenReadAsync(TSourceObjectClient objectClient); + + /// + /// Checks if the Object Client exists. + /// + /// Object Client to call exists on. + /// + protected abstract Task SourceExistsAsync(TSourceObjectClient objectClient); + + /// + /// Gets a service-specific disposing container for use with tests in this class. + /// + /// Optionally specified service client to get container from. + /// Optional container name specification. + protected abstract Task> GetDisposingContainerAsync( + TDestinationServiceClient service = default, + string containerName = default); + + /// + /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a + /// BlobContainerClient or a TDestinationObjectClient from a ShareClient. + /// + /// Container to get resource from. + /// Sets the resource size in bytes, for resources that require this upfront. + /// Whether to call CreateAsync on the resource, if necessary. + /// Optional name for the resource. + /// ClientOptions for the resource client. + /// If specified, the contents will be uploaded to the object client. + protected abstract Task GeTDestinationObjectClientAsync( + TDestinationContainerClient container, + long? objectLength = default, + bool createResource = false, + string objectName = default, + TDestinationClientOptions options = default, + Stream contents = default); + + /// + /// Gets the specific storage resource from the given TDestinationObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceItem GetStorageResourceItem(TDestinationObjectClient objectClient); + + /// + /// Calls the OpenRead method on the TDestinationObjectClient. + /// + /// This is mainly used to verify the contents of the Object Client. + /// + /// The object client to get the Open Read Stream from. + /// + protected abstract Task OpenReadAsync(TDestinationObjectClient objectClient); + + /// + /// Checks if the Object Client exists. + /// + /// Object Client to call exists on. + /// + protected abstract Task ExistsAsync(TDestinationObjectClient objectClient); + #endregion + } +} From 8cc588d99a16f21409579c5c36d621f5c638a7fa Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Sun, 15 Oct 2023 21:30:25 -0700 Subject: [PATCH 02/11] WIP --- .../Shared/StartTransferDirectoryTestBase.cs | 793 +++++++++++++++++- 1 file changed, 787 insertions(+), 6 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs index 971efc6dddbc5..4a0f3dae240b1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs @@ -6,10 +6,12 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Core.TestFramework; using Azure.Storage.Test.Shared; +using NUnit.Framework; namespace Azure.Storage.DataMovement.Tests { @@ -22,6 +24,7 @@ public abstract class StartTransferDirectoryTestBase TDestinationContainerClient, TDestinationObjectClient, TDestinationClientOptions, + TDestinationResourceContainerOptions, TEnvironment> : StorageTestBase where TSourceServiceClient : class where TSourceContainerClient : class @@ -69,7 +72,7 @@ protected abstract Task> GetSourceDi /// /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a - /// BlobContainerClient or a TDestinationObjectClient from a ShareClient. + /// TSourceContainerClient or a TDestinationObjectClient from a ShareClient. /// /// Container to get resource from. /// Sets the resource size in bytes, for resources that require this upfront. @@ -85,6 +88,15 @@ protected abstract Task GetSourceObjectClientAsync( TSourceClientOptions options = default, Stream contents = default); + /// + /// Gets the specific storage resource from the given TDestinationObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// The prefix for a Storage Resource Container. + /// + protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string prefix); + /// /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. @@ -114,13 +126,13 @@ protected abstract Task GetSourceObjectClientAsync( /// /// Optionally specified service client to get container from. /// Optional container name specification. - protected abstract Task> GetDisposingContainerAsync( + protected abstract Task> GetDestinationDisposingContainerAsync( TDestinationServiceClient service = default, string containerName = default); /// /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a - /// BlobContainerClient or a TDestinationObjectClient from a ShareClient. + /// TSourceContainerClient or a TDestinationObjectClient from a ShareClient. /// /// Container to get resource from. /// Sets the resource size in bytes, for resources that require this upfront. @@ -136,13 +148,22 @@ protected abstract Task GeTDestinationObjectClientAsyn TDestinationClientOptions options = default, Stream contents = default); + /// + /// Gets the specific storage resource from the given TDestinationObjectClient + /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// + /// The object client to create the storage resource object. + /// The prefix for a Storage Resource Container. + /// + protected abstract StorageResourceContainer GetDestinationStorageResourceContainer(TDestinationContainerClient containerClient, string prefix); + /// /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. /// /// The object client to create the storage resource object. /// - protected abstract StorageResourceItem GetStorageResourceItem(TDestinationObjectClient objectClient); + protected abstract StorageResourceItem GetDestinationStorageResourceItem(TDestinationObjectClient objectClient); /// /// Calls the OpenRead method on the TDestinationObjectClient. @@ -151,14 +172,774 @@ protected abstract Task GeTDestinationObjectClientAsyn /// /// The object client to get the Open Read Stream from. /// - protected abstract Task OpenReadAsync(TDestinationObjectClient objectClient); + protected abstract Task DestinationOpenReadAsync(TDestinationObjectClient objectClient); /// /// Checks if the Object Client exists. /// /// Object Client to call exists on. /// - protected abstract Task ExistsAsync(TDestinationObjectClient objectClient); + protected abstract Task DestinationExistsAsync(TDestinationObjectClient objectClient); + #endregion + + /// + /// Upload and verify the contents of the blob + /// + /// By default in this function an event argument will be added to the options event handler + /// to detect when the upload has finished. + /// + /// The source container which will contains the source blobs + /// The source blob prefix/folder + /// The local source file prefix to join together with the source prefixes below. + /// The source file paths relative to the sourceFilePrefix + /// The destination local path to download the blobs to + /// + /// How long we should wait until we cancel the operation. If this timeout is reached the test will fail. + /// + /// Options for the transfer manager + /// Options for the transfer Options + /// + private async Task CopyBlobDirectoryAndVerify( + TSourceContainerClient sourceContainer, + TDestinationContainerClient destinationContainer, + string sourceBlobPrefix, + string sourceFilePrefix, + string destinationBlobPrefix, + List sourceFiles, + int waitTimeInSec = 30, + TransferManagerOptions transferManagerOptions = default, + DataTransferOptions options = default) + { + // Set transfer options + options ??= new DataTransferOptions(); + TestEventsRaised testEventFailed = new TestEventsRaised(options); + + transferManagerOptions ??= new TransferManagerOptions() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure + }; + + // Initialize transferManager + TransferManager transferManager = new TransferManager(transferManagerOptions); + + StorageResourceContainer sourceResource = + GetSourceStorageResourceContainer(sourceContainer, sourceBlobPrefix); + StorageResourceContainer destinationResource = + GetDestinationStorageResourceContainer(destinationContainer, destinationBlobPrefix); + + DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); + + // Assert + CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); + await transfer.WaitForCompletionAsync(tokenSource.Token); + + await testEventFailed.AssertContainerCompletedCheck(sourceFiles.Count); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + + // List all files in source blob folder path + List sourceblobNames = new List(); + await foreach (Page page in sourceContainer.GetBlobsAsync(prefix: sourceBlobPrefix).AsPages()) + { + sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + + // List all files in the destination blob folder path + List destblobNames = new List(); + await foreach (Page page in sourceContainer.GetBlobsAsync(prefix: destinationBlobPrefix).AsPages()) + { + destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); + sourceFiles.Sort(); + sourceblobNames.Sort(); + destblobNames.Sort(); + for (int i = 0; i < sourceFiles.Count; i++) + { + // Verify file name to match the + // (prefix folder path) + (the blob name without the blob folder prefix) + string sourceNonPrefixed = sourceblobNames[i].Substring(sourceBlobPrefix.Length + 1); + Assert.AreEqual( + sourceNonPrefixed, + destblobNames[i].Substring(destinationBlobPrefix.Length + 1)); + + // Verify Download + string sourceFileName = Path.Combine(sourceFilePrefix, sourceNonPrefixed); + using (FileStream fileStream = File.OpenRead(sourceFileName)) + { + BlockBlobClient destinationBlob = sourceContainer.GetBlockBlobClient(destblobNames[i]); + Assert.IsTrue(await destinationBlob.ExistsAsync()); + await DownloadAndAssertAsync(fileStream, destinationBlob); + } + } + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + [TestCase(0, 10)] + [TestCase(100, 10)] + [TestCase(Constants.KB, 10)] + public async Task BlockBlobDirectoryToDirectory_SmallSize(long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobDirectoryName = "sourceFolder"; + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + blobNames.Add(blobName1); + blobNames.Add(blobName2); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + blobNames.Add(blobName3); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + blobNames.Add(blobName4); + + string destinationFolder = "destFolder"; + + await CopyBlobDirectoryAndVerify( + test.Container, + sourceBlobDirectoryName, + sourceFolderPath, + destinationFolder, + blobNames, + waitTimeInSec).ConfigureAwait(false); + } + + [Ignore("These tests currently take 40+ mins for little additional coverage")] + [Test] + [LiveOnly] + [TestCase(4 * Constants.MB, 20)] + [TestCase(4 * Constants.MB, 200)] + [TestCase(257 * Constants.MB, 500)] + [TestCase(Constants.GB, 500)] + public async Task BlockBlobDirectoryToDirectory_LargeSize(long size, int waitTimeInSec) + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobDirectoryName = "sourceFolder"; + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + blobNames.Add(blobName1); + blobNames.Add(blobName2); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + blobNames.Add(blobName3); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + blobNames.Add(blobName4); + + string destinationFolder = "destFolder"; + + await CopyBlobDirectoryAndVerify( + test.Container, + sourceBlobDirectoryName, + sourceFolderPath, + destinationFolder, + blobNames, + waitTimeInSec).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_EmptyFolder() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + // Set up directory to upload + var dirName = GetNewBlobDirectoryName(); + var dirName2 = GetNewBlobDirectoryName(); + string folder = CreateRandomDirectory(testDirectory.DirectoryPath); + + // Set up destination client + StorageResourceContainer destinationResource = new BlobStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = dirName }); + StorageResourceContainer sourceResource = new BlobStorageResourceContainer(test.Container, + new BlobStorageResourceContainerOptions() + { + BlobDirectoryPrefix = dirName2, + }); + + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + ErrorHandling = DataTransferErrorMode.ContinueOnFailure, + MaximumConcurrency = 1, + }; + TransferManager transferManager = new TransferManager(managerOptions); + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Act + DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); + + CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(tokenSource.Token); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + + // Assert + List blobs = ((List)await test.Container.GetBlobsAsync().ToListAsync()) + .Select((BlobItem blob) => blob.Name).ToList(); + // Assert + Assert.IsEmpty(blobs); + testEventsRaised.AssertUnexpectedFailureCheck(); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_SingleFile() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + string sourceFolderName = "sourceFolder"; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceFolderName); + + string blobName1 = Path.Combine(sourceFolderName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); + List blobNames = new List() { blobName1 }; + + string destinationFolder = "destFolder"; + + await CopyBlobDirectoryAndVerify( + container: test.Container, + sourceBlobPrefix: sourceFolderName, + sourceFilePrefix: sourceFolderPath, + destinationBlobPrefix: destinationFolder, + blobNames).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_ManySubDirectories() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + string blobDirectoryName = "sourceFolder"; + string fullSourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, blobDirectoryName); + + List blobNames = new List(); + string subDir1 = CreateRandomDirectory(fullSourceFolderPath, "bar").Substring(fullSourceFolderPath.Length + 1); + string blobName1 = Path.Combine(blobDirectoryName, subDir1, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); + blobNames.Add(blobName1); + string subDir2 = CreateRandomDirectory(fullSourceFolderPath, "rul").Substring(fullSourceFolderPath.Length + 1); + string blobName2 = Path.Combine(blobDirectoryName, subDir2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, Constants.KB); + blobNames.Add(blobName2); + string subDir3 = CreateRandomDirectory(fullSourceFolderPath, "pik").Substring(fullSourceFolderPath.Length + 1); + string blobName3 = Path.Combine(blobDirectoryName, subDir3, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, Constants.KB); + blobNames.Add(blobName3); + + string destinationFolder = "destFolder"; + + string sourceBlobPrefix = fullSourceFolderPath.Substring(testDirectory.DirectoryPath.Length + 1); + + await CopyBlobDirectoryAndVerify( + container: test.Container, + sourceBlobPrefix: sourceBlobPrefix, + sourceFilePrefix: fullSourceFolderPath, + destinationBlobPrefix: destinationFolder, + blobNames).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + public async Task BlockBlobDirectoryToDirectory_SubDirectoriesLevels(int level) + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + string sourceBlobDirectoryName = "sourceFolder"; + string fullSourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + + string subDir = default; + for (int i = 0; i < level; i++) + { + subDir = CreateRandomDirectory(fullSourceFolderPath, $"folder{i}"); + string blobName = Path.Combine(sourceBlobDirectoryName, subDir.Substring(fullSourceFolderPath.Length + 1), GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName, Constants.KB); + blobNames.Add(blobName); + } + + string destinationFolder = "destFolder"; + + await CopyBlobDirectoryAndVerify( + test.Container, + sourceBlobDirectoryName, + fullSourceFolderPath, + destinationBlobPrefix: destinationFolder, + blobNames).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_OverwriteTrue() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + long size = Constants.KB; + string sourceBlobDirectoryName = "sourceFolder"; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + blobNames.Add(blobName1); + blobNames.Add(blobName2); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + blobNames.Add(blobName3); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + blobNames.Add(blobName4); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists + }; + + string destinationFolder = "destFolder"; + + // Act + await CopyBlobDirectoryAndVerify( + test.Container, + sourceBlobDirectoryName, + sourceFolderPath, + destinationBlobPrefix: destinationFolder, + blobNames, + options: options).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_OverwriteFalse() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + + long size = Constants.KB; + string sourceBlobDirectoryName = "sourceFolder"; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + blobNames.Add(blobName1); + blobNames.Add(blobName2); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + blobNames.Add(blobName3); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + blobNames.Add(blobName4); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists + }; + + string destinationFolder = "destFolder"; + + // Act + await CopyBlobDirectoryAndVerify( + test.Container, + sourceBlobDirectoryName, + sourceFolderPath, + destinationBlobPrefix: destinationFolder, + blobNames, + options: options).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task BlockBlobDirectoryToDirectory_OAuth() + { + // Arrange + long size = Constants.KB; + int waitTimeInSec = 10; + TSourceServiceClient service = BlobsClientBuilder.GetServiceClient_OAuth(); + var containerName = GetNewContainerName(); + await using IDisposingContainer testContainer = await GetTestContainerAsync( + service, + containerName, + publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobDirectoryName = "sourceFolder"; + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + + List blobNames = new List(); + + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName1, size); + await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName2, size); + blobNames.Add(blobName1); + blobNames.Add(blobName2); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName3, size); + blobNames.Add(blobName3); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName4, size); + blobNames.Add(blobName4); + + string destinationFolder = "destFolder"; + + await CopyBlobDirectoryAndVerify( + testContainer.Container, + sourceBlobDirectoryName, + sourceFolderPath, + destinationFolder, + blobNames, + waitTimeInSec).ConfigureAwait(false); + } + + #region Single Concurrency + private async Task CreateBlobDirectoryTree( + TSourceContainerClient client, + string sourceFolderPath, + string sourceBlobDirectoryName, + int size) + { + string blobName1 = Path.Combine(sourceBlobDirectoryName, "blob1"); + string blobName2 = Path.Combine(sourceBlobDirectoryName, "blob2"); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName1, size); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName2, size); + + string subDirName = "bar"; + CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, "blob3"); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName3, size); + + string subDirName2 = "pik"; + CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, "blob4"); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName4, size); + } + + private async Task CreateStartTransfer( + TSourceContainerClient containerClient, + int concurrency, + bool createFailedCondition = false, + DataTransferOptions options = default, + int size = Constants.KB) + { + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + // Arrange + // Create source local file for checking, and source blob + string sourceBlobPrefix = "sourceFolder"; + string destBlobPrefix = "destFolder"; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobPrefix); + await CreateBlobDirectoryTree(containerClient, sourceFolderPath, sourceBlobPrefix, size); + + // Create new source block blob. + StorageResourceContainer sourceResource = new BlobStorageResourceContainer(containerClient, new() { BlobDirectoryPrefix = sourceBlobPrefix }); + StorageResourceContainer destinationResource = new BlobStorageResourceContainer(containerClient, + new BlobStorageResourceContainerOptions() + { + BlobDirectoryPrefix = destBlobPrefix, + }); + + // If we want a failure condition to happen + if (createFailedCondition) + { + string destBlobName = $"{destBlobPrefix}/blob1"; + await CreateBlockBlob(containerClient, Path.Combine(testDirectory.DirectoryPath, "blob1"), destBlobName, size); + } + + // Create Transfer Manager with single threaded operation + TransferManagerOptions managerOptions = new TransferManagerOptions() + { + MaximumConcurrency = concurrency, + }; + TransferManager transferManager = new TransferManager(managerOptions); + + // Start transfer and await for completion. + return await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options).ConfigureAwait(false); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_AwaitCompletion() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + // Create transfer to do a AwaitCompletion + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + DataTransfer transfer = await CreateStartTransfer(test.Container, 1, options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + testEventsRaised.AssertUnexpectedFailureCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_AwaitCompletion_Failed() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + test.Container, + 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_AwaitCompletion_Skipped() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + // Create transfer options with Skipping available + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + test.Container, + 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await transfer.WaitForCompletionAsync(cancellationTokenSource.Token).ConfigureAwait(false); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); + await testEventsRaised.AssertContainerCompletedWithSkippedCheck(1); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_EnsureCompleted() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + // Create transfer to do a EnsureCompleted + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + DataTransfer transfer = await CreateStartTransfer(test.Container, 1, options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + testEventsRaised.AssertUnexpectedFailureCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_EnsureCompleted_Failed() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + test.Container, + 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_EnsureCompleted_Skipped() + { + // Arrange + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + + // Create transfer options with Skipping available + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.SkipIfExists + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a EnsureCompleted + DataTransfer transfer = await CreateStartTransfer( + test.Container, + 1, + createFailedCondition: true, + options: options); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + testEventsRaised.AssertUnexpectedFailureCheck(); + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); + } + + [Test] + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() + { + // Arrange + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); + + DataTransferOptions options = new DataTransferOptions() + { + CreationPreference = StorageResourceCreationPreference.FailIfExists, + InitialTransferSize = 512, + MaximumTransferChunkSize = 512 + }; + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + + // Create transfer to do a AwaitCompletion + DataTransfer transfer = await CreateStartTransfer( + test.Container, + 1, + createFailedCondition: true, + options: options, + size: Constants.KB * 4); + + // Act + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + transfer.WaitForCompletion(cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); + } #endregion } } From 08c4bcacfc7628cca78b16aa363b5b09252b1790 Mon Sep 17 00:00:00 2001 From: amnguye Date: Wed, 18 Oct 2023 14:02:21 -0700 Subject: [PATCH 03/11] WIP --- ...age.DataMovement.Files.Shares.Tests.csproj | 1 + .../ShareDirectoryStartTransferCopyTests.cs | 125 ++++++++++ ... => StartTransferDirectoryCopyTestBase.cs} | 214 ++++++------------ .../Azure.Storage.Files.Shares.Tests.csproj | 50 ++++ 4 files changed, 251 insertions(+), 139 deletions(-) create mode 100644 sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs rename sdk/storage/Azure.Storage.DataMovement/tests/Shared/{StartTransferDirectoryTestBase.cs => StartTransferDirectoryCopyTestBase.cs} (82%) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index e0440459ba4b6..9724b443a51ca 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -44,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs new file mode 100644 index 0000000000000..7cb15744fb90a --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.DataMovement.Tests; +using Azure.Storage.Files.Shares; +using Azure.Storage.Files.Shares.Models; +using Azure.Storage.Test.Shared; +using NUnit.Framework; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + public class ShareDirectoryStartTransferCopyTests : StartTransferDirectoryCopyTestBase< + ShareServiceClient, + ShareClient, + ShareClientOptions, + ShareServiceClient, + ShareClient, + ShareClientOptions, + StorageTestEnvironment> + { + protected override async Task CreateObjectInSource( + ShareClient containerClient, + string objectName = null, + long? size = null) + { + objectName ??= GetNewObjectName(); + ShareFileClient fileClient = containerClient.GetRootDirectoryClient().GetFileClient(objectName); + size ??= 0; + await fileClient.CreateAsync(size.Value); + + // Upload random content if the size is non-zero + if () + } + + protected override async Task CreateObjectInSource( + ShareClient containerClient, + string objectName = null, + Stream content = null) + { + objectName ??= GetNewObjectName(); + ShareFileClient fileClient = containerClient.GetRootDirectoryClient().GetFileClient(objectName); + if (content != null) + { + await fileClient.UploadAsync(content); + } + using Stream originalStream = await CreateLimitedMemoryStream(size); + } + + protected override Task> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + { + throw new NotImplementedException(); + } + + protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string prefix) + { + throw new NotImplementedException(); + } + + protected override Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + { + throw new NotImplementedException(); + } + + protected override StorageResourceContainer GetSourceStorageResourceContainer(ShareClient containerClient, string prefix = null) + { + throw new NotImplementedException(); + } + + protected override async Task VerifyResults( + ShareClient sourceContainer, + ShareClient destinationContainer, + string sourcePrefix = default, + string destiantionPrefix = default) + { + // List all files in source blob folder path + List sourceNames = new List(); + + // Get source directory client and list the paths + ShareDirectoryClient sourceDirectory = string.IsNullOrEmpty(sourcePrefix) ? + sourceContainer.GetRootDirectoryClient() : + sourceContainer.GetDirectoryClient(sourcePrefix); + await foreach (Page page in sourceDirectory.GetFilesAndDirectoriesAsync().AsPages()) + { + sourceNames.AddRange(page.Values.Select((ShareFileItem item) => item.Name)); + } + + // List all files in the destination blob folder path + List destinationNames = new List(); + + ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destiantionPrefix) ? + destinationContainer.GetRootDirectoryClient() : + destinationContainer.GetDirectoryClient(sourcePrefix); + await foreach (Page page in destinationDirectory.GetFilesAndDirectoriesAsync().AsPages()) + { + destinationNames.AddRange(page.Values.Select((ShareFileItem item) => item.Name)); + } + Assert.AreEqual(sourceNames.Count, destinationNames.Count); + sourceNames.Sort(); + destinationNames.Sort(); + for (int i = 0; i < sourceNames.Count; i++) + { + // Verify file name to match the + // (prefix folder path) + (the blob name without the blob folder prefix) + // TODO: verify if files returns the entire path, and if we really need to be parsing the prefix + string sourceNonPrefixed = sourceNames[i].Substring(sourcePrefix.Length + 1); + Assert.AreEqual( + sourceNonPrefixed, + destinationNames[i].Substring(destiantionPrefix.Length + 1)); + + // Verify Download + string sourceFileName = Path.Combine(sourcePrefix, sourceNonPrefixed); + using Stream sourceStream = await sourceDirectory.GetFileClient(sourceNames[i]).OpenReadAsync(); + using Stream destinationStream = await destinationDirectory.GetFileClient(destinationNames[i]).OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs similarity index 82% rename from sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs rename to sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index 4a0f3dae240b1..e4fe23630584f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -15,24 +15,19 @@ namespace Azure.Storage.DataMovement.Tests { - public abstract class StartTransferDirectoryTestBase + public abstract class StartTransferDirectoryCopyTestBase : StorageTestBase where TSourceServiceClient : class where TSourceContainerClient : class - where TSourceObjectClient : class where TSourceClientOptions : ClientOptions where TDestinationServiceClient : class where TDestinationContainerClient : class - where TDestinationObjectClient : class where TDestinationClientOptions : ClientOptions where TEnvironment : StorageTestEnvironment, new() { @@ -49,7 +44,7 @@ public abstract class StartTransferDirectoryTestBase /// /// /// - public StartTransferDirectoryTestBase( + public StartTransferDirectoryCopyTestBase( bool async, string expectedOverwriteExceptionMessage, string generatedResourceNamePrefix = default, @@ -70,56 +65,32 @@ protected abstract Task> GetSourceDi TSourceServiceClient service = default, string containerName = default); - /// - /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a - /// TSourceContainerClient or a TDestinationObjectClient from a ShareClient. - /// - /// Container to get resource from. - /// Sets the resource size in bytes, for resources that require this upfront. - /// Whether to call CreateAsync on the resource, if necessary. - /// Optional name for the resource. - /// ClientOptions for the resource client. - /// If specified, the contents will be uploaded to the object client. - protected abstract Task GetSourceObjectClientAsync( - TSourceContainerClient container, - long? objectLength = default, - bool createResource = false, - string objectName = default, - TSourceClientOptions options = default, - Stream contents = default); - /// /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. /// /// The object client to create the storage resource object. - /// The prefix for a Storage Resource Container. + /// The prefix for a Storage Resource Container. If not specified, will default to root directory. /// - protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string prefix); + protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string prefix = default); /// - /// Gets the specific storage resource from the given TDestinationObjectClient - /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. + /// Creates the object in the source storage resource container. /// - /// The object client to create the storage resource object. + /// The container client to create the storage resource object. + /// The name of the object to create. + /// Optional. The size of the object. /// - protected abstract StorageResourceItem GetSourceStorageResourceItem(TSourceObjectClient objectClient); + protected abstract Task CreateObjectInSource(TSourceContainerClient containerClient, string objectName = default, long? size = default); /// - /// Calls the OpenRead method on the TDestinationObjectClient. - /// - /// This is mainly used to verify the contents of the Object Client. + /// Creates the object in the source storage resource container. /// - /// The object client to get the Open Read Stream from. + /// The container client to create the storage resource object. + /// The name of the object to create. + /// Optional. The contents to set in the object resource. /// - protected abstract Task SourceOpenReadAsync(TSourceObjectClient objectClient); - - /// - /// Checks if the Object Client exists. - /// - /// Object Client to call exists on. - /// - protected abstract Task SourceExistsAsync(TSourceObjectClient objectClient); + protected abstract Task CreateObjectInSource(TSourceContainerClient containerClient, string objectName = default, Stream content = default); /// /// Gets a service-specific disposing container for use with tests in this class. @@ -130,24 +101,6 @@ protected abstract Task> GetDes TDestinationServiceClient service = default, string containerName = default); - /// - /// Gets a new service-specific child object client from a given container, e.g. a BlobClient from a - /// TSourceContainerClient or a TDestinationObjectClient from a ShareClient. - /// - /// Container to get resource from. - /// Sets the resource size in bytes, for resources that require this upfront. - /// Whether to call CreateAsync on the resource, if necessary. - /// Optional name for the resource. - /// ClientOptions for the resource client. - /// If specified, the contents will be uploaded to the object client. - protected abstract Task GeTDestinationObjectClientAsync( - TDestinationContainerClient container, - long? objectLength = default, - bool createResource = false, - string objectName = default, - TDestinationClientOptions options = default, - Stream contents = default); - /// /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. @@ -158,30 +111,23 @@ protected abstract Task GeTDestinationObjectClientAsyn protected abstract StorageResourceContainer GetDestinationStorageResourceContainer(TDestinationContainerClient containerClient, string prefix); /// - /// Gets the specific storage resource from the given TDestinationObjectClient - /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. - /// - /// The object client to create the storage resource object. - /// - protected abstract StorageResourceItem GetDestinationStorageResourceItem(TDestinationObjectClient objectClient); - - /// - /// Calls the OpenRead method on the TDestinationObjectClient. - /// - /// This is mainly used to verify the contents of the Object Client. - /// - /// The object client to get the Open Read Stream from. - /// - protected abstract Task DestinationOpenReadAsync(TDestinationObjectClient objectClient); - - /// - /// Checks if the Object Client exists. + /// Verifies the results between the source and the destination container. /// - /// Object Client to call exists on. + /// The source client to check the contents and compare against the destination. + /// The destinatiojn client to check the contents and compare against the source. + /// Optional. The prefix to start listing at the source container. + /// Optional. The prefix to start listing at the destination container. /// - protected abstract Task DestinationExistsAsync(TDestinationObjectClient objectClient); + protected abstract Task VerifyResults( + TSourceContainerClient sourceClient, + TDestinationContainerClient destinationContainer, + string sourcePrefix = default, + string destinationPrefix = default); #endregion + protected string GetNewObjectName() + => _generatedResourceNamePrefix + SourceClientBuilder.Recording.Random.NewGuid(); + /// /// Upload and verify the contents of the blob /// @@ -190,8 +136,6 @@ protected abstract Task GeTDestinationObjectClientAsyn /// /// The source container which will contains the source blobs /// The source blob prefix/folder - /// The local source file prefix to join together with the source prefixes below. - /// The source file paths relative to the sourceFilePrefix /// The destination local path to download the blobs to /// /// How long we should wait until we cancel the operation. If this timeout is reached the test will fail. @@ -203,9 +147,7 @@ private async Task CopyBlobDirectoryAndVerify( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, string sourceBlobPrefix, - string sourceFilePrefix, string destinationBlobPrefix, - List sourceFiles, int waitTimeInSec = 30, TransferManagerOptions transferManagerOptions = default, DataTransferOptions options = default) @@ -251,7 +193,6 @@ private async Task CopyBlobDirectoryAndVerify( destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); } Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); - sourceFiles.Sort(); sourceblobNames.Sort(); destblobNames.Sort(); for (int i = 0; i < sourceFiles.Count; i++) @@ -282,40 +223,38 @@ private async Task CopyBlobDirectoryAndVerify( public async Task BlockBlobDirectoryToDirectory_SmallSize(long size, int waitTimeInSec) { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); string sourceBlobDirectoryName = "sourceFolder"; using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName1, size); + await CreateObjectInSource(source.Container, blobName2, size); blobNames.Add(blobName1); blobNames.Add(blobName2); string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName3, size); blobNames.Add(blobName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName4, size); blobNames.Add(blobName4); - string destinationFolder = "destFolder"; - await CopyBlobDirectoryAndVerify( - test.Container, + source.Container, + destination.Container, sourceBlobDirectoryName, sourceFolderPath, - destinationFolder, - blobNames, waitTimeInSec).ConfigureAwait(false); } @@ -329,40 +268,36 @@ await CopyBlobDirectoryAndVerify( public async Task BlockBlobDirectoryToDirectory_LargeSize(long size, int waitTimeInSec) { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); string sourceBlobDirectoryName = "sourceFolder"; - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName1, size); + await CreateObjectInSource(source.Container, blobName2, size); blobNames.Add(blobName1); blobNames.Add(blobName2); string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName3, size); blobNames.Add(blobName3); string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); + await CreateObjectInSource(source.Container, blobName4, size); blobNames.Add(blobName4); string destinationFolder = "destFolder"; await CopyBlobDirectoryAndVerify( - test.Container, + source.Container, + destination.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationFolder, - blobNames, waitTimeInSec).ConfigureAwait(false); } @@ -371,17 +306,18 @@ await CopyBlobDirectoryAndVerify( public async Task BlockBlobDirectoryToDirectory_EmptyFolder() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); // Set up directory to upload - var dirName = GetNewBlobDirectoryName(); - var dirName2 = GetNewBlobDirectoryName(); + var dirName = GetNewObjectName(); + var dirName2 = GetNewObjectName(); string folder = CreateRandomDirectory(testDirectory.DirectoryPath); // Set up destination client - StorageResourceContainer destinationResource = new BlobStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = dirName }); - StorageResourceContainer sourceResource = new BlobStorageResourceContainer(test.Container, + StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = dirName }); + StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(test.Container, new BlobStorageResourceContainerOptions() { BlobDirectoryPrefix = dirName2, @@ -417,13 +353,13 @@ public async Task BlockBlobDirectoryToDirectory_EmptyFolder() public async Task BlockBlobDirectoryToDirectory_SingleFile() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string sourceFolderName = "sourceFolder"; string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceFolderName); - string blobName1 = Path.Combine(sourceFolderName, GetNewBlobName()); + string blobName1 = Path.Combine(sourceFolderName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); List blobNames = new List() { blobName1 }; @@ -450,15 +386,15 @@ public async Task BlockBlobDirectoryToDirectory_ManySubDirectories() List blobNames = new List(); string subDir1 = CreateRandomDirectory(fullSourceFolderPath, "bar").Substring(fullSourceFolderPath.Length + 1); - string blobName1 = Path.Combine(blobDirectoryName, subDir1, GetNewBlobName()); + string blobName1 = Path.Combine(blobDirectoryName, subDir1, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); blobNames.Add(blobName1); string subDir2 = CreateRandomDirectory(fullSourceFolderPath, "rul").Substring(fullSourceFolderPath.Length + 1); - string blobName2 = Path.Combine(blobDirectoryName, subDir2, GetNewBlobName()); + string blobName2 = Path.Combine(blobDirectoryName, subDir2, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, Constants.KB); blobNames.Add(blobName2); string subDir3 = CreateRandomDirectory(fullSourceFolderPath, "pik").Substring(fullSourceFolderPath.Length + 1); - string blobName3 = Path.Combine(blobDirectoryName, subDir3, GetNewBlobName()); + string blobName3 = Path.Combine(blobDirectoryName, subDir3, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, Constants.KB); blobNames.Add(blobName3); @@ -494,7 +430,7 @@ public async Task BlockBlobDirectoryToDirectory_SubDirectoriesLevels(int level) for (int i = 0; i < level; i++) { subDir = CreateRandomDirectory(fullSourceFolderPath, $"folder{i}"); - string blobName = Path.Combine(sourceBlobDirectoryName, subDir.Substring(fullSourceFolderPath.Length + 1), GetNewBlobName()); + string blobName = Path.Combine(sourceBlobDirectoryName, subDir.Substring(fullSourceFolderPath.Length + 1), GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName, Constants.KB); blobNames.Add(blobName); } @@ -522,8 +458,8 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteTrue() string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); blobNames.Add(blobName1); @@ -531,13 +467,13 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteTrue() string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); blobNames.Add(blobName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); blobNames.Add(blobName4); @@ -571,8 +507,8 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteFalse() string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); blobNames.Add(blobName1); @@ -580,13 +516,13 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteFalse() string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); blobNames.Add(blobName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); blobNames.Add(blobName4); @@ -626,8 +562,8 @@ public async Task BlockBlobDirectoryToDirectory_OAuth() List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewBlobName()); + string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName1, size); await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName2, size); blobNames.Add(blobName1); @@ -635,13 +571,13 @@ public async Task BlockBlobDirectoryToDirectory_OAuth() string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewBlobName()); + string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName3, size); blobNames.Add(blobName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewBlobName()); + string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName4, size); blobNames.Add(blobName4); diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj index 398a4b6367489..d501967eb6e11 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj @@ -17,6 +17,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest From 94c9dff29dd5e0309d21e50986caff872cc5f07a Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 19 Oct 2023 10:47:48 -0700 Subject: [PATCH 04/11] WIP --- .../ShareDirectoryStartTransferCopyTests.cs | 106 ++-- .../StartTransferDirectoryCopyTestBase.cs | 538 ++++++++---------- 2 files changed, 314 insertions(+), 330 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index 7cb15744fb90a..5c71bff96cc5d 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -8,14 +8,18 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Azure.Core.TestFramework; using Azure.Storage.DataMovement.Tests; using Azure.Storage.Files.Shares; using Azure.Storage.Files.Shares.Models; using Azure.Storage.Test.Shared; +using Azure.Storage.Files.Shares.Tests; using NUnit.Framework; +using System.Security.AccessControl; namespace Azure.Storage.DataMovement.Files.Shares.Tests { + [ShareClientTestFixture] public class ShareDirectoryStartTransferCopyTests : StartTransferDirectoryCopyTestBase< ShareServiceClient, ShareClient, @@ -25,59 +29,59 @@ public class ShareDirectoryStartTransferCopyTests : StartTransferDirectoryCopyTe ShareClientOptions, StorageTestEnvironment> { - protected override async Task CreateObjectInSource( - ShareClient containerClient, - string objectName = null, - long? size = null) - { - objectName ??= GetNewObjectName(); - ShareFileClient fileClient = containerClient.GetRootDirectoryClient().GetFileClient(objectName); - size ??= 0; - await fileClient.CreateAsync(size.Value); + private const string _fileResourcePrefix = "test-file-"; + private const string _expectedOverwriteExceptionMessage = "Cannot overwrite file."; - // Upload random content if the size is non-zero - if () + public ShareDirectoryStartTransferCopyTests(bool async, ShareClientOptions.ServiceVersion serviceVersion) + : base(async, _expectedOverwriteExceptionMessage, _fileResourcePrefix, null /* RecordedTestMode.Record /* to re-record */) + { + SourceClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion); + DestinationClientBuilder = ClientBuilderExtensions.GetNewShareClientBuilder(Tenants, serviceVersion); } - protected override async Task CreateObjectInSource( - ShareClient containerClient, + protected override async Task CreateObjectInSourceAsync( + ShareClient container, + long? objectLength = null, string objectName = null, - Stream content = null) - { - objectName ??= GetNewObjectName(); - ShareFileClient fileClient = containerClient.GetRootDirectoryClient().GetFileClient(objectName); - if (content != null) - { - await fileClient.UploadAsync(content); - } - using Stream originalStream = await CreateLimitedMemoryStream(size); - } + Stream contents = default) + => await CreateShareFileAsync(container, objectLength, objectName, contents); - protected override Task> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) - { - throw new NotImplementedException(); - } + protected override async Task CreateObjectInDestinationAsync( + ShareClient container, + long? objectLength = null, + string objectName = null, + Stream contents = null) + => await CreateShareFileAsync(container, objectLength, objectName, contents); + + protected override async Task> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + => await DestinationClientBuilder.GetTestShareAsync(service, containerName); protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string prefix) - { - throw new NotImplementedException(); - } + => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); - protected override Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) - { - throw new NotImplementedException(); - } + protected override ShareServiceClient GetOAuthSourceServiceClient() + => SourceClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth); + + protected override async Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + => await SourceClientBuilder.GetTestShareAsync(service, containerName); protected override StorageResourceContainer GetSourceStorageResourceContainer(ShareClient containerClient, string prefix = null) + => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); + + protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient destinationContainer, string destinationPrefix) { - throw new NotImplementedException(); + ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ? + destinationContainer.GetRootDirectoryClient() : + destinationContainer.GetDirectoryClient(destinationPrefix); + IList items = await destinationDirectory.GetFilesAndDirectoriesAsync().ToListAsync(); + Assert.IsEmpty(items); } - protected override async Task VerifyResults( + protected override async Task VerifyResultsAsync( ShareClient sourceContainer, + string sourcePrefix, ShareClient destinationContainer, - string sourcePrefix = default, - string destiantionPrefix = default) + string destinationPrefix) { // List all files in source blob folder path List sourceNames = new List(); @@ -94,9 +98,9 @@ protected override async Task VerifyResults( // List all files in the destination blob folder path List destinationNames = new List(); - ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destiantionPrefix) ? + ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ? destinationContainer.GetRootDirectoryClient() : - destinationContainer.GetDirectoryClient(sourcePrefix); + destinationContainer.GetDirectoryClient(destinationPrefix); await foreach (Page page in destinationDirectory.GetFilesAndDirectoriesAsync().AsPages()) { destinationNames.AddRange(page.Values.Select((ShareFileItem item) => item.Name)); @@ -112,7 +116,7 @@ protected override async Task VerifyResults( string sourceNonPrefixed = sourceNames[i].Substring(sourcePrefix.Length + 1); Assert.AreEqual( sourceNonPrefixed, - destinationNames[i].Substring(destiantionPrefix.Length + 1)); + destinationNames[i].Substring(destinationPrefix.Length + 1)); // Verify Download string sourceFileName = Path.Combine(sourcePrefix, sourceNonPrefixed); @@ -121,5 +125,25 @@ protected override async Task VerifyResults( Assert.AreEqual(sourceStream, destinationStream); } } + + private async Task CreateShareFileAsync( + ShareClient container, + long? objectLength = null, + string objectName = null, + Stream contents = default) + { + objectName ??= GetNewObjectName(); + if (!objectLength.HasValue) + { + throw new InvalidOperationException($"Cannot create share file without size specified. Specify {nameof(objectLength)}."); + } + ShareFileClient fileClient = container.GetRootDirectoryClient().GetFileClient(objectName); + await fileClient.CreateAsync(objectLength.Value); + + if (contents != default) + { + await fileClient.UploadAsync(contents); + } + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index e4fe23630584f..07fecee7cd738 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Core; @@ -56,6 +55,11 @@ public StartTransferDirectoryCopyTestBase( } #region Service-Specific Methods + /// + /// Gets the service client using OAuth to authenticate. + /// + protected abstract TSourceServiceClient GetOAuthSourceServiceClient(); + /// /// Gets a service-specific disposing container for use with tests in this class. /// @@ -70,27 +74,18 @@ protected abstract Task> GetSourceDi /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. /// /// The object client to create the storage resource object. - /// The prefix for a Storage Resource Container. If not specified, will default to root directory. - /// - protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string prefix = default); - - /// - /// Creates the object in the source storage resource container. - /// - /// The container client to create the storage resource object. - /// The name of the object to create. - /// Optional. The size of the object. + /// The path of the directory. /// - protected abstract Task CreateObjectInSource(TSourceContainerClient containerClient, string objectName = default, long? size = default); + protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient directoryClient, string prefix); /// /// Creates the object in the source storage resource container. /// - /// The container client to create the storage resource object. + /// The length to create the object of. /// The name of the object to create. - /// Optional. The contents to set in the object resource. + /// The contents to set in the object. /// - protected abstract Task CreateObjectInSource(TSourceContainerClient containerClient, string objectName = default, Stream content = default); + protected abstract Task CreateObjectInSourceAsync(TSourceContainerClient container, long? objectLength = null, string objectName = null, Stream contents = default); /// /// Gets a service-specific disposing container for use with tests in this class. @@ -105,24 +100,41 @@ protected abstract Task> GetDes /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. /// - /// The object client to create the storage resource object. - /// The prefix for a Storage Resource Container. + /// The object client to create the storage resource object. + /// + protected abstract StorageResourceContainer GetDestinationStorageResourceContainer(TDestinationContainerClient sourceContainerClient, string directoryPath); + + /// + /// Creates the object in the source storage resource container. + /// + /// The length to create the object of. + /// The name of the object to create. + /// The contents to set in the object. /// - protected abstract StorageResourceContainer GetDestinationStorageResourceContainer(TDestinationContainerClient containerClient, string prefix); + protected abstract Task CreateObjectInDestinationAsync(TDestinationContainerClient container, long? objectLength = null, string objectName = null, Stream contents = default); + + /// + /// Verifies that the destination container is empty when we expect it to be. + /// + /// + /// The respective destination container to verify empty contents. + /// + /// + protected abstract Task VerifyEmptyDestinationContainerAsync(TDestinationContainerClient destinationContainer, string destinationPrefix); /// /// Verifies the results between the source and the destination container. /// - /// The source client to check the contents and compare against the destination. + /// The source client to check the contents and compare against the destination. /// The destinatiojn client to check the contents and compare against the source. /// Optional. The prefix to start listing at the source container. /// Optional. The prefix to start listing at the destination container. /// - protected abstract Task VerifyResults( - TSourceContainerClient sourceClient, + protected abstract Task VerifyResultsAsync( + TSourceContainerClient sourceContainer, + string sourcePrefix, TDestinationContainerClient destinationContainer, - string sourcePrefix = default, - string destinationPrefix = default); + string destinationPrefix); #endregion protected string GetNewObjectName() @@ -135,19 +147,20 @@ protected string GetNewObjectName() /// to detect when the upload has finished. /// /// The source container which will contains the source blobs - /// The source blob prefix/folder - /// The destination local path to download the blobs to + /// The source blob prefix/folder + /// The destination local path to download the blobs to /// /// How long we should wait until we cancel the operation. If this timeout is reached the test will fail. /// /// Options for the transfer manager /// Options for the transfer Options /// - private async Task CopyBlobDirectoryAndVerify( + private async Task CopyDirectoryAndVerifyAsync( TSourceContainerClient sourceContainer, TDestinationContainerClient destinationContainer, - string sourceBlobPrefix, - string destinationBlobPrefix, + string sourcePrefix, + string destinationPrefix, + int itemTransferCount, int waitTimeInSec = 30, TransferManagerOptions transferManagerOptions = default, DataTransferOptions options = default) @@ -165,9 +178,9 @@ private async Task CopyBlobDirectoryAndVerify( TransferManager transferManager = new TransferManager(transferManagerOptions); StorageResourceContainer sourceResource = - GetSourceStorageResourceContainer(sourceContainer, sourceBlobPrefix); + GetSourceStorageResourceContainer(sourceContainer, sourcePrefix); StorageResourceContainer destinationResource = - GetDestinationStorageResourceContainer(destinationContainer, destinationBlobPrefix); + GetDestinationStorageResourceContainer(destinationContainer, destinationPrefix); DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); @@ -175,44 +188,16 @@ private async Task CopyBlobDirectoryAndVerify( CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(waitTimeInSec)); await transfer.WaitForCompletionAsync(tokenSource.Token); - await testEventFailed.AssertContainerCompletedCheck(sourceFiles.Count); + await testEventFailed.AssertContainerCompletedCheck(itemTransferCount); Assert.IsTrue(transfer.HasCompleted); Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // List all files in source blob folder path - List sourceblobNames = new List(); - await foreach (Page page in sourceContainer.GetBlobsAsync(prefix: sourceBlobPrefix).AsPages()) - { - sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); - } - - // List all files in the destination blob folder path - List destblobNames = new List(); - await foreach (Page page in sourceContainer.GetBlobsAsync(prefix: destinationBlobPrefix).AsPages()) - { - destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); - } - Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); - sourceblobNames.Sort(); - destblobNames.Sort(); - for (int i = 0; i < sourceFiles.Count; i++) - { - // Verify file name to match the - // (prefix folder path) + (the blob name without the blob folder prefix) - string sourceNonPrefixed = sourceblobNames[i].Substring(sourceBlobPrefix.Length + 1); - Assert.AreEqual( - sourceNonPrefixed, - destblobNames[i].Substring(destinationBlobPrefix.Length + 1)); - - // Verify Download - string sourceFileName = Path.Combine(sourceFilePrefix, sourceNonPrefixed); - using (FileStream fileStream = File.OpenRead(sourceFileName)) - { - BlockBlobClient destinationBlob = sourceContainer.GetBlockBlobClient(destblobNames[i]); - Assert.IsTrue(await destinationBlob.ExistsAsync()); - await DownloadAndAssertAsync(fileStream, destinationBlob); - } - } + await VerifyResultsAsync( + sourceContainer: sourceContainer, + sourcePrefix: sourcePrefix, + destinationContainer: destinationContainer, + destinationPrefix: destinationPrefix); } [Test] @@ -220,40 +205,40 @@ private async Task CopyBlobDirectoryAndVerify( [TestCase(0, 10)] [TestCase(100, 10)] [TestCase(Constants.KB, 10)] - public async Task BlockBlobDirectoryToDirectory_SmallSize(long size, int waitTimeInSec) + public async Task DirectoryToDirectory_SmallSize(long size, int waitTimeInSec) { // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - string sourceBlobDirectoryName = "sourceFolder"; + string sourceDirectoryName = "sourceFolder"; using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName1, size); - await CreateObjectInSource(source.Container, blobName2, size); - blobNames.Add(blobName1); - blobNames.Add(blobName2); + string itemName1 = Path.Combine(sourceDirectoryName, GetNewObjectName()); + string itemName2 = Path.Combine(sourceDirectoryName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName1); + await CreateObjectInSourceAsync(source.Container, size, itemName2); + blobNames.Add(itemName1); + blobNames.Add(itemName2); string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName3, size); - blobNames.Add(blobName3); + string itemName3 = Path.Combine(sourceDirectoryName, subDirName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName3); + blobNames.Add(itemName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName4, size); - blobNames.Add(blobName4); + string itemName4 = Path.Combine(sourceDirectoryName, subDirName2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName4); + blobNames.Add(itemName4); - await CopyBlobDirectoryAndVerify( + await CopyDirectoryAndVerifyAsync( source.Container, destination.Container, - sourceBlobDirectoryName, + sourceDirectoryName, sourceFolderPath, waitTimeInSec).ConfigureAwait(false); } @@ -265,7 +250,7 @@ await CopyBlobDirectoryAndVerify( [TestCase(4 * Constants.MB, 200)] [TestCase(257 * Constants.MB, 500)] [TestCase(Constants.GB, 500)] - public async Task BlockBlobDirectoryToDirectory_LargeSize(long size, int waitTimeInSec) + public async Task DirectoryToDirectory_LargeSize(long size, int waitTimeInSec) { // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); @@ -274,26 +259,26 @@ public async Task BlockBlobDirectoryToDirectory_LargeSize(long size, int waitTim List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName1, size); - await CreateObjectInSource(source.Container, blobName2, size); - blobNames.Add(blobName1); - blobNames.Add(blobName2); + string itemName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + string itemName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName1); + await CreateObjectInSourceAsync(source.Container, size, itemName2); + blobNames.Add(itemName1); + blobNames.Add(itemName2); string subDirName = "bar"; - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName3, size); - blobNames.Add(blobName3); + string itemName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName3); + blobNames.Add(itemName3); string subDirName2 = "pik"; - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); - await CreateObjectInSource(source.Container, blobName4, size); - blobNames.Add(blobName4); + string itemName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName4); + blobNames.Add(itemName4); string destinationFolder = "destFolder"; - await CopyBlobDirectoryAndVerify( + await CopyDirectoryAndVerifyAsync( source.Container, destination.Container, sourceBlobDirectoryName, @@ -303,25 +288,19 @@ await CopyBlobDirectoryAndVerify( [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_EmptyFolder() + public async Task DirectoryToDirectory_EmptyFolder() { // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); // Set up directory to upload - var dirName = GetNewObjectName(); - var dirName2 = GetNewObjectName(); - string folder = CreateRandomDirectory(testDirectory.DirectoryPath); + var destinationName = GetNewObjectName(); + var sourceName = GetNewObjectName(); // Set up destination client - StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = dirName }); - StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(test.Container, - new BlobStorageResourceContainerOptions() - { - BlobDirectoryPrefix = dirName2, - }); + StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destination.Container, destinationName); + StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(source.Container, sourceName); TransferManagerOptions managerOptions = new TransferManagerOptions() { @@ -341,73 +320,63 @@ public async Task BlockBlobDirectoryToDirectory_EmptyFolder() Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // Assert - List blobs = ((List)await test.Container.GetBlobsAsync().ToListAsync()) - .Select((BlobItem blob) => blob.Name).ToList(); - // Assert - Assert.IsEmpty(blobs); + await VerifyEmptyDestinationContainerAsync(destination.Container, destinationName); testEventsRaised.AssertUnexpectedFailureCheck(); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_SingleFile() + public async Task DirectoryToDirectory_SingleFile() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string sourceFolderName = "sourceFolder"; - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceFolderName); + string sourcePrefix = "sourceFolder"; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourcePrefix); - string blobName1 = Path.Combine(sourceFolderName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); - List blobNames = new List() { blobName1 }; + string itemName1 = Path.Combine(sourcePrefix, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName1); - string destinationFolder = "destFolder"; + string destinationPrefix = "destFolder"; - await CopyBlobDirectoryAndVerify( - container: test.Container, - sourceBlobPrefix: sourceFolderName, - sourceFilePrefix: sourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + await CopyDirectoryAndVerifyAsync( + sourceContainer: source.Container, + destinationContainer: destination.Container, + sourcePrefix: sourcePrefix, + destinationPrefix: destinationPrefix, + itemTransferCount: 1).ConfigureAwait(false); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_ManySubDirectories() + public async Task DirectoryToDirectory_ManySubDirectories() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - - string blobDirectoryName = "sourceFolder"; - string fullSourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, blobDirectoryName); - - List blobNames = new List(); - string subDir1 = CreateRandomDirectory(fullSourceFolderPath, "bar").Substring(fullSourceFolderPath.Length + 1); - string blobName1 = Path.Combine(blobDirectoryName, subDir1, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, Constants.KB); - blobNames.Add(blobName1); - string subDir2 = CreateRandomDirectory(fullSourceFolderPath, "rul").Substring(fullSourceFolderPath.Length + 1); - string blobName2 = Path.Combine(blobDirectoryName, subDir2, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, Constants.KB); - blobNames.Add(blobName2); - string subDir3 = CreateRandomDirectory(fullSourceFolderPath, "pik").Substring(fullSourceFolderPath.Length + 1); - string blobName3 = Path.Combine(blobDirectoryName, subDir3, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, Constants.KB); - blobNames.Add(blobName3); - - string destinationFolder = "destFolder"; - - string sourceBlobPrefix = fullSourceFolderPath.Substring(testDirectory.DirectoryPath.Length + 1); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - await CopyBlobDirectoryAndVerify( - container: test.Container, - sourceBlobPrefix: sourceBlobPrefix, - sourceFilePrefix: fullSourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + string sourcePrefix = "sourceFolder"; + + string subDir1 = string.Join("/", sourcePrefix, "foo"); + string itemName1 = Path.Combine(sourcePrefix, subDir1, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName1); + string subDir2 = string.Join("/", sourcePrefix, "rul"); + string itemName2 = Path.Combine(sourcePrefix, subDir2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName2); + string subDir3 = string.Join("/", sourcePrefix, "pik"); + string itemName3 = Path.Combine(sourcePrefix, subDir3, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName3); + + string destinationPrefix = "destFolder"; + + await CopyDirectoryAndVerifyAsync( + sourceContainer: source.Container, + destinationContainer: destination.Container, + sourcePrefix: sourcePrefix, + destinationPrefix: destinationPrefix, + itemTransferCount: 3).ConfigureAwait(false); } [Test] @@ -415,208 +384,186 @@ await CopyBlobDirectoryAndVerify( [TestCase(1)] [TestCase(2)] [TestCase(3)] - public async Task BlockBlobDirectoryToDirectory_SubDirectoriesLevels(int level) + public async Task DirectoryToDirectory_SubDirectoriesLevels(int level) { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - - string sourceBlobDirectoryName = "sourceFolder"; - string fullSourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - List blobNames = new List(); + string sourcePrefix = "sourceFolder"; - string subDir = default; for (int i = 0; i < level; i++) { - subDir = CreateRandomDirectory(fullSourceFolderPath, $"folder{i}"); - string blobName = Path.Combine(sourceBlobDirectoryName, subDir.Substring(fullSourceFolderPath.Length + 1), GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName, Constants.KB); - blobNames.Add(blobName); + string subDirPrefix = $"{sourcePrefix}/folder{i}/{GetNewObjectName()}"; + await CreateObjectInSourceAsync(source.Container, Constants.KB, subDirPrefix); } - string destinationFolder = "destFolder"; + string destinationPrefix = "destFolder"; - await CopyBlobDirectoryAndVerify( - test.Container, - sourceBlobDirectoryName, - fullSourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + await CopyDirectoryAndVerifyAsync( + source.Container, + destination.Container, + sourcePrefix, + destinationPrefix: destinationPrefix, + itemTransferCount: level).ConfigureAwait(false); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_OverwriteTrue() + public async Task DirectoryToDirectory_OverwriteTrue() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); long size = Constants.KB; - string sourceBlobDirectoryName = "sourceFolder"; - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + string sourcePrefix = "sourceFolder"; - List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); - blobNames.Add(blobName1); - blobNames.Add(blobName2); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName1); + await CreateObjectInSourceAsync(source.Container, size, itemName2); string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); - blobNames.Add(blobName3); + string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName3); string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); - blobNames.Add(blobName4); + string itemName4 = string.Join("/", sourcePrefix, subDirName2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName4); DataTransferOptions options = new DataTransferOptions() { CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - string destinationFolder = "destFolder"; + string destinationPrefix = "destPrefix"; // Act - await CopyBlobDirectoryAndVerify( - test.Container, - sourceBlobDirectoryName, - sourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames, + await CopyDirectoryAndVerifyAsync( + source.Container, + destination.Container, + sourcePrefix, + destinationPrefix, + itemTransferCount: 4, options: options).ConfigureAwait(false); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_OverwriteFalse() + public async Task DirectoryToDirectory_OverwriteFalse() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); long size = Constants.KB; - string sourceBlobDirectoryName = "sourceFolder"; - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); + string sourcePrefix = "sourceFolder"; List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName1, size); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName2, size); - blobNames.Add(blobName1); - blobNames.Add(blobName2); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName1); + await CreateObjectInSourceAsync(source.Container, size, itemName2); + blobNames.Add(itemName1); + blobNames.Add(itemName2); string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName3, size); - blobNames.Add(blobName3); + string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName3); + blobNames.Add(itemName3); string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(test.Container, testDirectory.DirectoryPath, blobName4, size); - blobNames.Add(blobName4); + string itemName4 = string.Join("/", sourcePrefix, subDirName2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName4); + blobNames.Add(itemName4); DataTransferOptions options = new DataTransferOptions() { CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - string destinationFolder = "destFolder"; + string destinationPrefix = "destFolder"; // Act - await CopyBlobDirectoryAndVerify( - test.Container, - sourceBlobDirectoryName, - sourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames, + await CopyDirectoryAndVerifyAsync( + source.Container, + destination.Container, + sourcePrefix, + destinationPrefix, + itemTransferCount: 4, options: options).ConfigureAwait(false); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task BlockBlobDirectoryToDirectory_OAuth() + public async Task DirectoryToDirectory_OAuth() { // Arrange long size = Constants.KB; int waitTimeInSec = 10; - TSourceServiceClient service = BlobsClientBuilder.GetServiceClient_OAuth(); - var containerName = GetNewContainerName(); - await using IDisposingContainer testContainer = await GetTestContainerAsync( - service, - containerName, - publicAccessType: PublicAccessType.BlobContainer); - string sourceBlobDirectoryName = "sourceFolder"; + TSourceServiceClient service = GetOAuthSourceServiceClient(); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(service); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + + string sourcePrefix = "sourceFolder"; using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobDirectoryName); List blobNames = new List(); - string blobName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string blobName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName1, size); - await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName2, size); - blobNames.Add(blobName1); - blobNames.Add(blobName2); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName1); + await CreateObjectInSourceAsync(source.Container, size, itemName2); + blobNames.Add(itemName1); + blobNames.Add(itemName2); string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName3, size); - blobNames.Add(blobName3); + string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName3); + blobNames.Add(itemName3); string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); - await CreateBlockBlobAndSourceFile(testContainer.Container, testDirectory.DirectoryPath, blobName4, size); - blobNames.Add(blobName4); + string itemName4 = Path.Combine(sourcePrefix, subDirName2, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, size, itemName4); + blobNames.Add(itemName4); - string destinationFolder = "destFolder"; + string destinationPrefix = "destFolder"; - await CopyBlobDirectoryAndVerify( - testContainer.Container, - sourceBlobDirectoryName, - sourceFolderPath, - destinationFolder, - blobNames, + await CopyDirectoryAndVerifyAsync( + source.Container, + destination.Container, + sourcePrefix, + destinationPrefix, waitTimeInSec).ConfigureAwait(false); } #region Single Concurrency - private async Task CreateBlobDirectoryTree( + private async Task CreateDirectoryTree( TSourceContainerClient client, string sourceFolderPath, string sourceBlobDirectoryName, int size) { - string blobName1 = Path.Combine(sourceBlobDirectoryName, "blob1"); - string blobName2 = Path.Combine(sourceBlobDirectoryName, "blob2"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName1, size); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName2, size); + string itemName1 = Path.Combine(sourceBlobDirectoryName, "blob1"); + string itemName2 = Path.Combine(sourceBlobDirectoryName, "blob2"); + await CreateObjectInSourceAsync(client, size, itemName1); + await CreateObjectInSourceAsync(client, size, itemName2); string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, "blob3"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName3, size); + string itemName3 = Path.Combine(sourceBlobDirectoryName, subDirName, "blob3"); + await CreateObjectInSourceAsync(client, size, itemName3); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, "blob4"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName4, size); + string itemName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, "blob4"); + await CreateObjectInSourceAsync(client, size, itemName4); } private async Task CreateStartTransfer( - TSourceContainerClient containerClient, + TSourceContainerClient sourceContainer, + TDestinationContainerClient destinationContainer, int concurrency, bool createFailedCondition = false, DataTransferOptions options = default, @@ -628,21 +575,17 @@ private async Task CreateStartTransfer( string sourceBlobPrefix = "sourceFolder"; string destBlobPrefix = "destFolder"; string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobPrefix); - await CreateBlobDirectoryTree(containerClient, sourceFolderPath, sourceBlobPrefix, size); + await CreateDirectoryTree(sourceContainer, sourceFolderPath, sourceBlobPrefix, size); // Create new source block blob. - StorageResourceContainer sourceResource = new BlobStorageResourceContainer(containerClient, new() { BlobDirectoryPrefix = sourceBlobPrefix }); - StorageResourceContainer destinationResource = new BlobStorageResourceContainer(containerClient, - new BlobStorageResourceContainerOptions() - { - BlobDirectoryPrefix = destBlobPrefix, - }); + StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(sourceContainer, sourceBlobPrefix); + StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destinationContainer, destBlobPrefix); // If we want a failure condition to happen if (createFailedCondition) { string destBlobName = $"{destBlobPrefix}/blob1"; - await CreateBlockBlob(containerClient, Path.Combine(testDirectory.DirectoryPath, "blob1"), destBlobName, size); + await CreateObjectInDestinationAsync(destinationContainer, size, destBlobName); } // Create Transfer Manager with single threaded operation @@ -664,12 +607,17 @@ private async Task CreateStartTransfer( public async Task StartTransfer_AwaitCompletion() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); // Create transfer to do a AwaitCompletion DataTransferOptions options = new DataTransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - DataTransfer transfer = await CreateStartTransfer(test.Container, 1, options: options); + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + 1, + options: options); // Act CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); @@ -687,7 +635,8 @@ public async Task StartTransfer_AwaitCompletion() public async Task StartTransfer_AwaitCompletion_Failed() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); DataTransferOptions options = new DataTransferOptions() { @@ -697,7 +646,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() // Create transfer to do a AwaitCompletion DataTransfer transfer = await CreateStartTransfer( - test.Container, + source.Container, + destination.Container, 1, createFailedCondition: true, options: options); @@ -720,7 +670,8 @@ public async Task StartTransfer_AwaitCompletion_Failed() public async Task StartTransfer_AwaitCompletion_Skipped() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); // Create transfer options with Skipping available DataTransferOptions options = new DataTransferOptions() @@ -731,7 +682,8 @@ public async Task StartTransfer_AwaitCompletion_Skipped() // Create transfer to do a AwaitCompletion DataTransfer transfer = await CreateStartTransfer( - test.Container, + source.Container, + destination.Container, 1, createFailedCondition: true, options: options); @@ -753,13 +705,18 @@ public async Task StartTransfer_AwaitCompletion_Skipped() public async Task StartTransfer_EnsureCompleted() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); // Create transfer to do a EnsureCompleted DataTransferOptions options = new DataTransferOptions(); TestEventsRaised testEventsRaised = new TestEventsRaised(options); - DataTransfer transfer = await CreateStartTransfer(test.Container, 1, options: options); + DataTransfer transfer = await CreateStartTransfer( + source.Container, + destination.Container, + 1, + options: options); // Act CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); @@ -777,7 +734,8 @@ public async Task StartTransfer_EnsureCompleted() public async Task StartTransfer_EnsureCompleted_Failed() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); DataTransferOptions options = new DataTransferOptions() { @@ -787,7 +745,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() // Create transfer to do a AwaitCompletion DataTransfer transfer = await CreateStartTransfer( - test.Container, + source.Container, + destination.Container, 1, createFailedCondition: true, options: options); @@ -810,7 +769,8 @@ public async Task StartTransfer_EnsureCompleted_Failed() public async Task StartTransfer_EnsureCompleted_Skipped() { // Arrange - await using IDisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); // Create transfer options with Skipping available DataTransferOptions options = new DataTransferOptions() @@ -821,7 +781,8 @@ public async Task StartTransfer_EnsureCompleted_Skipped() // Create transfer to do a EnsureCompleted DataTransfer transfer = await CreateStartTransfer( - test.Container, + source.Container, + destination.Container, 1, createFailedCondition: true, options: options); @@ -845,8 +806,6 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string destinationFolder = CreateRandomDirectory(testDirectory.DirectoryPath); DataTransferOptions options = new DataTransferOptions() { @@ -858,7 +817,8 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() // Create transfer to do a AwaitCompletion DataTransfer transfer = await CreateStartTransfer( - test.Container, + source.Container, + destination.Container, 1, createFailedCondition: true, options: options, From 6467551dbe9739c8e82cda1d10b66014915abba2 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Wed, 25 Oct 2023 22:21:00 -0700 Subject: [PATCH 05/11] Enabled directory creation during enumeration; Tests; Some download directory tests don't work --- .../src/BlobStorageResourceContainer.cs | 14 + .../src/DataMovementBlobsExtensions.cs | 18 ++ .../src/DataMovementSharesExtensions.cs | 4 +- .../ShareDirectoryStorageResourceContainer.cs | 23 +- .../src/ShareFileStorageResource.cs | 4 +- .../tests/ClientBuilderExtensions.cs | 12 +- .../ShareDirectoryStartTransferCopyTests.cs | 79 +++-- ...eDirectoryStorageResourceContainerTests.cs | 111 ++++++- .../LocalDirectoryStorageResourceContainer.cs | 27 +- .../src/ServiceToServiceTransferJob.cs | 50 +-- .../StorageResourceContainerInternal.cs | 7 + .../src/StorageResourceContainer.cs | 16 + .../src/StreamToUriTransferJob.cs | 50 +-- .../BlobStorageResourceContainerTests.cs | 23 ++ .../LocalDirectoryStorageResourceTests.cs | 45 +++ .../Shared/MemoryStorageResourceContainer.cs | 10 + .../StartTransferDirectoryCopyTestBase.cs | 286 ++++++++++-------- .../src/ShareFileClient.cs | 2 +- .../Azure.Storage.Files.Shares.Tests.csproj | 50 --- 19 files changed, 574 insertions(+), 257 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs index 02ea5e1f1fef6..ef99e5e8f20a2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs @@ -150,5 +150,19 @@ private string ApplyOptionalPrefix(string path) => IsDirectory ? string.Join("/", DirectoryPrefix, path) : path; + + // We will require containers to be created before the transfer starts + // Since blobs is a flat namespace, we do not need to create directories (as they are virtual). + protected override Task CreateIfNotExistsAsync(CancellationToken cancellationToken) + => Task.CompletedTask; + + protected override StorageResourceContainer GetChildStorageResourceContainer(string path) + { + BlobStorageResourceContainerOptions options = _options.DeepCopy(); + options.BlobDirectoryPrefix = string.Join("/", DirectoryPrefix, path); + return new BlobStorageResourceContainer( + BlobContainerClient, + options); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs index 52243239fb1d3..0debaf844ff99 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs @@ -471,5 +471,23 @@ internal static BlobStorageResourceContainerOptions GetBlobContainerOptions( BlobOptions = baseOptions, }; } + + internal static BlobStorageResourceContainerOptions DeepCopy(this BlobStorageResourceContainerOptions options) + => new BlobStorageResourceContainerOptions() + { + BlobType = options?.BlobType ?? BlobType.Block, + BlobDirectoryPrefix = options?.BlobDirectoryPrefix, + BlobOptions = new BlobStorageResourceOptions() + { + Metadata = options?.BlobOptions?.Metadata, + Tags = options?.BlobOptions?.Tags, + HttpHeaders = options?.BlobOptions?.HttpHeaders, + AccessTier = options?.BlobOptions?.AccessTier, + DestinationImmutabilityPolicy = options?.BlobOptions?.DestinationImmutabilityPolicy, + LegalHold = options?.BlobOptions?.LegalHold, + UploadTransferValidationOptions = options?.BlobOptions?.UploadTransferValidationOptions, + DownloadTransferValidationOptions = options?.BlobOptions?.DownloadTransferValidationOptions, + } + }; } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs index eca0c2ed3d99b..58cf1f38ef18a 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementSharesExtensions.cs @@ -25,10 +25,12 @@ internal static ShareFileUploadRangeOptions ToShareFileUploadRangeOptions( }; internal static ShareFileUploadRangeFromUriOptions ToShareFileUploadRangeFromUriOptions( - this ShareFileStorageResourceOptions options) + this ShareFileStorageResourceOptions options, + HttpAuthorization sourceAuthorization) => new() { Conditions = options?.DestinationConditions, + SourceAuthentication = sourceAuthorization }; internal static StorageResourceProperties ToStorageResourceProperties( diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs index 9f3a44265bfde..2c1579bdfa70c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs @@ -43,10 +43,17 @@ protected override StorageResourceItem GetStorageResourceReference(string path) protected override async IAsyncEnumerable GetStorageResourcesAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (ShareFileClient client in PathScanner.ScanFilesAsync( + await foreach ((ShareDirectoryClient dir, ShareFileClient file) in PathScanner.ScanAsync( ShareDirectoryClient, cancellationToken).ConfigureAwait(false)) { - yield return new ShareFileStorageResource(client, ResourceOptions); + if (file != default) + { + yield return new ShareFileStorageResource(file, ResourceOptions); + } + else + { + yield return new ShareDirectoryStorageResourceContainer(dir, ResourceOptions); + } } } @@ -59,5 +66,17 @@ public override StorageResourceCheckpointData GetDestinationCheckpointData() { return new ShareFileDestinationCheckpointData(); } + + protected override async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) + { + await ShareDirectoryClient.CreateIfNotExistsAsync( + metadata: default, + smbProperties: default, + filePermission: default, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + protected override StorageResourceContainer GetChildStorageResourceContainer(string path) + => new ShareDirectoryStorageResourceContainer(ShareDirectoryClient.GetSubdirectoryClient(path), ResourceOptions); } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index 9f7f2aea0cb6f..fcec15632705c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -116,7 +116,7 @@ await ShareFileClient.UploadRangeFromUriAsync( sourceUri: sourceResource.Uri, range: range, sourceRange: range, - options: _options.ToShareFileUploadRangeFromUriOptions(), + options: _options.ToShareFileUploadRangeFromUriOptions(options?.SourceAuthentication), cancellationToken: cancellationToken).ConfigureAwait(false); } @@ -165,7 +165,7 @@ await ShareFileClient.UploadRangeFromUriAsync( sourceUri: sourceResource.Uri, range: new HttpRange(0, completeLength), sourceRange: new HttpRange(0, completeLength), - options: _options.ToShareFileUploadRangeFromUriOptions(), + options: _options.ToShareFileUploadRangeFromUriOptions(options?.SourceAuthentication), cancellationToken: cancellationToken).ConfigureAwait(false); } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs index 973d321c3e8ee..5270b092cfd15 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs @@ -11,6 +11,7 @@ Azure.Storage.Files.Shares.ShareServiceClient, Azure.Storage.Files.Shares.ShareClientOptions>; using Azure.Storage.Files.Shares.Models; +using Azure.Core.TestFramework; namespace Azure.Storage.DataMovement.Files.Shares.Tests { @@ -43,17 +44,6 @@ public static SharesClientBuilder GetNewShareClientBuilder(TenantConfigurationBu (uri, azureSasCredential, clientOptions) => new ShareServiceClient(uri, azureSasCredential, clientOptions), () => new ShareClientOptions(serviceVersion)); - public static ShareServiceClient GetServiceClient_OAuthAccount_SharedKey(this SharesClientBuilder clientBuilder) => - clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigOAuth); - - public static ShareServiceClient GetServiceClient_OAuth( - this SharesClientBuilder clientBuilder, ShareClientOptions options = default) - { - options ??= clientBuilder.GetOptions(); - options.ShareTokenIntent = ShareTokenIntent.Backup; - return clientBuilder.GetServiceClientFromOauthConfig(clientBuilder.Tenants.TestConfigOAuth, options); - } - public static async Task GetTestShareAsync( this SharesClientBuilder clientBuilder, ShareServiceClient service = default, diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index 5c71bff96cc5d..41933b19e8ff3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -16,6 +16,7 @@ using Azure.Storage.Files.Shares.Tests; using NUnit.Framework; using System.Security.AccessControl; +using Microsoft.Extensions.Options; namespace Azure.Storage.DataMovement.Files.Shares.Tests { @@ -59,15 +60,42 @@ protected override async Task> GetDestinationDi protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string prefix) => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); - protected override ShareServiceClient GetOAuthSourceServiceClient() - => SourceClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth); + protected override ShareClient GetOAuthSourceContainerClient(string containerName) + { + ShareClientOptions options = SourceClientBuilder.GetOptions(); + options.ShareTokenIntent = ShareTokenIntent.Backup; + ShareServiceClient oauthService = SourceClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth, options); + return oauthService.GetShareClient(containerName); + } + + protected override ShareClient GetOAuthDestinationContainerClient(string containerName) + { + ShareClientOptions options = DestinationClientBuilder.GetOptions(); + options.ShareTokenIntent = ShareTokenIntent.Backup; + ShareServiceClient oauthService = DestinationClientBuilder.GetServiceClientFromOauthConfig(Tenants.TestConfigOAuth, options); + return oauthService.GetShareClient(containerName); + } protected override async Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) - => await SourceClientBuilder.GetTestShareAsync(service, containerName); + { + service ??= SourceClientBuilder.GetServiceClientFromSharedKeyConfig(SourceClientBuilder.Tenants.TestConfigDefault, SourceClientBuilder.GetOptions()); + ShareServiceClient sasService = new ShareServiceClient(service.GenerateAccountSasUri( + Sas.AccountSasPermissions.All, + SourceClientBuilder.Recording.UtcNow.AddDays(1), + Sas.AccountSasResourceTypes.All), + SourceClientBuilder.GetOptions()); + return await SourceClientBuilder.GetTestShareAsync(sasService, containerName); + } protected override StorageResourceContainer GetSourceStorageResourceContainer(ShareClient containerClient, string prefix = null) => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); + protected override async Task CreateDirectoryInSourceAsync(ShareClient sourceContainer, string directoryPath) + => await CreateDirectoryTree(sourceContainer, directoryPath); + + protected override async Task CreateDirectoryInDestinationAsync(ShareClient destinationContainer, string directoryPath) + => await CreateDirectoryTree(destinationContainer, directoryPath); + protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient destinationContainer, string destinationPrefix) { ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ? @@ -84,7 +112,8 @@ protected override async Task VerifyResultsAsync( string destinationPrefix) { // List all files in source blob folder path - List sourceNames = new List(); + List sourceFileNames = new List(); + List sourceDirectoryNames = new List(); // Get source directory client and list the paths ShareDirectoryClient sourceDirectory = string.IsNullOrEmpty(sourcePrefix) ? @@ -92,36 +121,39 @@ protected override async Task VerifyResultsAsync( sourceContainer.GetDirectoryClient(sourcePrefix); await foreach (Page page in sourceDirectory.GetFilesAndDirectoriesAsync().AsPages()) { - sourceNames.AddRange(page.Values.Select((ShareFileItem item) => item.Name)); + sourceFileNames.AddRange(page.Values.Where((ShareFileItem item) => !item.IsDirectory).Select((ShareFileItem item) => item.Name)); + sourceDirectoryNames.AddRange(page.Values.Where((ShareFileItem item) => item.IsDirectory).Select((ShareFileItem item) => item.Name)); } // List all files in the destination blob folder path - List destinationNames = new List(); + List destinationFileNames = new List(); + List destinationDirectoryNames = new List(); ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ? destinationContainer.GetRootDirectoryClient() : destinationContainer.GetDirectoryClient(destinationPrefix); await foreach (Page page in destinationDirectory.GetFilesAndDirectoriesAsync().AsPages()) { - destinationNames.AddRange(page.Values.Select((ShareFileItem item) => item.Name)); + destinationFileNames.AddRange(page.Values.Where((ShareFileItem item) => !item.IsDirectory).Select((ShareFileItem item) => item.Name)); + destinationDirectoryNames.AddRange(page.Values.Where((ShareFileItem item) => item.IsDirectory).Select((ShareFileItem item) => item.Name)); } - Assert.AreEqual(sourceNames.Count, destinationNames.Count); - sourceNames.Sort(); - destinationNames.Sort(); - for (int i = 0; i < sourceNames.Count; i++) + + // Assert subdirectories + Assert.AreEqual(sourceDirectoryNames.Count, destinationDirectoryNames.Count); + Assert.AreEqual(sourceDirectoryNames, destinationDirectoryNames); + + // Assert file and file contents + Assert.AreEqual(sourceFileNames.Count, destinationFileNames.Count); + for (int i = 0; i < sourceFileNames.Count; i++) { - // Verify file name to match the - // (prefix folder path) + (the blob name without the blob folder prefix) - // TODO: verify if files returns the entire path, and if we really need to be parsing the prefix - string sourceNonPrefixed = sourceNames[i].Substring(sourcePrefix.Length + 1); Assert.AreEqual( - sourceNonPrefixed, - destinationNames[i].Substring(destinationPrefix.Length + 1)); + sourceFileNames[i], + destinationFileNames[i]); // Verify Download - string sourceFileName = Path.Combine(sourcePrefix, sourceNonPrefixed); - using Stream sourceStream = await sourceDirectory.GetFileClient(sourceNames[i]).OpenReadAsync(); - using Stream destinationStream = await destinationDirectory.GetFileClient(destinationNames[i]).OpenReadAsync(); + string sourceFileName = Path.Combine(sourcePrefix, sourceFileNames[i]); + using Stream sourceStream = await sourceDirectory.GetFileClient(sourceFileNames[i]).OpenReadAsync(); + using Stream destinationStream = await destinationDirectory.GetFileClient(destinationFileNames[i]).OpenReadAsync(); Assert.AreEqual(sourceStream, destinationStream); } } @@ -145,5 +177,12 @@ private async Task CreateShareFileAsync( await fileClient.UploadAsync(contents); } } + + private async Task CreateDirectoryTree(ShareClient container, string directoryPath) + { + // Parse for parent directory names and create the parent directory(s). + ShareDirectoryClient directory = container.GetRootDirectoryClient().GetSubdirectoryClient(directoryPath); + await directory.CreateIfNotExistsAsync(); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStorageResourceContainerTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStorageResourceContainerTests.cs index 327b072cd6376..1b06dcd0a35f6 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStorageResourceContainerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStorageResourceContainerTests.cs @@ -7,8 +7,12 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Azure.Core.TestFramework; using Azure.Storage.Files.Shares; +using Azure.Storage.Files.Shares.Models; +using Azure.Storage.Test; using Azure.Storage.Tests; +using BenchmarkDotNet.Toolchains.Roslyn; using Moq; using NUnit.Framework; @@ -16,13 +20,19 @@ namespace Azure.Storage.DataMovement.Files.Shares.Tests { internal class ShareDirectoryStorageResourceContainerTests { - private async IAsyncEnumerable ToAsyncEnumerable(IEnumerable items) + private async IAsyncEnumerable<(TDirectory Directory, TFile File)> ToAsyncEnumerable( + IEnumerable directories, + IEnumerable files) { - foreach (var item in items) + if (files.Count() != directories.Count()) + { + throw new ArgumentException("Items and Directories should be the same amount"); + } + for (int i = 0; i < files.Count(); i++) { // returning async enumerable must be an async method // so we need something to await - yield return await Task.FromResult(item); + yield return await Task.FromResult((directories.ElementAt(i), files.ElementAt(i))); } } @@ -31,15 +41,21 @@ public async Task GetStorageResourcesCallsPathScannerCorrectly() { // Given clients Mock mainClient = new(); - List> expectedFiles = Enumerable.Range(0, 10) + int pathCount = 10; + List> expectedFiles = Enumerable.Range(0, pathCount) .Select(i => new Mock()) .ToList(); + List> expectedDirectories = Enumerable.Range(0, pathCount) + .Select(i => new Mock()) + .ToList(); // And a mock path scanner Mock pathScanner = new(); - pathScanner.Setup(p => p.ScanFilesAsync(mainClient.Object, It.IsAny())) + pathScanner.Setup(p => p.ScanAsync(mainClient.Object, It.IsAny())) .Returns( - (dir, cancellationToken) => ToAsyncEnumerable(expectedFiles.Select(m => m.Object))); + (dir, cancellationToken) => ToAsyncEnumerable( + expectedDirectories.Select(m => m.Object), + expectedFiles.Select(m => m.Object))); // Setup StorageResourceContainer ShareDirectoryStorageResourceContainer resource = new(mainClient.Object, default) @@ -61,7 +77,7 @@ public async Task GetStorageResourcesCallsPathScannerCorrectly() public void GetCorrectStorageResourceItem() { // Given a resource container - ShareDirectoryClient startingDir = new(new Uri("https://myaccount.file.core.windows.net/myshare/mydir")); + ShareDirectoryClient startingDir = new(new Uri("https://myaccount.file.core.windows.net/myshare/mydir"), new ShareClientOptions()); ShareDirectoryStorageResourceContainer resourceContainer = new(startingDir, default); // and a subpath to get @@ -79,5 +95,86 @@ public void GetCorrectStorageResourceItem() Is.EqualTo(startingDir.Path + "/" + string.Join("/", pathSegments))); Assert.That(fileResourceItem.ShareFileClient.Name, Is.EqualTo(pathSegments.Last())); } + + [Test] + public async Task CreateIfNotExists() + { + // Arrange + Mock mock = new(new Uri("https://myaccount.file.core.windows.net/myshare/mydir"), new ShareClientOptions()); + mock.Setup(b => b.CreateIfNotExistsAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + SharesModelFactory.StorageDirectoryInfo( + eTag: new ETag("etag"), + lastModified: DateTimeOffset.UtcNow, + filePermissionKey: default, + fileAttributes: default, + fileCreationTime: DateTimeOffset.MinValue, + fileLastWriteTime: DateTimeOffset.MinValue, + fileChangeTime: DateTimeOffset.MinValue, + fileId: default, + fileParentId: default), + new MockResponse(200)))); + + ShareDirectoryStorageResourceContainer resourceContainer = new(mock.Object, default); + + // Act + await resourceContainer.CreateIfNotExistsInternalAsync(); + + // Assert + mock.Verify(b => b.CreateIfNotExistsAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CreateIfNotExists_Error() + { + // Arrange + Mock mock = new(new Uri("https://myaccount.file.core.windows.net/myshare/mydir"), new ShareClientOptions()); + mock.Setup(b => b.CreateIfNotExistsAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new RequestFailedException(status: 404, message: "The parent path does not exist.", errorCode: "ResourceNotFound", default)); + + ShareDirectoryStorageResourceContainer resourceContainer = new(mock.Object, default); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + resourceContainer.CreateIfNotExistsInternalAsync(), + e => + { + Assert.AreEqual("ResourceNotFound", e.ErrorCode); + }); + + mock.Verify(b => b.CreateIfNotExistsAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public void GetChildStorageResourceContainer() + { + // Arrange + Uri uri = new Uri("https://storageaccount.file.core.windows.net/container/directory"); + Mock mock = new Mock(uri, new ShareClientOptions()); + mock.Setup(b => b.Uri).Returns(uri); + mock.Setup(b => b.GetSubdirectoryClient(It.IsAny())) + .Returns((path) => + { + UriBuilder builder = new UriBuilder(uri); + builder.Path = string.Join("/", builder.Path, path); + return new ShareDirectoryClient(builder.Uri); + }); + + ShareDirectoryStorageResourceContainer containerResource = + new(mock.Object, new ShareFileStorageResourceOptions()); + + // Act + string childPath = "foo"; + StorageResourceContainer childContainer = containerResource.GetChildStorageResourceContainerInternal(childPath); + + // Assert + UriBuilder builder = new UriBuilder(containerResource.Uri); + builder.Path = string.Join("/", builder.Path, childPath); + Assert.AreEqual(builder.Uri, childContainer.Uri); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs index a919df7668101..cb0682180b3a6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs @@ -6,6 +6,7 @@ using System.IO; using System.Runtime.CompilerServices; using System.Threading; +using System.Threading.Tasks; using Azure.Core; namespace Azure.Storage.DataMovement @@ -72,9 +73,17 @@ protected internal override async IAsyncEnumerable GetStorageRe PathScanner scanner = new PathScanner(_uri.LocalPath); foreach (FileSystemInfo fileSystemInfo in scanner.Scan(false)) { - // Skip over directories for now since directory creation is unnecessary. - if (!fileSystemInfo.Attributes.HasFlag(FileAttributes.Directory)) + if (fileSystemInfo.Attributes.HasFlag(FileAttributes.Directory)) { + // Directory - but check for the case where it returns the directory you're currently listing + if (fileSystemInfo.FullName != _uri.LocalPath) + { + yield return new LocalDirectoryStorageResourceContainer(fileSystemInfo.FullName); + } + } + else + { + // File yield return new LocalFileStorageResource(fileSystemInfo.FullName); } } @@ -89,5 +98,19 @@ public override StorageResourceCheckpointData GetDestinationCheckpointData() { return new LocalDestinationCheckpointData(); } + + protected internal override Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) + { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); + Directory.CreateDirectory(_uri.LocalPath); + return Task.CompletedTask; + } + + protected internal override StorageResourceContainer GetChildStorageResourceContainer(string path) + { + UriBuilder uri = new UriBuilder(_uri); + uri.Path = Path.Combine(uri.Path, path); + return new LocalDirectoryStorageResourceContainer(uri.Uri); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs index 94c4ab9d0e36c..1cb8a0b68efd1 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs @@ -173,29 +173,39 @@ private async IAsyncEnumerable GetStorageResourcesAsync() ? current.Uri.GetPath() : current.Uri.GetPath().Substring(containerUriPath.Length + 1); - if (!existingSources.Contains(sourceName)) + if (current.IsContainer) { - // Because AsyncEnumerable doesn't let us know which storage resource is the last resource - // we only yield return when we know this is not the last storage resource to be listed - // from the container. - ServiceToServiceJobPart part; - try - { - part = await ServiceToServiceJobPart.CreateJobPartAsync( - job: this, - partNumber: partNumber, - sourceResource: (StorageResourceItem)current, - destinationResource: _destinationResourceContainer.GetStorageResourceReference(sourceName)) - .ConfigureAwait(false); - AppendJobPart(part); - } - catch (Exception ex) + // Create sub-container + StorageResourceContainer subContainer = + _destinationResourceContainer.GetChildStorageResourceContainer(sourceName); + await subContainer.CreateIfNotExistsAsync().ConfigureAwait(false); + } + else + { + if (!existingSources.Contains(sourceName)) { - await InvokeFailedArgAsync(ex).ConfigureAwait(false); - yield break; + // Because AsyncEnumerable doesn't let us know which storage resource is the last resource + // we only yield return when we know this is not the last storage resource to be listed + // from the container. + ServiceToServiceJobPart part; + try + { + part = await ServiceToServiceJobPart.CreateJobPartAsync( + job: this, + partNumber: partNumber, + sourceResource: (StorageResourceItem)current, + destinationResource: _destinationResourceContainer.GetStorageResourceReference(sourceName)) + .ConfigureAwait(false); + AppendJobPart(part); + } + catch (Exception ex) + { + await InvokeFailedArgAsync(ex).ConfigureAwait(false); + yield break; + } + yield return part; + partNumber++; } - yield return part; - partNumber++; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceContainerInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceContainerInternal.cs index 9aeb4e0d5b908..2edd22a507e5a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceContainerInternal.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceContainerInternal.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; +using System.Threading.Tasks; namespace Azure.Storage.DataMovement { @@ -20,5 +21,11 @@ internal IAsyncEnumerable GetStorageResourcesInternalAsync( internal StorageResourceItem GetStorageResourceReferenceInternal(string path) => GetStorageResourceReference(path); + + internal Task CreateIfNotExistsInternalAsync(CancellationToken cancellationToken = default) + => CreateIfNotExistsAsync(cancellationToken); + + internal StorageResourceContainer GetChildStorageResourceContainerInternal(string path) + => GetChildStorageResourceContainer(path); } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceContainer.cs index 1503623a4d605..886f2b9449f03 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceContainer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; +using System.Threading.Tasks; namespace Azure.Storage.DataMovement { @@ -28,6 +29,21 @@ protected internal abstract IAsyncEnumerable GetStorageResource /// protected internal abstract StorageResourceItem GetStorageResourceReference(string path); + /// + /// Creates storage resource container if it does not already exists. + /// + /// + protected internal abstract Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default); + + /// + /// Gets the child StorageResourceContainer of the respective container. + /// + /// + /// The path of the child container. + /// + /// + protected internal abstract StorageResourceContainer GetChildStorageResourceContainer(string path); + /// /// Storage Resource is a container. /// diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs index 7b8872edefb82..9258cbe8a5677 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs @@ -171,29 +171,39 @@ private async IAsyncEnumerable GetStorageResourcesAsync() ? current.Uri.GetPath() : current.Uri.GetPath().Substring(containerUriPath.Length + 1); - if (!existingSources.Contains(sourceName)) + if (current.IsContainer) { - // Because AsyncEnumerable doesn't let us know which storage resource is the last resource - // we only yield return when we know this is not the last storage resource to be listed - // from the container. - StreamToUriJobPart part; - try - { - part = await StreamToUriJobPart.CreateJobPartAsync( - job: this, - partNumber: partNumber, - sourceResource: (StorageResourceItem)current, - destinationResource: _destinationResourceContainer.GetStorageResourceReference(sourceName)) - .ConfigureAwait(false); - AppendJobPart(part); - } - catch (Exception ex) + // Create sub-container + StorageResourceContainer subContainer = + _destinationResourceContainer.GetChildStorageResourceContainer(sourceName); + await subContainer.CreateIfNotExistsAsync().ConfigureAwait(false); + } + else + { + if (!existingSources.Contains(sourceName)) { - await InvokeFailedArgAsync(ex).ConfigureAwait(false); - yield break; + // Because AsyncEnumerable doesn't let us know which storage resource is the last resource + // we only yield return when we know this is not the last storage resource to be listed + // from the container. + StreamToUriJobPart part; + try + { + part = await StreamToUriJobPart.CreateJobPartAsync( + job: this, + partNumber: partNumber, + sourceResource: (StorageResourceItem)current, + destinationResource: _destinationResourceContainer.GetStorageResourceReference(sourceName)) + .ConfigureAwait(false); + AppendJobPart(part); + } + catch (Exception ex) + { + await InvokeFailedArgAsync(ex).ConfigureAwait(false); + yield break; + } + yield return part; + partNumber++; } - yield return part; - partNumber++; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/BlobStorageResourceContainerTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/BlobStorageResourceContainerTests.cs index 2f5206a3489fe..e3713589485de 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/BlobStorageResourceContainerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/BlobStorageResourceContainerTests.cs @@ -12,6 +12,7 @@ using Azure.Storage.Blobs.Tests; using Azure.Storage.DataMovement.Tests; using DMBlobs::Azure.Storage.DataMovement.Blobs; +using Moq; using NUnit.Framework; namespace Azure.Storage.DataMovement.Blobs.Tests @@ -114,5 +115,27 @@ public async Task GetChildStorageResourceAsync() Assert.IsNotNull(properties); Assert.IsNotNull(properties.ETag); } + + [Test] + public void GetChildStorageResourceContainer() + { + // Arrange + Uri uri = new Uri("https://storageaccount.blob.core.windows.net/container"); + Mock mock = new(uri, new BlobClientOptions()); + mock.Setup(b => b.Uri).Returns(uri); + + string prefix = "foo"; + StorageResourceContainer containerResource = + new BlobStorageResourceContainer(mock.Object, new() { BlobDirectoryPrefix = prefix }); + + // Act + string childPath = "bar"; + StorageResourceContainer childContainer = containerResource.GetChildStorageResourceContainer(childPath); + + // Assert + UriBuilder builder = new UriBuilder(containerResource.Uri); + builder.Path = string.Join("/", builder.Path, childPath); + Assert.AreEqual(builder.Uri, childContainer.Uri); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs index e0801b9795729..5e4a7a9618c1c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; using NUnit.Framework; @@ -134,5 +135,49 @@ public async Task GetChildStorageResourceAsync_SubDir() await resource.GetPropertiesAsync().ConfigureAwait(false); } } + + [Test] + public async Task CreateIfNotExistsAsync_NotExists() + { + using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); + string folderPath = test.DirectoryPath; + + string testPath = Path.Combine(folderPath, "testPath"); + + StorageResourceContainer container = new LocalDirectoryStorageResourceContainer(testPath); + await container.CreateIfNotExistsAsync(); + + Assert.IsTrue(Directory.Exists(testPath)); + } + + [Test] + public async Task CreateIfNotExistsAsync_Exists() + { + using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); + string folderPath = test.DirectoryPath; + + string testPath = Path.Combine(folderPath, "testPath"); + Directory.CreateDirectory(testPath); + + StorageResourceContainer container = new LocalDirectoryStorageResourceContainer(testPath); + await container.CreateIfNotExistsAsync(); + + Assert.IsTrue(Directory.Exists(testPath)); + } + + [Test] + public void GetChildStorageResourceContainer() + { + using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); + string folderPath = test.DirectoryPath; + + StorageResourceContainer container = new LocalDirectoryStorageResourceContainer(folderPath); + + string childPath = "childPath"; + StorageResourceContainer childContainer = container.GetChildStorageResourceContainer(childPath); + + string fullPath = Path.Combine(folderPath, childPath); + Assert.AreEqual(childContainer.Uri, new Uri(fullPath)); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs index 978e75fa39d77..9430964cc25dc 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceContainer.cs @@ -93,5 +93,15 @@ private IEnumerable GetStorageResources(bool includeContainers) } } } + + protected internal override Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + protected internal override StorageResourceContainer GetChildStorageResourceContainer(string path) + { + throw new NotImplementedException(); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index 07fecee7cd738..4a49dae88fcae 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -32,6 +31,7 @@ public abstract class StartTransferDirectoryCopyTestBase { private readonly string _generatedResourceNamePrefix; private readonly string _expectedOverwriteExceptionMessage; + private readonly string _firstItemName; public ClientBuilder SourceClientBuilder { get; protected set; } public ClientBuilder DestinationClientBuilder { get; protected set; } @@ -52,13 +52,14 @@ public StartTransferDirectoryCopyTestBase( Argument.CheckNotNullOrEmpty(expectedOverwriteExceptionMessage, nameof(expectedOverwriteExceptionMessage)); _generatedResourceNamePrefix = generatedResourceNamePrefix ?? "test-resource-"; _expectedOverwriteExceptionMessage = expectedOverwriteExceptionMessage; + _firstItemName = "item1"; } #region Service-Specific Methods /// /// Gets the service client using OAuth to authenticate. /// - protected abstract TSourceServiceClient GetOAuthSourceServiceClient(); + protected abstract TSourceContainerClient GetOAuthSourceContainerClient(string containerName); /// /// Gets a service-specific disposing container for use with tests in this class. @@ -78,6 +79,17 @@ protected abstract Task> GetSourceDi /// protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient directoryClient, string prefix); + /// + /// Creates the directory within the source container. Will also create any parent directories if required and is a hierarchical structure. + /// + /// + /// The respective source container to create the directory in. + /// + /// + /// The directory path. If parent paths are required, will also create any parent directories if required and is a hierarchical structure. + /// + protected abstract Task CreateDirectoryInSourceAsync(TSourceContainerClient sourceContainer, string directoryPath); + /// /// Creates the object in the source storage resource container. /// @@ -96,6 +108,11 @@ protected abstract Task> GetDes TDestinationServiceClient service = default, string containerName = default); + /// + /// Gets the service client using OAuth to authenticate. + /// + protected abstract TDestinationContainerClient GetOAuthDestinationContainerClient(string containerName); + /// /// Gets the specific storage resource from the given TDestinationObjectClient /// e.g. ShareFileClient to a ShareFileStorageResource, BlockBlobClient to a BlockBlobStorageResource. @@ -104,6 +121,17 @@ protected abstract Task> GetDes /// protected abstract StorageResourceContainer GetDestinationStorageResourceContainer(TDestinationContainerClient sourceContainerClient, string directoryPath); + /// + /// Creates the directory within the source container. Will also create any parent directories if required and is a hierarchical structure. + /// + /// + /// The respective source container to create the directory in. + /// + /// + /// The directory path. If parent paths are required, will also create any parent directories if required and is a hierarchical structure. + /// + protected abstract Task CreateDirectoryInDestinationAsync(TDestinationContainerClient destinationContainer, string directoryPath); + /// /// Creates the object in the source storage resource container. /// @@ -210,36 +238,33 @@ public async Task DirectoryToDirectory_SmallSize(long size, int waitTimeInSec) // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - string sourceDirectoryName = "sourceFolder"; - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceDirectoryName); - - List blobNames = new List(); + string sourcePrefix = "sourceFolder"; + string destinationPrefix = "destinationFolder"; - string itemName1 = Path.Combine(sourceDirectoryName, GetNewObjectName()); - string itemName2 = Path.Combine(sourceDirectoryName, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName1); await CreateObjectInSourceAsync(source.Container, size, itemName2); - blobNames.Add(itemName1); - blobNames.Add(itemName2); - string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string itemName3 = Path.Combine(sourceDirectoryName, subDirName, GetNewObjectName()); + string subDirName = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(source.Container, subDirName); + string itemName3 = string.Join("/", subDirName, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName3); - blobNames.Add(itemName3); - string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string itemName4 = Path.Combine(sourceDirectoryName, subDirName2, GetNewObjectName()); + string subDirName2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(source.Container, subDirName2); + string itemName4 = string.Join("/", subDirName2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName4); - blobNames.Add(itemName4); + + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); await CopyDirectoryAndVerifyAsync( source.Container, destination.Container, - sourceDirectoryName, - sourceFolderPath, + sourcePrefix, + destinationPrefix, + 4, waitTimeInSec).ConfigureAwait(false); } @@ -255,34 +280,32 @@ public async Task DirectoryToDirectory_LargeSize(long size, int waitTimeInSec) // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - string sourceBlobDirectoryName = "sourceFolder"; - - List blobNames = new List(); + string sourcePrefix = "sourceFolder"; - string itemName1 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); - string itemName2 = Path.Combine(sourceBlobDirectoryName, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName1); await CreateObjectInSourceAsync(source.Container, size, itemName2); - blobNames.Add(itemName1); - blobNames.Add(itemName2); - string subDirName = "bar"; - string itemName3 = Path.Combine(sourceBlobDirectoryName, subDirName, GetNewObjectName()); + string subDirName = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(source.Container, subDirName); + string itemName3 = string.Join("/", subDirName, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName3); - blobNames.Add(itemName3); - string subDirName2 = "pik"; - string itemName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, GetNewObjectName()); + string subDirName2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(source.Container, subDirName2); + string itemName4 = string.Join("/", subDirName2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName4); - blobNames.Add(itemName4); - string destinationFolder = "destFolder"; + string destinationPrefix = "destFolder"; + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); await CopyDirectoryAndVerifyAsync( source.Container, destination.Container, - sourceBlobDirectoryName, - destinationFolder, + sourcePrefix, + destinationPrefix, waitTimeInSec).ConfigureAwait(false); } @@ -293,14 +316,14 @@ public async Task DirectoryToDirectory_EmptyFolder() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + var sourcePath = GetNewObjectName(); + var destinationPath = GetNewObjectName(); - // Set up directory to upload - var destinationName = GetNewObjectName(); - var sourceName = GetNewObjectName(); - - // Set up destination client - StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destination.Container, destinationName); - StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(source.Container, sourceName); + // Set up resources + await CreateDirectoryInSourceAsync(source.Container, sourcePath); + StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(source.Container, sourcePath); + await CreateDirectoryInDestinationAsync(destination.Container, destinationPath); + StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destination.Container, destinationPath); TransferManagerOptions managerOptions = new TransferManagerOptions() { @@ -314,13 +337,13 @@ public async Task DirectoryToDirectory_EmptyFolder() // Act DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource, options); - CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10)); await transfer.WaitForCompletionAsync(tokenSource.Token); Assert.IsTrue(transfer.HasCompleted); Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); // Assert - await VerifyEmptyDestinationContainerAsync(destination.Container, destinationName); + await VerifyEmptyDestinationContainerAsync(destination.Container, destinationPath); testEventsRaised.AssertUnexpectedFailureCheck(); } @@ -331,16 +354,16 @@ public async Task DirectoryToDirectory_SingleFile() // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); string sourcePrefix = "sourceFolder"; - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourcePrefix); - string itemName1 = Path.Combine(sourcePrefix, GetNewObjectName()); + string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName1); string destinationPrefix = "destFolder"; + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); await CopyDirectoryAndVerifyAsync( sourceContainer: source.Container, destinationContainer: destination.Container, @@ -359,18 +382,23 @@ public async Task DirectoryToDirectory_ManySubDirectories() string sourcePrefix = "sourceFolder"; + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); string subDir1 = string.Join("/", sourcePrefix, "foo"); - string itemName1 = Path.Combine(sourcePrefix, subDir1, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, subDir1); + string itemName1 = string.Join("/", subDir1, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName1); string subDir2 = string.Join("/", sourcePrefix, "rul"); - string itemName2 = Path.Combine(sourcePrefix, subDir2, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, subDir2); + string itemName2 = string.Join("/", subDir2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName2); string subDir3 = string.Join("/", sourcePrefix, "pik"); - string itemName3 = Path.Combine(sourcePrefix, subDir3, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, subDir3); + string itemName3 = string.Join("/", subDir3, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, Constants.KB, itemName3); string destinationPrefix = "destFolder"; + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); await CopyDirectoryAndVerifyAsync( sourceContainer: source.Container, destinationContainer: destination.Container, @@ -392,25 +420,31 @@ public async Task DirectoryToDirectory_SubDirectoriesLevels(int level) string sourcePrefix = "sourceFolder"; + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); + + string subDirPrefix = sourcePrefix; for (int i = 0; i < level; i++) { - string subDirPrefix = $"{sourcePrefix}/folder{i}/{GetNewObjectName()}"; - await CreateObjectInSourceAsync(source.Container, Constants.KB, subDirPrefix); + subDirPrefix = string.Join("/", subDirPrefix, $"folder{i}"); + await CreateDirectoryInSourceAsync(source.Container, subDirPrefix); + string fullFilePath = string.Join("/", subDirPrefix, GetNewObjectName()); + await CreateObjectInSourceAsync(source.Container, Constants.KB, fullFilePath); } string destinationPrefix = "destFolder"; + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); await CopyDirectoryAndVerifyAsync( source.Container, destination.Container, sourcePrefix, - destinationPrefix: destinationPrefix, + destinationPrefix, itemTransferCount: level).ConfigureAwait(false); } [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task DirectoryToDirectory_OverwriteTrue() + public async Task DirectoryToDirectory_OverwriteExists() { // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); @@ -418,18 +452,29 @@ public async Task DirectoryToDirectory_OverwriteTrue() long size = Constants.KB; string sourcePrefix = "sourceFolder"; + string destinationPrefix = "destPrefix"; - string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); - string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); + + string itemName1 = string.Join("/", sourcePrefix, _firstItemName); await CreateObjectInSourceAsync(source.Container, size, itemName1); + + // Create same object in the destination, so when both files are seen overwrite will trigger. + string destItemName1 = string.Join("/", destinationPrefix, _firstItemName); + await CreateObjectInDestinationAsync(destination.Container, size, destItemName1); + + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName2); - string subDirName = "bar"; - string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + string subDirName = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(source.Container, subDirName); + string itemName3 = string.Join("/", subDirName, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName3); - string subDirName2 = "pik"; - string itemName4 = string.Join("/", sourcePrefix, subDirName2, GetNewObjectName()); + string subDirName2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(source.Container, subDirName2); + string itemName4 = string.Join("/", subDirName2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName4); DataTransferOptions options = new DataTransferOptions() @@ -437,8 +482,6 @@ public async Task DirectoryToDirectory_OverwriteTrue() CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - string destinationPrefix = "destPrefix"; - // Act await CopyDirectoryAndVerifyAsync( source.Container, @@ -451,7 +494,7 @@ await CopyDirectoryAndVerifyAsync( [Test] [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 - public async Task DirectoryToDirectory_OverwriteFalse() + public async Task DirectoryToDirectory_OverwriteNotExists() { // Arrange await using IDisposingContainer source = await GetSourceDisposingContainerAsync(); @@ -459,32 +502,32 @@ public async Task DirectoryToDirectory_OverwriteFalse() long size = Constants.KB; string sourcePrefix = "sourceFolder"; + string destinationPrefix = "destPrefix"; + + await CreateDirectoryInSourceAsync(source.Container, sourcePrefix); + await CreateDirectoryInDestinationAsync(destination.Container, destinationPrefix); - List blobNames = new List(); string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); - string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName1); + + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName2); - blobNames.Add(itemName1); - blobNames.Add(itemName2); - string subDirName = "bar"; - string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + string subDirName = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(source.Container, subDirName); + string itemName3 = string.Join("/", subDirName, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName3); - blobNames.Add(itemName3); - string subDirName2 = "pik"; - string itemName4 = string.Join("/", sourcePrefix, subDirName2, GetNewObjectName()); + string subDirName2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(source.Container, subDirName2); + string itemName4 = string.Join("/", subDirName2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName4); - blobNames.Add(itemName4); DataTransferOptions options = new DataTransferOptions() { CreationPreference = StorageResourceCreationPreference.OverwriteIfExists }; - string destinationPrefix = "destFolder"; - // Act await CopyDirectoryAndVerifyAsync( source.Container, @@ -501,63 +544,64 @@ public async Task DirectoryToDirectory_OAuth() { // Arrange long size = Constants.KB; - int waitTimeInSec = 10; - TSourceServiceClient service = GetOAuthSourceServiceClient(); - await using IDisposingContainer source = await GetSourceDisposingContainerAsync(service); + int waitTimeInSec = 20; + string sourceContainerName = GetNewObjectName(); + await using IDisposingContainer source = await GetSourceDisposingContainerAsync(containerName: sourceContainerName); + TSourceContainerClient oauthSourceContainer = GetOAuthSourceContainerClient(containerName: sourceContainerName); + await using IDisposingContainer destination = await GetDestinationDisposingContainerAsync(); + TDestinationContainerClient oauthDestinationContainer = GetOAuthDestinationContainerClient(containerName: sourceContainerName); string sourcePrefix = "sourceFolder"; - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string destinationPrefix = "destFolder"; - List blobNames = new List(); + await CreateDirectoryInSourceAsync(oauthSourceContainer, sourcePrefix); + await CreateDirectoryInDestinationAsync(oauthDestinationContainer, destinationPrefix); string itemName1 = string.Join("/", sourcePrefix, GetNewObjectName()); - string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName1); + + string itemName2 = string.Join("/", sourcePrefix, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName2); - blobNames.Add(itemName1); - blobNames.Add(itemName2); - string subDirName = "bar"; - string itemName3 = string.Join("/", sourcePrefix, subDirName, GetNewObjectName()); + string subDirName = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(source.Container, subDirName); + string itemName3 = string.Join("/", subDirName, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName3); - blobNames.Add(itemName3); - string subDirName2 = "pik"; - string itemName4 = Path.Combine(sourcePrefix, subDirName2, GetNewObjectName()); + string subDirName2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(source.Container, subDirName2); + string itemName4 = string.Join("/", subDirName2, GetNewObjectName()); await CreateObjectInSourceAsync(source.Container, size, itemName4); - blobNames.Add(itemName4); - - string destinationPrefix = "destFolder"; await CopyDirectoryAndVerifyAsync( - source.Container, - destination.Container, + oauthSourceContainer, + oauthDestinationContainer, sourcePrefix, destinationPrefix, + 4, waitTimeInSec).ConfigureAwait(false); } #region Single Concurrency private async Task CreateDirectoryTree( TSourceContainerClient client, - string sourceFolderPath, - string sourceBlobDirectoryName, + string sourcePrefix, int size) { - string itemName1 = Path.Combine(sourceBlobDirectoryName, "blob1"); - string itemName2 = Path.Combine(sourceBlobDirectoryName, "blob2"); + string itemName1 = string.Join("/", sourcePrefix, _firstItemName); + string itemName2 = string.Join("/", sourcePrefix, "item2"); await CreateObjectInSourceAsync(client, size, itemName1); await CreateObjectInSourceAsync(client, size, itemName2); - string subDirName = "bar"; - CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); - string itemName3 = Path.Combine(sourceBlobDirectoryName, subDirName, "blob3"); + string subDirPath = string.Join("/", sourcePrefix, "bar"); + await CreateDirectoryInSourceAsync(client, subDirPath); + string itemName3 = string.Join("/", subDirPath, "item3"); await CreateObjectInSourceAsync(client, size, itemName3); - string subDirName2 = "pik"; - CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); - string itemName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, "blob4"); + string subDirPath2 = string.Join("/", sourcePrefix, "pik"); + await CreateDirectoryInSourceAsync(client, subDirPath2); + string itemName4 = string.Join("/", subDirPath2, "item4"); await CreateObjectInSourceAsync(client, size, itemName4); } @@ -569,23 +613,23 @@ private async Task CreateStartTransfer( DataTransferOptions options = default, int size = Constants.KB) { - using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); // Arrange - // Create source local file for checking, and source blob - string sourceBlobPrefix = "sourceFolder"; - string destBlobPrefix = "destFolder"; - string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobPrefix); - await CreateDirectoryTree(sourceContainer, sourceFolderPath, sourceBlobPrefix, size); + string sourcePrefix = "sourceFolder"; + string destPrefix = "destFolder"; + await CreateDirectoryInSourceAsync(sourceContainer, sourcePrefix); + await CreateDirectoryInDestinationAsync(destinationContainer, destPrefix); + await CreateDirectoryTree(sourceContainer, sourcePrefix, size); - // Create new source block blob. - StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(sourceContainer, sourceBlobPrefix); - StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destinationContainer, destBlobPrefix); + // Create storage resource containers + StorageResourceContainer sourceResource = GetSourceStorageResourceContainer(sourceContainer, sourcePrefix); + StorageResourceContainer destinationResource = GetDestinationStorageResourceContainer(destinationContainer, destPrefix); - // If we want a failure condition to happen if (createFailedCondition) { - string destBlobName = $"{destBlobPrefix}/blob1"; - await CreateObjectInDestinationAsync(destinationContainer, size, destBlobName); + // To create an expected failure, create an item that is supposed to be transferred over. + // If we don't enable overwrite, a failure should be thrown. + string fullDestPath = string.Join("/", destPrefix, _firstItemName); + await CreateObjectInDestinationAsync(destinationContainer, size, fullDestPath); } // Create Transfer Manager with single threaded operation @@ -603,7 +647,7 @@ private async Task CreateStartTransfer( } [Test] - [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + //[LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 public async Task StartTransfer_AwaitCompletion() { // Arrange @@ -662,7 +706,7 @@ public async Task StartTransfer_AwaitCompletion_Failed() Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); - Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); } [Test] @@ -761,7 +805,7 @@ public async Task StartTransfer_EnsureCompleted_Failed() Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); - Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); } [Test] @@ -833,7 +877,7 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() Assert.IsTrue(transfer.HasCompleted); Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); Assert.AreEqual(true, transfer.TransferStatus.HasFailedItems); - Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains("BlobAlreadyExists")); + Assert.IsTrue(testEventsRaised.FailedEvents.First().Exception.Message.Contains(_expectedOverwriteExceptionMessage)); await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } #endregion diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index 545c30db3ebfc..42de338c1a86b 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -523,7 +523,7 @@ protected static async Task GetCopyAuthorizationHeaderAsync( if (client.ClientConfiguration.TokenCredential != default) { AccessToken accessToken = await client.ClientConfiguration.TokenCredential.GetTokenAsync( - new TokenRequestContext(new string[] { client.ClientConfiguration.Audience.ToString() }), + new TokenRequestContext(new string[] { client.ClientConfiguration.Audience.CreateDefaultScope() }), cancellationToken).ConfigureAwait(false); return new HttpAuthorization( Constants.CopyHttpAuthorization.BearerScheme, diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj index d501967eb6e11..398a4b6367489 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj @@ -17,56 +17,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - PreserveNewest From b159059de6bd32040f4e183d21f46fa474b435d4 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 26 Oct 2023 09:45:31 -0700 Subject: [PATCH 06/11] Export API --- .../tests/ClientBuilderExtensions.cs | 2 -- .../api/Azure.Storage.DataMovement.net6.0.cs | 2 ++ .../api/Azure.Storage.DataMovement.netstandard2.0.cs | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs index 5270b092cfd15..23267959a0fa5 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs @@ -10,8 +10,6 @@ using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder< Azure.Storage.Files.Shares.ShareServiceClient, Azure.Storage.Files.Shares.ShareClientOptions>; -using Azure.Storage.Files.Shares.Models; -using Azure.Core.TestFramework; namespace Azure.Storage.DataMovement.Files.Shares.Tests { diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index 4140218d0bdb5..58bdd57ce296a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -129,6 +129,8 @@ public abstract partial class StorageResourceContainer : Azure.Storage.DataMovem { protected StorageResourceContainer() { } protected internal override bool IsContainer { get { throw null; } } + protected internal abstract System.Threading.Tasks.Task CreateIfNotExistsAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + protected internal abstract Azure.Storage.DataMovement.StorageResourceContainer GetChildStorageResourceContainer(string path); protected internal abstract Azure.Storage.DataMovement.StorageResourceItem GetStorageResourceReference(string path); protected internal abstract System.Collections.Generic.IAsyncEnumerable GetStorageResourcesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); } diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index 4140218d0bdb5..58bdd57ce296a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -129,6 +129,8 @@ public abstract partial class StorageResourceContainer : Azure.Storage.DataMovem { protected StorageResourceContainer() { } protected internal override bool IsContainer { get { throw null; } } + protected internal abstract System.Threading.Tasks.Task CreateIfNotExistsAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + protected internal abstract Azure.Storage.DataMovement.StorageResourceContainer GetChildStorageResourceContainer(string path); protected internal abstract Azure.Storage.DataMovement.StorageResourceItem GetStorageResourceReference(string path); protected internal abstract System.Collections.Generic.IAsyncEnumerable GetStorageResourcesAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); } From 0ed3705ec0c0b3730d884ce0ef73d289e3332f2a Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Thu, 26 Oct 2023 10:30:26 -0700 Subject: [PATCH 07/11] Cleanup --- .../tests/Shared/StartTransferDirectoryCopyTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index 4a49dae88fcae..6d13f7f99756d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -647,7 +647,7 @@ private async Task CreateStartTransfer( } [Test] - //[LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 + [LiveOnly] // https://github.com/Azure/azure-sdk-for-net/issues/33082 public async Task StartTransfer_AwaitCompletion() { // Arrange From a8fca4e88fa4c58a4b6f4a229b33b96fd33d444b Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Mon, 30 Oct 2023 14:45:48 -0700 Subject: [PATCH 08/11] Cleanup and removing unnecessary comments --- .../tests/ClientBuilderExtensions.cs | 6 +- .../ShareDirectoryStartTransferCopyTests.cs | 59 ++++++++++++------- .../src/ServiceToServiceTransferJob.cs | 3 - .../src/StreamToUriTransferJob.cs | 3 - .../StartTransferDirectoryCopyTestBase.cs | 50 +++++++++++----- 5 files changed, 79 insertions(+), 42 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs index 23267959a0fa5..0bd6ecb2ee86d 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ClientBuilderExtensions.cs @@ -10,6 +10,8 @@ using SharesClientBuilder = Azure.Storage.Test.Shared.ClientBuilder< Azure.Storage.Files.Shares.ShareServiceClient, Azure.Storage.Files.Shares.ShareClientOptions>; +using System.Threading; +using Azure.Core; namespace Azure.Storage.DataMovement.Files.Shares.Tests { @@ -47,8 +49,10 @@ public static async Task GetTestShareAsync( ShareServiceClient service = default, string shareName = default, IDictionary metadata = default, - ShareClientOptions options = default) + ShareClientOptions options = default, + CancellationToken cancellationToken = default) { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); service ??= clientBuilder.GetServiceClientFromSharedKeyConfig(clientBuilder.Tenants.TestConfigDefault, options); metadata ??= new Dictionary(StringComparer.OrdinalIgnoreCase); shareName ??= clientBuilder.GetNewShareName(); diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index 41933b19e8ff3..886e7237abd3a 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -17,6 +17,8 @@ using NUnit.Framework; using System.Security.AccessControl; using Microsoft.Extensions.Options; +using System.Threading; +using Azure.Core; namespace Azure.Storage.DataMovement.Files.Shares.Tests { @@ -44,18 +46,23 @@ protected override async Task CreateObjectInSourceAsync( ShareClient container, long? objectLength = null, string objectName = null, - Stream contents = default) - => await CreateShareFileAsync(container, objectLength, objectName, contents); + Stream contents = default, + CancellationToken cancellationToken = default) + => await CreateShareFileAsync(container, objectLength, objectName, contents, cancellationToken); protected override async Task CreateObjectInDestinationAsync( ShareClient container, long? objectLength = null, string objectName = null, - Stream contents = null) - => await CreateShareFileAsync(container, objectLength, objectName, contents); + Stream contents = null, + CancellationToken cancellationToken = default) + => await CreateShareFileAsync(container, objectLength, objectName, contents, cancellationToken); - protected override async Task> GetDestinationDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) - => await DestinationClientBuilder.GetTestShareAsync(service, containerName); + protected override async Task> GetDestinationDisposingContainerAsync( + ShareServiceClient service = null, + string containerName = null, + CancellationToken cancellationToken = default) + => await DestinationClientBuilder.GetTestShareAsync(service, containerName, cancellationToken: cancellationToken); protected override StorageResourceContainer GetDestinationStorageResourceContainer(ShareClient containerClient, string prefix) => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); @@ -76,7 +83,7 @@ protected override ShareClient GetOAuthDestinationContainerClient(string contain return oauthService.GetShareClient(containerName); } - protected override async Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null) + protected override async Task> GetSourceDisposingContainerAsync(ShareServiceClient service = null, string containerName = null, CancellationToken cancellationToken = default) { service ??= SourceClientBuilder.GetServiceClientFromSharedKeyConfig(SourceClientBuilder.Tenants.TestConfigDefault, SourceClientBuilder.GetOptions()); ShareServiceClient sasService = new ShareServiceClient(service.GenerateAccountSasUri( @@ -84,24 +91,28 @@ protected override async Task> GetSourceDisposi SourceClientBuilder.Recording.UtcNow.AddDays(1), Sas.AccountSasResourceTypes.All), SourceClientBuilder.GetOptions()); - return await SourceClientBuilder.GetTestShareAsync(sasService, containerName); + return await SourceClientBuilder.GetTestShareAsync(sasService, containerName, cancellationToken: cancellationToken); } protected override StorageResourceContainer GetSourceStorageResourceContainer(ShareClient containerClient, string prefix = null) => new ShareDirectoryStorageResourceContainer(containerClient.GetDirectoryClient(prefix), default); - protected override async Task CreateDirectoryInSourceAsync(ShareClient sourceContainer, string directoryPath) - => await CreateDirectoryTree(sourceContainer, directoryPath); + protected override async Task CreateDirectoryInSourceAsync(ShareClient sourceContainer, string directoryPath, CancellationToken cancellationToken = default) + => await CreateDirectoryTreeAsync(sourceContainer, directoryPath, cancellationToken); - protected override async Task CreateDirectoryInDestinationAsync(ShareClient destinationContainer, string directoryPath) - => await CreateDirectoryTree(destinationContainer, directoryPath); + protected override async Task CreateDirectoryInDestinationAsync(ShareClient destinationContainer, string directoryPath, CancellationToken cancellationToken = default) + => await CreateDirectoryTreeAsync(destinationContainer, directoryPath, cancellationToken); - protected override async Task VerifyEmptyDestinationContainerAsync(ShareClient destinationContainer, string destinationPrefix) + protected override async Task VerifyEmptyDestinationContainerAsync( + ShareClient destinationContainer, + string destinationPrefix, + CancellationToken cancellationToken = default) { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); ShareDirectoryClient destinationDirectory = string.IsNullOrEmpty(destinationPrefix) ? destinationContainer.GetRootDirectoryClient() : destinationContainer.GetDirectoryClient(destinationPrefix); - IList items = await destinationDirectory.GetFilesAndDirectoriesAsync().ToListAsync(); + IList items = await destinationDirectory.GetFilesAndDirectoriesAsync(cancellationToken: cancellationToken).ToListAsync(); Assert.IsEmpty(items); } @@ -109,8 +120,11 @@ protected override async Task VerifyResultsAsync( ShareClient sourceContainer, string sourcePrefix, ShareClient destinationContainer, - string destinationPrefix) + string destinationPrefix, + CancellationToken cancellationToken = default) { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); + // List all files in source blob folder path List sourceFileNames = new List(); List sourceDirectoryNames = new List(); @@ -152,8 +166,8 @@ protected override async Task VerifyResultsAsync( // Verify Download string sourceFileName = Path.Combine(sourcePrefix, sourceFileNames[i]); - using Stream sourceStream = await sourceDirectory.GetFileClient(sourceFileNames[i]).OpenReadAsync(); - using Stream destinationStream = await destinationDirectory.GetFileClient(destinationFileNames[i]).OpenReadAsync(); + using Stream sourceStream = await sourceDirectory.GetFileClient(sourceFileNames[i]).OpenReadAsync(cancellationToken: cancellationToken); + using Stream destinationStream = await destinationDirectory.GetFileClient(destinationFileNames[i]).OpenReadAsync(cancellationToken: cancellationToken); Assert.AreEqual(sourceStream, destinationStream); } } @@ -162,8 +176,10 @@ private async Task CreateShareFileAsync( ShareClient container, long? objectLength = null, string objectName = null, - Stream contents = default) + Stream contents = default, + CancellationToken cancellationToken = default) { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); objectName ??= GetNewObjectName(); if (!objectLength.HasValue) { @@ -174,15 +190,16 @@ private async Task CreateShareFileAsync( if (contents != default) { - await fileClient.UploadAsync(contents); + await fileClient.UploadAsync(contents, cancellationToken: cancellationToken); } } - private async Task CreateDirectoryTree(ShareClient container, string directoryPath) + private async Task CreateDirectoryTreeAsync(ShareClient container, string directoryPath, CancellationToken cancellationToken = default) { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); // Parse for parent directory names and create the parent directory(s). ShareDirectoryClient directory = container.GetRootDirectoryClient().GetSubdirectoryClient(directoryPath); - await directory.CreateIfNotExistsAsync(); + await directory.CreateIfNotExistsAsync(cancellationToken: cancellationToken); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs index e2953db5ca317..733a1179880e4 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceTransferJob.cs @@ -192,9 +192,6 @@ private async IAsyncEnumerable GetStorageResourcesAsync() ? current.Uri.GetPath() : current.Uri.GetPath().Substring(containerUriPath.Length + 1); - // Because AsyncEnumerable doesn't let us know which storage resource is the last resource - // we only yield return when we know this is not the last storage resource to be listed - // from the container. ServiceToServiceJobPart part; try { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs index b486827d69d84..2ca04b94c144b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriTransferJob.cs @@ -190,9 +190,6 @@ private async IAsyncEnumerable GetStorageResourcesAsync() ? current.Uri.GetPath() : current.Uri.GetPath().Substring(containerUriPath.Length + 1); - // Because AsyncEnumerable doesn't let us know which storage resource is the last resource - // we only yield return when we know this is not the last storage resource to be listed - // from the container. StreamToUriJobPart part; try { diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index 6d13f7f99756d..aea718f2cbe4d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -68,7 +68,8 @@ public StartTransferDirectoryCopyTestBase( /// Optional container name specification. protected abstract Task> GetSourceDisposingContainerAsync( TSourceServiceClient service = default, - string containerName = default); + string containerName = default, + CancellationToken cancellationToken = default); /// /// Gets the specific storage resource from the given TDestinationObjectClient @@ -77,7 +78,7 @@ protected abstract Task> GetSourceDi /// The object client to create the storage resource object. /// The path of the directory. /// - protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient directoryClient, string prefix); + protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string directoryPath); /// /// Creates the directory within the source container. Will also create any parent directories if required and is a hierarchical structure. @@ -88,7 +89,10 @@ protected abstract Task> GetSourceDi /// /// The directory path. If parent paths are required, will also create any parent directories if required and is a hierarchical structure. /// - protected abstract Task CreateDirectoryInSourceAsync(TSourceContainerClient sourceContainer, string directoryPath); + protected abstract Task CreateDirectoryInSourceAsync( + TSourceContainerClient sourceContainer, + string directoryPath, + CancellationToken cancellationToken = default); /// /// Creates the object in the source storage resource container. @@ -97,7 +101,12 @@ protected abstract Task> GetSourceDi /// The name of the object to create. /// The contents to set in the object. /// - protected abstract Task CreateObjectInSourceAsync(TSourceContainerClient container, long? objectLength = null, string objectName = null, Stream contents = default); + protected abstract Task CreateObjectInSourceAsync( + TSourceContainerClient container, + long? objectLength = null, + string objectName = null, + Stream contents = default, + CancellationToken cancellationToken = default); /// /// Gets a service-specific disposing container for use with tests in this class. @@ -106,7 +115,8 @@ protected abstract Task> GetSourceDi /// Optional container name specification. protected abstract Task> GetDestinationDisposingContainerAsync( TDestinationServiceClient service = default, - string containerName = default); + string containerName = default, + CancellationToken cancellationToken = default); /// /// Gets the service client using OAuth to authenticate. @@ -130,7 +140,10 @@ protected abstract Task> GetDes /// /// The directory path. If parent paths are required, will also create any parent directories if required and is a hierarchical structure. /// - protected abstract Task CreateDirectoryInDestinationAsync(TDestinationContainerClient destinationContainer, string directoryPath); + protected abstract Task CreateDirectoryInDestinationAsync( + TDestinationContainerClient destinationContainer, + string directoryPath, + CancellationToken cancellationToken = default); /// /// Creates the object in the source storage resource container. @@ -139,7 +152,12 @@ protected abstract Task> GetDes /// The name of the object to create. /// The contents to set in the object. /// - protected abstract Task CreateObjectInDestinationAsync(TDestinationContainerClient container, long? objectLength = null, string objectName = null, Stream contents = default); + protected abstract Task CreateObjectInDestinationAsync( + TDestinationContainerClient container, + long? objectLength = null, + string objectName = null, + Stream contents = default, + CancellationToken cancellationToken = default); /// /// Verifies that the destination container is empty when we expect it to be. @@ -148,7 +166,10 @@ protected abstract Task> GetDes /// The respective destination container to verify empty contents. /// /// - protected abstract Task VerifyEmptyDestinationContainerAsync(TDestinationContainerClient destinationContainer, string destinationPrefix); + protected abstract Task VerifyEmptyDestinationContainerAsync( + TDestinationContainerClient destinationContainer, + string destinationPrefix, + CancellationToken cancellationToken = default); /// /// Verifies the results between the source and the destination container. @@ -162,21 +183,22 @@ protected abstract Task VerifyResultsAsync( TSourceContainerClient sourceContainer, string sourcePrefix, TDestinationContainerClient destinationContainer, - string destinationPrefix); + string destinationPrefix, + CancellationToken cancellationToken = default); #endregion protected string GetNewObjectName() => _generatedResourceNamePrefix + SourceClientBuilder.Recording.Random.NewGuid(); /// - /// Upload and verify the contents of the blob + /// Upload and verify the contents of the items /// /// By default in this function an event argument will be added to the options event handler /// to detect when the upload has finished. /// - /// The source container which will contains the source blobs - /// The source blob prefix/folder - /// The destination local path to download the blobs to + /// The source container which will contains the source items + /// The source prefix/folder + /// The destination local path to download the items to /// /// How long we should wait until we cancel the operation. If this timeout is reached the test will fail. /// @@ -220,7 +242,7 @@ private async Task CopyDirectoryAndVerifyAsync( Assert.IsTrue(transfer.HasCompleted); Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); - // List all files in source blob folder path + // List all files in source folder path await VerifyResultsAsync( sourceContainer: sourceContainer, sourcePrefix: sourcePrefix, From b147c585e677494661dc8b0112bec9bf0d414bf7 Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Tue, 31 Oct 2023 13:08:10 -0700 Subject: [PATCH 09/11] Cleanup --- .../tests/ShareDirectoryStartTransferCopyTests.cs | 1 - .../tests/Shared/StartTransferDirectoryCopyTestBase.cs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs index 886e7237abd3a..794681a64f2c3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDirectoryStartTransferCopyTests.cs @@ -197,7 +197,6 @@ private async Task CreateShareFileAsync( private async Task CreateDirectoryTreeAsync(ShareClient container, string directoryPath, CancellationToken cancellationToken = default) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - // Parse for parent directory names and create the parent directory(s). ShareDirectoryClient directory = container.GetRootDirectoryClient().GetSubdirectoryClient(directoryPath); await directory.CreateIfNotExistsAsync(cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs index aea718f2cbe4d..2f834ebafee8d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferDirectoryCopyTestBase.cs @@ -81,13 +81,13 @@ protected abstract Task> GetSourceDi protected abstract StorageResourceContainer GetSourceStorageResourceContainer(TSourceContainerClient containerClient, string directoryPath); /// - /// Creates the directory within the source container. Will also create any parent directories if required and is a hierarchical structure. + /// Creates the directory within the source container. /// /// /// The respective source container to create the directory in. /// /// - /// The directory path. If parent paths are required, will also create any parent directories if required and is a hierarchical structure. + /// The directory path. /// protected abstract Task CreateDirectoryInSourceAsync( TSourceContainerClient sourceContainer, From f16ef32b46255893a285fdc3ccd45d81c5a6345a Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Tue, 31 Oct 2023 13:29:36 -0700 Subject: [PATCH 10/11] Changed LocalDirectory Create to no op --- .../src/LocalDirectoryStorageResourceContainer.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs index 67a00944d4504..28c67ffee4a74 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalDirectoryStorageResourceContainer.cs @@ -100,11 +100,7 @@ protected internal override StorageResourceCheckpointData GetDestinationCheckpoi } protected internal override Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default) - { - CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - Directory.CreateDirectory(_uri.LocalPath); - return Task.CompletedTask; - } + => Task.CompletedTask; protected internal override StorageResourceContainer GetChildStorageResourceContainer(string path) { From 6db00b8a44339a2ae6bd9cb321e5069f1d49782b Mon Sep 17 00:00:00 2001 From: Amanda Nguyen Date: Tue, 31 Oct 2023 18:08:02 -0700 Subject: [PATCH 11/11] Remove unnecessary tests --- .../LocalDirectoryStorageResourceTests.cs | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs index 5e4a7a9618c1c..b136b68175cb8 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/LocalDirectoryStorageResourceTests.cs @@ -136,35 +136,6 @@ public async Task GetChildStorageResourceAsync_SubDir() } } - [Test] - public async Task CreateIfNotExistsAsync_NotExists() - { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); - string folderPath = test.DirectoryPath; - - string testPath = Path.Combine(folderPath, "testPath"); - - StorageResourceContainer container = new LocalDirectoryStorageResourceContainer(testPath); - await container.CreateIfNotExistsAsync(); - - Assert.IsTrue(Directory.Exists(testPath)); - } - - [Test] - public async Task CreateIfNotExistsAsync_Exists() - { - using DisposingLocalDirectory test = DisposingLocalDirectory.GetTestDirectory(); - string folderPath = test.DirectoryPath; - - string testPath = Path.Combine(folderPath, "testPath"); - Directory.CreateDirectory(testPath); - - StorageResourceContainer container = new LocalDirectoryStorageResourceContainer(testPath); - await container.CreateIfNotExistsAsync(); - - Assert.IsTrue(Directory.Exists(testPath)); - } - [Test] public void GetChildStorageResourceContainer() {