From 6e80e344de789a6f01ec3457c7678b7cc7510634 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon <96087589+jalauzon-msft@users.noreply.github.com> Date: Fri, 24 Mar 2023 11:11:25 -0700 Subject: [PATCH] [Storage] [DataMovement] Add BlobType option to `BlobStorageResourceContainerOptions` (#35081) --- .../CHANGELOG.md | 1 + ...orage.DataMovement.Blobs.netstandard2.0.cs | 1 + .../BlobDirectoryStorageResourceContainer.cs | 25 ++++++--- .../src/BlobStorageResourceContainer.cs | 19 ++++++- .../BlobStorageResourceContainerOptions.cs | 10 ++-- .../StartTransferUploadDirectoryTests.cs | 52 +++++++++++++++++++ 6 files changed, 95 insertions(+), 13 deletions(-) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md index f43ecea2616f8..eda728dc2b56f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md @@ -2,6 +2,7 @@ ## 12.0.0-beta.2 (Unreleased) - This release contains bug fixes to improve quality. +- Added option to `BlobStorageResourceContainerOptions` to choose `BlobType` when uploading blobs. ## 12.0.0-beta.1 (2022-12-15) - This preview is the first release of a ground-up rewrite of our client data movement diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs index f4d77cdd1a504..1cd330055faa6 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs @@ -58,6 +58,7 @@ public BlobStorageResourceContainer(Azure.Storage.Blobs.BlobContainerClient blob public partial class BlobStorageResourceContainerOptions { public BlobStorageResourceContainerOptions() { } + public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } set { } } public Azure.Storage.DataMovement.Models.TransferCopyMethod CopyMethod { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } } public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs index ac10da40a38e6..016d65ca99752 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs @@ -3,18 +3,13 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Runtime.CompilerServices; -using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; -using Azure.Storage.DataMovement; -using Azure.Storage.DataMovement.Models; namespace Azure.Storage.DataMovement.Blobs { @@ -85,9 +80,23 @@ public BlobDirectoryStorageResourceContainer( public override StorageResource GetChildStorageResource(string path) { // Recreate the blobName using the existing parent directory path - return new BlockBlobStorageResource( - _blobContainerClient.GetBlockBlobClient(System.IO.Path.Combine(_directoryPrefix, path)), - _options.ToBlockBlobStorageResourceOptions()); + switch (_options.BlobType) + { + case BlobType.Block: + return new BlockBlobStorageResource( + _blobContainerClient.GetBlockBlobClient(System.IO.Path.Combine(_directoryPrefix, path)), + _options.ToBlockBlobStorageResourceOptions()); + case BlobType.Append: + return new AppendBlobStorageResource( + _blobContainerClient.GetAppendBlobClient(System.IO.Path.Combine(_directoryPrefix, path)), + _options.ToAppendBlobStorageResourceOptions()); + case BlobType.Page: + return new PageBlobStorageResource( + _blobContainerClient.GetPageBlobClient(System.IO.Path.Combine(_directoryPrefix, path)), + _options.ToPageBlobStorageResourceOptions()); + default: + throw new ArgumentException("Invalid BlobType."); + } } /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs index fc08f7284c620..436b90937b367 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs @@ -9,7 +9,6 @@ using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; -using Azure.Storage.DataMovement.Models; namespace Azure.Storage.DataMovement.Blobs { @@ -58,7 +57,23 @@ public BlobStorageResourceContainer(BlobContainerClient blobContainerClient, Blo /// The path to the storage resource. public override StorageResource GetChildStorageResource(string path) { - return new BlockBlobStorageResource(_blobContainerClient.GetBlockBlobClient(string.Join("/", path))); + switch (_options.BlobType) + { + case BlobType.Block: + return new BlockBlobStorageResource( + _blobContainerClient.GetBlockBlobClient(string.Join("/", path)), + _options.ToBlockBlobStorageResourceOptions()); + case BlobType.Append: + return new AppendBlobStorageResource( + _blobContainerClient.GetAppendBlobClient(string.Join("/", path)), + _options.ToAppendBlobStorageResourceOptions()); + case BlobType.Page: + return new PageBlobStorageResource( + _blobContainerClient.GetPageBlobClient(string.Join("/", path)), + _options.ToPageBlobStorageResourceOptions()); + default: + throw new ArgumentException("Invalid BlobType."); + } } /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs index e1b00b5567d0e..2321a7975cb25 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System; -using System.Collections.Generic; -using Azure.Storage.DataMovement; using Azure.Storage.Blobs.Models; using Azure.Storage.DataMovement.Models; @@ -34,5 +31,12 @@ public class BlobStorageResourceContainerOptions /// Only applies when calling . /// public TransferCopyMethod CopyMethod { get; set; } + + /// + /// The that will be used when uploading blobs to the destination. + /// + /// Defaults to . + /// + public BlobType BlobType { get; set; } = BlobType.Block; } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs index e79d4aa2756fe..8f99dd0e139b5 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs @@ -579,6 +579,58 @@ await UploadBlobDirectoryAndVerify( Directory.Delete(folder, true); } } + + [Test] + [LiveOnly] + [TestCase(BlobType.Block)] + [TestCase(BlobType.Append)] + [TestCase(BlobType.Page)] + public async Task DirectoryUpload_BlobType(BlobType blobType) + { + // Arrange + await using DisposingBlobContainer test = await GetTestContainerAsync(); + + string dirName = GetNewBlobName(); + string folder = CreateRandomDirectory(Path.GetTempPath()); + try + { + string file1 = await CreateRandomFileAsync(folder); + string openSubfolder = CreateRandomDirectory(folder); + string file2 = await CreateRandomFileAsync(openSubfolder); + string destinationPrefix = "foo"; + + TransferManager transferManager = new TransferManager(); + + StorageResourceContainer sourceResource = new LocalDirectoryStorageResourceContainer(folder); + BlobStorageResourceContainerOptions options = new BlobStorageResourceContainerOptions() + { + BlobType = blobType + }; + StorageResourceContainer destinationResource = new BlobDirectoryStorageResourceContainer( + test.Container, + destinationPrefix, + options); + + // Act + DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource); + await transfer.AwaitCompletion(); + + // Assert + AsyncPageable blobs = test.Container.GetBlobsAsync(prefix: destinationPrefix); + await foreach (BlobItem blob in blobs) + { + Assert.AreEqual(blob.Properties.BlobType, blobType); + } + } + catch (Exception ex) + { + Assert.Fail(ex.StackTrace); + } + finally + { + Directory.Delete(folder, true); + } + } #endregion DirectoryUploadTests #region Single Concurrency