[Storage] [DataMovement] Add BlobType option to `BlobStorageResourceC…

…ontainerOptions` (#35081)
Azure · Mar 24, 2023 · 6e80e34 · 6e80e34
1 parent ac06f86
commit 6e80e34
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 13 deletions.
diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## 12.0.0-beta.2 (Unreleased)
 - This release contains bug fixes to improve quality.
+- Added option to `BlobStorageResourceContainerOptions` to choose `BlobType` when uploading blobs.
 
 ## 12.0.0-beta.1 (2022-12-15)
 - This preview is the first release of a ground-up rewrite of our client data movement

diff --git a/...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs b/...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs
@@ -58,6 +58,7 @@ public BlobStorageResourceContainer(Azure.Storage.Blobs.BlobContainerClient blob
     public partial class BlobStorageResourceContainerOptions
     {
         public BlobStorageResourceContainerOptions() { }
+        public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } set { } }
         public Azure.Storage.DataMovement.Models.TransferCopyMethod CopyMethod { get { throw null; } set { } }
         public Azure.Storage.Blobs.Models.BlobStates States { get { throw null; } set { } }
         public Azure.Storage.Blobs.Models.BlobTraits Traits { get { throw null; } set { } }

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDirectoryStorageResourceContainer.cs
@@ -3,18 +3,13 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
 using System.Runtime.CompilerServices;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Core;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
-using Azure.Storage.DataMovement;
-using Azure.Storage.DataMovement.Models;
 
 namespace Azure.Storage.DataMovement.Blobs
 {
@@ -85,9 +80,23 @@ public BlobDirectoryStorageResourceContainer(
         public override StorageResource GetChildStorageResource(string path)
         {
             // Recreate the blobName using the existing parent directory path
-            return new BlockBlobStorageResource(
-                _blobContainerClient.GetBlockBlobClient(System.IO.Path.Combine(_directoryPrefix, path)),
-                _options.ToBlockBlobStorageResourceOptions());
+            switch (_options.BlobType)
+            {
+                case BlobType.Block:
+                    return new BlockBlobStorageResource(
+                        _blobContainerClient.GetBlockBlobClient(System.IO.Path.Combine(_directoryPrefix, path)),
+                        _options.ToBlockBlobStorageResourceOptions());
+                case BlobType.Append:
+                    return new AppendBlobStorageResource(
+                        _blobContainerClient.GetAppendBlobClient(System.IO.Path.Combine(_directoryPrefix, path)),
+                        _options.ToAppendBlobStorageResourceOptions());
+                case BlobType.Page:
+                    return new PageBlobStorageResource(
+                        _blobContainerClient.GetPageBlobClient(System.IO.Path.Combine(_directoryPrefix, path)),
+                        _options.ToPageBlobStorageResourceOptions());
+                default:
+                    throw new ArgumentException("Invalid BlobType.");
+            }
         }
 
         /// <summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs
@@ -9,7 +9,6 @@
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
-using Azure.Storage.DataMovement.Models;
 
 namespace Azure.Storage.DataMovement.Blobs
 {
@@ -58,7 +57,23 @@ public BlobStorageResourceContainer(BlobContainerClient blobContainerClient, Blo
         /// <param name="path">The path to the storage resource.</param>
         public override StorageResource GetChildStorageResource(string path)
         {
-            return new BlockBlobStorageResource(_blobContainerClient.GetBlockBlobClient(string.Join("/", path)));
+            switch (_options.BlobType)
+            {
+                case BlobType.Block:
+                    return new BlockBlobStorageResource(
+                        _blobContainerClient.GetBlockBlobClient(string.Join("/", path)),
+                        _options.ToBlockBlobStorageResourceOptions());
+                case BlobType.Append:
+                    return new AppendBlobStorageResource(
+                        _blobContainerClient.GetAppendBlobClient(string.Join("/", path)),
+                        _options.ToAppendBlobStorageResourceOptions());
+                case BlobType.Page:
+                    return new PageBlobStorageResource(
+                        _blobContainerClient.GetPageBlobClient(string.Join("/", path)),
+                        _options.ToPageBlobStorageResourceOptions());
+                default:
+                    throw new ArgumentException("Invalid BlobType.");
+            }
         }
 
         /// <summary>

diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainerOptions.cs
@@ -1,9 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using System;
-using System.Collections.Generic;
-using Azure.Storage.DataMovement;
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.DataMovement.Models;
 
@@ -34,5 +31,12 @@ public class BlobStorageResourceContainerOptions
         /// Only applies when calling <see cref="BlockBlobStorageResource.CopyBlockFromUriAsync(StorageResource, HttpRange, bool, long, StorageResourceCopyFromUriOptions, System.Threading.CancellationToken)"/>.
         /// </summary>
         public TransferCopyMethod CopyMethod { get; set; }
+
+        /// <summary>
+        /// The <see cref="BlobType"/> that will be used when uploading blobs to the destination.
+        ///
+        /// Defaults to <see cref="BlobType.Block"/>.
+        /// </summary>
+        public BlobType BlobType { get; set; } = BlobType.Block;
     }
 }
diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferUploadDirectoryTests.cs
@@ -579,6 +579,58 @@ await UploadBlobDirectoryAndVerify(
                 Directory.Delete(folder, true);
             }
         }
+
+        [Test]
+        [LiveOnly]
+        [TestCase(BlobType.Block)]
+        [TestCase(BlobType.Append)]
+        [TestCase(BlobType.Page)]
+        public async Task DirectoryUpload_BlobType(BlobType blobType)
+        {
+            // Arrange
+            await using DisposingBlobContainer test = await GetTestContainerAsync();
+
+            string dirName = GetNewBlobName();
+            string folder = CreateRandomDirectory(Path.GetTempPath());
+            try
+            {
+                string file1 = await CreateRandomFileAsync(folder);
+                string openSubfolder = CreateRandomDirectory(folder);
+                string file2 = await CreateRandomFileAsync(openSubfolder);
+                string destinationPrefix = "foo";
+
+                TransferManager transferManager = new TransferManager();
+
+                StorageResourceContainer sourceResource = new LocalDirectoryStorageResourceContainer(folder);
+                BlobStorageResourceContainerOptions options = new BlobStorageResourceContainerOptions()
+                {
+                    BlobType = blobType
+                };
+                StorageResourceContainer destinationResource = new BlobDirectoryStorageResourceContainer(
+                    test.Container,
+                    destinationPrefix,
+                    options);
+
+                // Act
+                DataTransfer transfer = await transferManager.StartTransferAsync(sourceResource, destinationResource);
+                await transfer.AwaitCompletion();
+
+                // Assert
+                AsyncPageable<BlobItem> blobs = test.Container.GetBlobsAsync(prefix: destinationPrefix);
+                await foreach (BlobItem blob in blobs)
+                {
+                    Assert.AreEqual(blob.Properties.BlobType, blobType);
+                }
+            }
+            catch (Exception ex)
+            {
+                Assert.Fail(ex.StackTrace);
+            }
+            finally
+            {
+                Directory.Delete(folder, true);
+            }
+        }
         #endregion DirectoryUploadTests
 
         #region Single Concurrency