[Storage][DataMovement] Refactor/fix support for CreationPreference (#…

…39881)

sdk/storage/Azure.Storage.DataMovement/CHANGELOG.md

@@ -54,7 +54,7 @@
 - [BREAKING CHANGE] Renamed `StorageTransferStatus` to `DataTransferStatus`
 - [BREAKING CHANGE] Changed `DataTransferStatus` from `enum` to a `class`.
 - [BREAKING CHANGE] Renamed `StorageResourceCreateMode` to `StorageResourceCreationPreference`.
-- [BREAKING CHANGE] Renamed `StorageResourceCreationPreference` values from `Fail` to `FailIfExists`, `Overwrite` to `OverwriteIfExists` and `Skip` to `SkipIfExists`. `None` was removed, use `FailIfExists` instead.
+- [BREAKING CHANGE] Renamed `StorageResourceCreationPreference` values from `Fail` to `FailIfExists`, `Overwrite` to `OverwriteIfExists`, `Skip` to `SkipIfExists` and `None` to `Default` which will default to `FailIfExists`.
 - [BREAKING CHANGE] Renamed `DataTransferOptions.CreateMode` to `CreationPreference`.
 - [BREAKING CHANGE] Changed `StorageTransferProgress` constructor from `public` to `protected internal`. 
 - [BREAKING CHANGE] Renamed `StorageTransferProgress` to `DataTransferProgress`.

sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs

@@ -142,9 +142,10 @@ public StorageResourceCopyFromUriOptions() { }
     }
     public enum StorageResourceCreationPreference
     {
-        FailIfExists = 0,
-        OverwriteIfExists = 1,
-        SkipIfExists = 2,
+        Default = 0,
+        FailIfExists = 1,
+        OverwriteIfExists = 2,
+        SkipIfExists = 3,
     }
     public abstract partial class StorageResourceItem : Azure.Storage.DataMovement.StorageResource
     {

sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs

@@ -142,9 +142,10 @@ public StorageResourceCopyFromUriOptions() { }
     }
     public enum StorageResourceCreationPreference
     {
-        FailIfExists = 0,
-        OverwriteIfExists = 1,
-        SkipIfExists = 2,
+        Default = 0,
+        FailIfExists = 1,
+        OverwriteIfExists = 2,
+        SkipIfExists = 3,
     }
     public abstract partial class StorageResourceItem : Azure.Storage.DataMovement.StorageResource
     {

sdk/storage/Azure.Storage.DataMovement/src/DataTransferOptions.cs

@@ -90,8 +90,15 @@ public bool Equals(DataTransferOptions obj)
             && InitialTransferSize == obj?.InitialTransferSize;
 
         /// <summary>
-        /// Optional <see cref="StorageResourceCreationPreference"/> to configure overwrite
-        /// behavior. Will default to <see cref="StorageResourceCreationPreference.OverwriteIfExists"/>.
+        /// Configures the behavior when a transfer encounters a resource that
+        /// already exists.
+        /// <para/>
+        /// Will default to <see cref="StorageResourceCreationPreference.Default"/>
+        /// which will be <see cref="StorageResourceCreationPreference.FailIfExists"/> when
+        /// starting a new transfer.
+        /// When resuming a transfer, the value will default to the value used when first starting
+        /// the transfer for all resources that were successfully enumerated and the regular default
+        /// for any remaining resources.
         /// </summary>
         public StorageResourceCreationPreference CreationPreference { get; set; }
 

sdk/storage/Azure.Storage.DataMovement/src/JobPartInternal.cs

@@ -168,7 +168,6 @@ internal JobPartInternal(
             _sourceResource = sourceResource;
             _destinationResource = destinationResource;
             _errorHandling = errorHandling;
-            _createMode = createMode;
             _failureType = JobPartFailureType.None;
             _checkpointer = checkpointer;
             _progressTracker = progressTracker;
@@ -189,6 +188,9 @@ internal JobPartInternal(
             _transferChunkSize = Math.Min(
                 transferChunkSize ?? DataMovementConstants.DefaultChunkSize,
                 _destinationResource.MaxSupportedChunkSize);
+            // Set the default create mode
+            _createMode = createMode == StorageResourceCreationPreference.Default ?
+                StorageResourceCreationPreference.FailIfExists : createMode;
 
             Length = length;
             _chunkTasks = new List<Task<bool>>();

sdk/storage/Azure.Storage.DataMovement/src/JobPlan/JobPartPlanHeader.cs

@@ -51,12 +51,12 @@ internal class JobPartPlanHeader
         public string DestinationPath;
 
         /// <summary>
-        /// Whether the destination should be overriden or not.
+        /// The resource creation preference.
         /// </summary>
-        public bool Overwrite;
+        public StorageResourceCreationPreference CreatePreference;
 
         /// <summary>
-        /// Ths intial transfer size for the transfer.
+        /// Ths initial transfer size for the transfer.
         /// </summary>
         public long InitialTransferSize;
 
@@ -84,7 +84,7 @@ public JobPartPlanHeader(
             string destinationTypeId,
             string sourcePath,
             string destinationPath,
-            bool overwrite,
+            StorageResourceCreationPreference createPreference,
             long initialTransferSize,
             long chunkSize,
             byte priority,
@@ -133,7 +133,7 @@ public JobPartPlanHeader(
             DestinationTypeId = destinationTypeId;
             SourcePath = sourcePath;
             DestinationPath = destinationPath;
-            Overwrite = overwrite;
+            CreatePreference = createPreference;
             InitialTransferSize = initialTransferSize;
             ChunkSize = chunkSize;
             Priority = priority;
@@ -173,8 +173,8 @@ public void Serialize(Stream stream)
             byte[] destinationPathBytes = Encoding.UTF8.GetBytes(DestinationPath);
             writer.WriteVariableLengthFieldInfo(destinationPathBytes.Length, ref currentVariableLengthIndex);
 
-            // Overwrite
-            writer.Write(Overwrite);
+            // CreatePreference
+            writer.Write((byte)CreatePreference);
 
             // InitialTransferSize
             writer.Write(InitialTransferSize);
@@ -233,9 +233,8 @@ public static JobPartPlanHeader Deserialize(Stream stream)
             int destinationPathOffset = reader.ReadInt32();
             int destinationPathLength = reader.ReadInt32();
 
-            // Overwrite
-            byte overwriteByte = reader.ReadByte();
-            bool overwrite = Convert.ToBoolean(overwriteByte);
+            // CreatePreference
+            StorageResourceCreationPreference createPreference = (StorageResourceCreationPreference)reader.ReadByte();
 
             // InitialTransferSize
             long initialTransferSize = reader.ReadInt64();
@@ -277,7 +276,7 @@ public static JobPartPlanHeader Deserialize(Stream stream)
                 destinationTypeId,
                 sourcePath,
                 destinationPath,
-                overwrite,
+                createPreference,
                 initialTransferSize,
                 chunkSize,
                 priority,
@@ -303,7 +302,7 @@ internal bool Equals(JobPartPlanHeader other)
                 (DestinationTypeId == other.DestinationTypeId) &&
                 (SourcePath == other.SourcePath) &&
                 (DestinationPath == other.DestinationPath) &&
-                (Overwrite == other.Overwrite) &&
+                (CreatePreference == other.CreatePreference) &&
                 (InitialTransferSize == other.InitialTransferSize) &&
                 (ChunkSize == other.ChunkSize) &&
                 (Priority == other.Priority) &&

sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs

@@ -53,10 +53,7 @@ private ServiceToServiceJobPart(
             int partNumber,
             StorageResourceItem sourceResource,
             StorageResourceItem destinationResource,
-            DataTransferStatus jobPartStatus = default,
-            long? length = default,
-            long? initialTransferSize = default,
-            long? transferChunkSize = default)
+            long? length = default)
             : base(dataTransfer: job._dataTransfer,
                   partNumber: partNumber,
                   sourceResource: sourceResource,
@@ -75,20 +72,44 @@ private ServiceToServiceJobPart(
                   singleTransferEventHandler: job.TransferItemCompletedEventHandler,
                   clientDiagnostics: job.ClientDiagnostics,
                   cancellationToken: job._cancellationToken,
-                  jobPartStatus: jobPartStatus,
+                  jobPartStatus: default,
                   length: length)
         {
-            // If transfer sizes null at the job level (from options bag) then
-            // override the default with the provided values if present.
-            // Else, they were set correctly by the base constructor.
-            if (!job._maximumTransferChunkSize.HasValue && transferChunkSize.HasValue)
-            {
-                _transferChunkSize = transferChunkSize.Value;
-            }
-            if (!job._initialTransferSize.HasValue && initialTransferSize.HasValue)
-            {
-                _initialTransferSize = initialTransferSize.Value;
-            }
+        }
+
+        /// <summary>
+        /// Creating transfer job based on a checkpoint file.
+        /// </summary>
+        private ServiceToServiceJobPart(
+            ServiceToServiceTransferJob job,
+            int partNumber,
+            StorageResourceItem sourceResource,
+            StorageResourceItem destinationResource,
+            DataTransferStatus jobPartStatus,
+            long initialTransferSize,
+            long transferChunkSize,
+            StorageResourceCreationPreference createPreference)
+            : base(dataTransfer: job._dataTransfer,
+                  partNumber: partNumber,
+                  sourceResource: sourceResource,
+                  destinationResource: destinationResource,
+                  transferChunkSize: transferChunkSize,
+                  initialTransferSize: initialTransferSize,
+                  errorHandling: job._errorMode,
+                  createMode: createPreference,
+                  checkpointer: job._checkpointer,
+                  progressTracker: job._progressTracker,
+                  arrayPool: job.UploadArrayPool,
+                  jobPartEventHandler: job.GetJobPartStatus(),
+                  statusEventHandler: job.TransferStatusEventHandler,
+                  failedEventHandler: job.TransferFailedEventHandler,
+                  skippedEventHandler: job.TransferSkippedEventHandler,
+                  singleTransferEventHandler: job.TransferItemCompletedEventHandler,
+                  clientDiagnostics: job.ClientDiagnostics,
+                  cancellationToken: job._cancellationToken,
+                  jobPartStatus: jobPartStatus,
+                  length: default)
+        {
         }
 
         public async ValueTask DisposeAsync()
@@ -140,7 +161,8 @@ public static ServiceToServiceJobPart CreateJobPartFromCheckpoint(
             StorageResourceItem destinationResource,
             DataTransferStatus jobPartStatus,
             long initialTransferSize,
-            long transferChunkSize)
+            long transferChunkSize,
+            StorageResourceCreationPreference createPreference)
         {
             return new ServiceToServiceJobPart(
                 job: job,
@@ -149,7 +171,8 @@ public static ServiceToServiceJobPart CreateJobPartFromCheckpoint(
                 destinationResource: destinationResource,
                 jobPartStatus: jobPartStatus,
                 initialTransferSize: initialTransferSize,
-                transferChunkSize: transferChunkSize);
+                transferChunkSize: transferChunkSize,
+                createPreference: createPreference);
         }
 
         public override async Task ProcessPartToChunkAsync()

sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs

@@ -126,8 +126,8 @@ internal static class JobPartPlanFile
             internal const int SourcePathLengthIndex = SourcePathOffsetIndex + IntSizeInBytes;
             internal const int DestinationPathOffsetIndex = SourcePathLengthIndex + IntSizeInBytes;
             internal const int DestinationPathLengthIndex = DestinationPathOffsetIndex + IntSizeInBytes;
-            internal const int OverwriteIndex = DestinationPathLengthIndex + IntSizeInBytes;
-            internal const int InitialTransferSizeIndex = OverwriteIndex + OneByte;
+            internal const int CreatePreferenceIndex = DestinationPathLengthIndex + IntSizeInBytes;
+            internal const int InitialTransferSizeIndex = CreatePreferenceIndex + OneByte;
             internal const int ChunkSizeIndex = InitialTransferSizeIndex + LongSizeInBytes;
             internal const int PriorityIndex = ChunkSizeIndex + LongSizeInBytes;
             internal const int JobPartStatusIndex = PriorityIndex + OneByte;