[Storage] Parallelize live tests. (#17454)

Fixes: #6905

In this PR:
- introduce LiveParallizable attribute.
- changed scope of DiagosticScopeInterceptor as it's not thread safe, see:
    - https://dev.azure.com/azure-sdk/internal/_build/results?buildId=649697&view=results
    - https://dev.azure.com/azure-sdk/internal/_build/results?buildId=649767&view=results

sdk/core/Azure.Core.TestFramework/src/ClientDiagnosticListener.cs

@@ -104,12 +104,14 @@ public void OnNext(KeyValuePair<string, object> value)
 
         public void OnNext(DiagnosticListener value)
         {
-            List<IDisposable> subscriptions = _subscriptions;
-            if (_sourceNameFilter(value.Name) && subscriptions != null)
+            if (_sourceNameFilter(value.Name) && _subscriptions != null)
             {
                 lock (Scopes)
                 {
-                    subscriptions.Add(value.Subscribe(this));
+                    if (_subscriptions != null)
+                    {
+                        _subscriptions.Add(value.Subscribe(this));
+                    }
                 }
             }
         }

sdk/core/Azure.Core.TestFramework/src/LiveParallelizableAttribute.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using NUnit.Framework;
+
+namespace Azure.Core.TestFramework
+{
+    /// <summary>
+    /// Attribute on test assemblies, classes, or methods that defines parallelization behavior when tests are run in <see cref="RecordedTestMode.Live"/> mode.
+    /// In other modes it will enforce no parallelization.
+    /// </summary>
+    [AttributeUsage( AttributeTargets.Assembly | AttributeTargets.Class | AttributeTargets.Method, AllowMultiple=false, Inherited=true )]
+    public class LiveParallelizableAttribute : ParallelizableAttribute
+    {
+        public LiveParallelizableAttribute(ParallelScope scope) : base(ApplyModeToParallelScope(scope))
+        {
+        }
+
+        private static ParallelScope ApplyModeToParallelScope(ParallelScope scope)
+        {
+            RecordedTestMode mode = RecordedTestUtilities.GetModeFromEnvironment();
+            return mode == RecordedTestMode.Live ? scope : ParallelScope.None;
+        }
+    }
+}

sdk/storage/Azure.Storage.Blobs/tests/BlockBlobClientTests.cs

@@ -3126,7 +3126,7 @@ public async Task SyncUploadFromUriAsync_Lease()
 
             // Take out lease on destination blob
             string leaseId = Recording.Random.NewGuid().ToString();
-            TimeSpan duration = TimeSpan.FromSeconds(15);
+            TimeSpan duration = TimeSpan.FromSeconds(60);
             await InstrumentClient(destBlob.GetBlobLeaseClient(leaseId)).AcquireAsync(duration);
 
             // Upload data to source blob

sdk/storage/Azure.Storage.Common/tests/Shared/AssemblyInfo.cs

@@ -1,11 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using Azure.Core.TestFramework;
 using NUnit.Framework;
 
-// TODO: Look into writing a custom Parallelizable attribute that will run
-// sequentially for record/playback but let us run in parallel for live tests.
-// [assembly: Parallelizable(ParallelScope.Children)]
+[assembly: LiveParallelizable(ParallelScope.Fixtures)]
 
 // Set per-test timeout to 20 minutes to prevent a single test from hanging the whole suite
 [assembly: NUnit.Framework.Timeout(20 * 60 * 1000)]

sdk/storage/Azure.Storage.Files.DataLake/tests/DirectoryClientTests.cs

@@ -1122,6 +1122,7 @@ public async Task SetAccessControlRecursiveAsync_InBatches()
         [Test]
         [LiveOnly]
         [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)]
+        [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/17465")]
         public async Task SetAccessControlRecursiveAsync_InBatches_StopAndResume()
         {
             await using DisposingFileSystem test = await GetNewFileSystem();
@@ -1399,6 +1400,7 @@ public async Task SetAccessControlRecursiveAsync_ContinueOnFailure()
         [Test]
         [LiveOnly]
         [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)]
+        [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/17465")]
         public async Task SetAccessControlRecursiveAsync_ContinueOnFailure_Batches_StopAndResume()
         {
             string fileSystemName = GetNewFileSystemName();
@@ -2093,6 +2095,7 @@ public async Task UpdateAccessControlRecursiveAsync_ContinueOnFailure()
         }
 
         [Test]
+        [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/17465")]
         [LiveOnly]
         [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2020_02_10)]
         public async Task UpdateAccessControlRecursiveAsync_ContinueOnFailure_Batches_StopAndResume()
@@ -2506,6 +2509,7 @@ public async Task RemoveAccessControlRecursiveAsync_InBatches()
         [Test]
         [LiveOnly]
         [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2019_12_12)]
+        [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/17465")]
         public async Task RemoveAccessControlRecursiveAsync_InBatches_StopAndResume()
         {
             await using DisposingFileSystem test = await GetNewFileSystem();

sdk/storage/Azure.Storage.Queues/tests/ServiceClientTests.cs

@@ -15,6 +15,7 @@
 
 namespace Azure.Storage.Queues.Test
 {
+    [NonParallelizable]
     public class ServiceClientTests : QueueTestBase
     {
         public ServiceClientTests(bool async)