[Storage] Parallel Spock in Live Tests (#21102)

* let's see * faster! * try this. * oh lord.. * more... * this too... * huh... * does this help ? * and this. * convert to resource locks. * more. * undo that. * try this. * Focus on blobs in live mode. * this isn't necessary * lock this. * wip * lock this. * wip * Revert "wip" This reverts commit f0427d4. * Revert "lock this." This reverts commit 8fd9585. * Revert "wip" This reverts commit 79ccf80. * how about ? * ... * imports. * avro is fine. * races * shares. * race * Revert "shares." This reverts commit d16e093. * this. * dynamic 5 * revert that * imports. * virtual time... * undo this. * fix whitespace * races.. * unflaky that test. * try parallel datalake. * now this guys is lagging behind. * be kind to data lake. * these require exclusive access.
Azure · May 3, 2021 · 5b4a493 · 5b4a493
1 parent f6defa7
commit 5b4a493
Show file tree

Hide file tree

Showing 24 changed files with 95 additions and 34 deletions.
diff --git a/sdk/storage/azure-storage-blob-batch/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-blob-batch/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled false
+        dynamic(5)
+    }
+}
diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled false
+        dynamic(5)
+    }
+}
diff --git a/...java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobOutputStreamTest.groovy b/...java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobOutputStreamTest.groovy
@@ -14,11 +14,7 @@ class EncryptedBlobOutputStreamTest extends APISpec {
     BlobContainerClient cc
 
     String keyId
-
-    @Shared
     def fakeKey
-
-    @Shared
     def fakeKeyResolver
 
 

diff --git a/...test/java/com/azure/storage/blob/specialized/cryptography/EncyptedBlockBlobAPITest.groovy b/...test/java/com/azure/storage/blob/specialized/cryptography/EncyptedBlockBlobAPITest.groovy
@@ -57,11 +57,7 @@ class EncyptedBlockBlobAPITest extends APISpec {
     EncryptedBlobClient ebc // encrypted client for download
 
     String keyId
-
-    @Shared
     FakeKey fakeKey
-
-    @Shared
     def fakeKeyResolver
 
 

diff --git a/sdk/storage/azure-storage-blob-cryptography/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-blob-cryptography/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled false
+        dynamic(5)
+    }
+}
diff --git a/sdk/storage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/APISpec.groovy b/sdk/storage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/APISpec.groovy
@@ -54,10 +54,7 @@ class APISpec extends Specification {
     Integer entityNo = 0 // Used to generate stable container names for recording tests requiring multiple containers.
 
     // both sync and async clients point to same container
-    @Shared
     BlobContainerClient cc
-
-    @Shared
     BlobContainerAsyncClient ccAsync
 
     // Fields used for conveniently creating blobs with data.
@@ -131,7 +128,11 @@ class APISpec extends Specification {
         String className = specificationContext.getCurrentSpec().getName()
         int iterationIndex = fullTestName.lastIndexOf("[")
         int substringIndex = (int) Math.min((iterationIndex != -1) ? iterationIndex : fullTestName.length(), 50)
-        this.testName = fullTestName.substring(0, substringIndex)
+        if (liveMode()) {
+            this.testName = UUID.randomUUID().toString().replaceAll("-", "")
+        } else {
+            this.testName = fullTestName.substring(0, substringIndex)
+        }
         this.interceptorManager = new InterceptorManager(className + fullTestName, testMode)
         this.resourceNamer = new TestResourceNamer(className + testName, testMode, interceptorManager.getRecordedData())
 

diff --git a/...azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzureFileStoreTest.groovy b/...azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzureFileStoreTest.groovy
@@ -11,7 +11,6 @@ import java.nio.file.attribute.FileStoreAttributeView
 import java.nio.file.attribute.PosixFileAttributeView
 
 class AzureFileStoreTest extends APISpec {
-    @Shared
     AzureFileSystem fs
 
     // Just need one fs instance for creating the stores.

diff --git a/...rage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzurePathTest.groovy b/...rage/azure-storage-blob-nio/src/test/java/com/azure/storage/blob/nio/AzurePathTest.groovy
@@ -3,13 +3,13 @@
 
 package com.azure.storage.blob.nio
 
-import spock.lang.Shared
+import spock.lang.ResourceLock
 import spock.lang.Unroll
 
 import java.nio.file.FileSystems
 
+@ResourceLock("AzurePathTest")
 class AzurePathTest extends APISpec {
-    @Shared
     AzureFileSystem fs
 
     // Just need one fs instance for creating the paths.

diff --git a/sdk/storage/azure-storage-blob-nio/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-blob-nio/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled "LIVE".equalsIgnoreCase(System.getProperty("AZURE_TEST_MODE")) || "LIVE".equalsIgnoreCase(System.getenv("AZURE_TEST_MODE"))
+        dynamic(5)
+    }
+}
diff --git a/...e-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java b/...e-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java
@@ -256,7 +256,7 @@ Mono<Response<BlockBlobItem>> uploadWithResponse(BlockBlobSimpleUploadOptions op
         StorageImplUtils.assertNotNull("options", options);
         Flux<ByteBuffer> data = options.getDataFlux() == null ? Utility.convertStreamToByteBuffer(
             options.getDataStream(), options.getLength(), BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, true)
-            .subscribeOn(Schedulers.elastic())
+            .subscribeOn(Schedulers.boundedElastic())
             : options.getDataFlux();
         BlobRequestConditions requestConditions = options.getRequestConditions() == null ? new BlobRequestConditions()
             : options.getRequestConditions();

diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/APISpec.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/APISpec.groovy
@@ -67,13 +67,8 @@ class APISpec extends Specification {
     Integer entityNo = 0 // Used to generate stable container names for recording tests requiring multiple containers.
 
     // both sync and async clients point to same container
-    @Shared
     BlobContainerClient cc
-
-    @Shared
     BlobContainerClient ccPremium
-
-    @Shared
     BlobContainerAsyncClient ccAsync
 
     // Fields used for conveniently creating blobs with data.
@@ -176,7 +171,11 @@ class APISpec extends Specification {
         String className = specificationContext.getCurrentSpec().getName()
         int iterationIndex = fullTestName.lastIndexOf("[")
         int substringIndex = (int) Math.min((iterationIndex != -1) ? iterationIndex : fullTestName.length(), 50)
-        this.testName = fullTestName.substring(0, substringIndex)
+        if (liveMode()) {
+            this.testName = UUID.randomUUID().toString().replaceAll("-", "")
+        } else {
+            this.testName = fullTestName.substring(0, substringIndex)
+        }
         this.interceptorManager = new InterceptorManager(className + fullTestName, testMode)
         this.resourceNamer = new TestResourceNamer(className + testName, testMode, interceptorManager.getRecordedData())
 

diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAPITest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAPITest.groovy
@@ -46,6 +46,7 @@ import reactor.core.Exceptions
 import reactor.core.publisher.Hooks
 import reactor.test.StepVerifier
 import spock.lang.Requires
+import spock.lang.ResourceLock
 import spock.lang.Unroll
 import spock.lang.Ignore
 
@@ -2952,6 +2953,7 @@ class BlobAPITest extends APISpec {
         AccessTier.COOL | AccessTier.HOT
     }
 
+    @ResourceLock("ServiceProperties")
     def "Undelete"() {
         setup:
         enableSoftDelete()
@@ -2970,6 +2972,7 @@ class BlobAPITest extends APISpec {
         disableSoftDelete() == null
     }
 
+    @ResourceLock("ServiceProperties")
     def "Undelete min"() {
         setup:
         enableSoftDelete()

diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAPITest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ContainerAPITest.groovy
@@ -36,6 +36,7 @@ import com.azure.storage.common.Utility
 import com.azure.storage.common.implementation.StorageImplUtils
 import reactor.test.StepVerifier
 import spock.lang.Requires
+import spock.lang.ResourceLock
 import spock.lang.Unroll
 
 import java.time.Duration
@@ -858,6 +859,7 @@ class ContainerAPITest extends APISpec {
         blobs.size() == 5 // Normal, copy, metadata, tags, uncommitted
     }
 
+    @ResourceLock("ServiceProperties")
     def "List blobs flat options deleted"() {
         setup:
         enableSoftDelete()
@@ -1260,6 +1262,7 @@ class ContainerAPITest extends APISpec {
         blobs.size() == 5 // Normal, copy, metadata, tags, uncommitted
     }
 
+    @ResourceLock("ServiceProperties")
     def "List blobs hier options deleted"() {
         setup:
         enableSoftDelete()

diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/RetryTest.groovy
@@ -7,10 +7,12 @@ import com.azure.core.exception.UnexpectedLengthException
 import com.azure.storage.common.policy.RequestRetryOptions
 import com.azure.storage.common.policy.RetryPolicyType
 import reactor.test.StepVerifier
+import spock.lang.Requires
 import spock.lang.Unroll
 
 import java.time.Duration
 // Tests for package-private functionality.
+@Requires( { playbackMode() }) // https://github.com/reactor/reactor-core/issues/1098
 class RetryTest extends APISpec {
     static URL retryTestURL = new URL("https://" + RequestRetryTestFactory.RETRY_TEST_PRIMARY_HOST)
     static RequestRetryOptions retryTestOptions = new RequestRetryOptions(RetryPolicyType.EXPONENTIAL, 6, 2,

diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ServiceAPITest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/ServiceAPITest.groovy
@@ -3,7 +3,6 @@
 
 package com.azure.storage.blob
 
-import com.azure.core.credential.AzureSasCredential
 import com.azure.core.http.rest.Response
 import com.azure.core.test.TestMode
 import com.azure.core.util.BinaryData
@@ -16,7 +15,6 @@ import com.azure.storage.blob.models.BlobContainerItem
 import com.azure.storage.blob.models.BlobContainerListDetails
 import com.azure.storage.blob.models.BlobCorsRule
 import com.azure.storage.blob.models.BlobMetrics
-import com.azure.storage.blob.models.BlobRequestConditions
 import com.azure.storage.blob.models.BlobRetentionPolicy
 import com.azure.storage.blob.models.BlobServiceProperties
 import com.azure.storage.blob.models.BlobSignedIdentifier
@@ -37,11 +35,13 @@ import com.azure.storage.common.sas.AccountSasService
 import com.azure.storage.common.sas.AccountSasSignatureValues
 import reactor.core.publisher.Mono
 import reactor.test.StepVerifier
+import spock.lang.ResourceLock
 import spock.lang.Unroll
 
 import java.time.Duration
 import java.time.OffsetDateTime
 
+@ResourceLock("ServiceProperties")
 class ServiceAPITest extends APISpec {
 
     BlobServiceClient anonymousClient;

diff --git a/...b/src/test/java/com/azure/storage/blob/implementation/util/StorageBlockingSinkTest.groovy b/...b/src/test/java/com/azure/storage/blob/implementation/util/StorageBlockingSinkTest.groovy
@@ -2,12 +2,15 @@ package com.azure.storage.blob.implementation.util
 
 import com.azure.storage.common.implementation.Constants
 import reactor.core.publisher.Sinks
+import reactor.core.scheduler.Schedulers
 import reactor.test.StepVerifier
 import spock.lang.Specification
 import spock.lang.Unroll
 
 import java.nio.ByteBuffer
 import java.time.Duration
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
 class StorageBlockingSinkTest extends Specification {
 
@@ -195,11 +198,14 @@ class StorageBlockingSinkTest extends Specification {
         def blockingSink = new StorageBlockingSink()
 
         when:
+        CountDownLatch latch = new CountDownLatch(1)
         blockingSink.asFlux()
             .timeout(Duration.ofMillis(100))
+            .doFinally({s -> latch.countDown()})
+            .subscribeOn(Schedulers.boundedElastic())
             .subscribe()
 
-        sleep(500)
+        latch.await(1, TimeUnit.MINUTES)
 
         blockingSink.emitNext(ByteBuffer.wrap(new byte[0]))
 
@@ -209,11 +215,14 @@ class StorageBlockingSinkTest extends Specification {
 
         when:
         blockingSink = new StorageBlockingSink()
+        latch = new CountDownLatch(1)
         blockingSink.asFlux()
             .timeout(Duration.ofMillis(100))
+            .doFinally({s -> latch.countDown()})
+            .subscribeOn(Schedulers.boundedElastic())
             .subscribe()
 
-        sleep(500)
+        latch.await(1, TimeUnit.MINUTES)
 
         blockingSink.emitCompleteOrThrow()
 

diff --git a/...ure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAPITest.groovy b/...ure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAPITest.groovy
@@ -49,9 +49,7 @@ import java.nio.ByteBuffer
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.security.MessageDigest
-import java.time.Duration
 import java.time.OffsetDateTime
-import java.util.concurrent.atomic.AtomicBoolean
 
 class BlockBlobAPITest extends APISpec {
     BlockBlobClient blockBlobClient

diff --git a/...storage-blob/src/test/java/com/azure/storage/blob/specialized/DownloadResponseTest.groovy b/...storage-blob/src/test/java/com/azure/storage/blob/specialized/DownloadResponseTest.groovy
@@ -17,6 +17,7 @@ import com.azure.storage.blob.models.DownloadRetryOptions
 import reactor.core.Exceptions
 import reactor.core.publisher.Mono
 import reactor.test.StepVerifier
+import spock.lang.Requires
 import spock.lang.Unroll
 
 import java.time.Duration
@@ -181,6 +182,7 @@ class DownloadResponseTest extends APISpec {
     }
 
     @Unroll
+    @Requires( { playbackMode() }) // https://github.com/reactor/reactor-core/issues/1098
     def "Timeout"() {
         setup:
         DownloadResponseMockFlux flux = new DownloadResponseMockFlux(DownloadResponseMockFlux.DR_TEST_SCENARIO_TIMEOUT,

diff --git a/sdk/storage/azure-storage-blob/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-blob/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled "LIVE".equalsIgnoreCase(System.getProperty("AZURE_TEST_MODE")) || "LIVE".equalsIgnoreCase(System.getenv("AZURE_TEST_MODE"))
+        dynamic(5)
+    }
+}
diff --git a/.../azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/APISpec.groovy b/.../azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/APISpec.groovy
@@ -56,13 +56,8 @@ class APISpec extends Specification {
     Integer entityNo = 0 // Used to generate stable container names for recording tests requiring multiple containers.
 
     // both sync and async clients point to same container
-    @Shared
     DataLakeFileSystemClient fsc
-
-    @Shared
     DataLakeFileSystemClient fscPremium
-
-    @Shared
     DataLakeFileSystemAsyncClient fscAsync
 
     // Fields used for conveniently creating blobs with data.
@@ -143,7 +138,11 @@ class APISpec extends Specification {
         String className = specificationContext.getCurrentSpec().getName()
         int iterationIndex = fullTestName.lastIndexOf("[")
         int substringIndex = (int) Math.min((iterationIndex != -1) ? iterationIndex : fullTestName.length(), 50)
-        this.testName = fullTestName.substring(0, substringIndex)
+        if (liveMode()) {
+            this.testName = UUID.randomUUID().toString().replaceAll("-", "")
+        } else {
+            this.testName = fullTestName.substring(0, substringIndex)
+        }
         this.interceptorManager = new InterceptorManager(className + fullTestName, testMode)
         this.resourceNamer = new TestResourceNamer(className + testName, testMode, interceptorManager.getRecordedData())
 

diff --git a/sdk/storage/azure-storage-file-datalake/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-file-datalake/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled "LIVE".equalsIgnoreCase(System.getProperty("AZURE_TEST_MODE")) || "LIVE".equalsIgnoreCase(System.getenv("AZURE_TEST_MODE"))
+        dynamic(4)
+    }
+}
diff --git a/sdk/storage/azure-storage-file-share/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-file-share/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled false
+        dynamic(5)
+    }
+}
diff --git a/sdk/storage/azure-storage-internal-avro/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-internal-avro/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled true
+        dynamic(5)
+    }
+}
diff --git a/sdk/storage/azure-storage-queue/src/test/resources/SpockConfig.groovy b/sdk/storage/azure-storage-queue/src/test/resources/SpockConfig.groovy
@@ -0,0 +1,6 @@
+runner {
+    parallel {
+        enabled false
+        dynamic(5)
+    }
+}