multiple sharding

Flank · Jul 9, 2021 · 3ebc0de · 3ebc0de
1 parent b021475
commit 3ebc0de
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 18 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt
@@ -250,7 +250,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
 
     @set:CommandLine.Option(
         names = ["--parameterized-tests"],
-        description = ["Specifies how to handle tests which contain the parameterization annotation. Possible values: `default`, `ignore-all`, `shard-into-single`, leaving it blank will result in `default` sharding"]
+        description = [
+            "Specifies how to handle tests which contain the parameterization annotation. Possible values: `default`, `ignore-all`, `shard-into-single`, `shard-into-multiple`.\n" +
+                "leaving it blank will result in `default` sharding.\n" +
+                "Note: Making use of shard-into-single` or `shard-into-multiple will result in additional shards being created even if a max number of shards has been specified.\n" +
+                "Note: If shard-into-single is used, a single additional shard is created that will run the Parameterized tests separately.\n" +
+                "Note: If shard-into-multiple is used, each parameterized test will be matched by its corresponding name and sharded into a separate shard. This may dramatically increase the amount of expected shards depending upon how many parameterized tests are discovered."
+        ]
     )
     @set:JsonProperty("parameterized-tests")
     var parameterizedTests: String? by data

diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt
@@ -96,20 +96,15 @@ private fun InstrumentationTestContext.calculateShardsNormally(
 private fun InstrumentationTestContext.calculateShards(
     args: AndroidArgs,
     testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets, args.testTargetsForShard)
-): InstrumentationTestContext = when {
-    args.parameterizedTests.shouldShardIntoSingle() -> {
-        var flankTestMethods = getFlankTestMethods(testFilter, args.parameterizedTests)
-        val parameterizedTests = flankTestMethods.filter { it.isParameterizedClass }
-        flankTestMethods = flankTestMethods.filterNot { it.isParameterizedClass }
-        val shards = calculateParameterizedShards(flankTestMethods, args)
-        val parameterizedShard = calculateParameterizedShards(parameterizedTests, args, 1)
-        shards.copy(shards = shards.shards + parameterizedShard.shards)
-    }
-    args.parameterizedTests.shouldShardMultiple() -> {
-        calculateShardsNormally(args, testFilter)
-    }
-    else -> calculateShardsNormally(args, testFilter)
-}
+): InstrumentationTestContext = if (args.parameterizedTests.shouldShardSmartly()) {
+    var flankTestMethods = getFlankTestMethods(testFilter, args.parameterizedTests)
+    val parameterizedTests = flankTestMethods.filter { it.isParameterizedClass }
+    flankTestMethods = flankTestMethods.filterNot { it.isParameterizedClass }
+    val shards = calculateParameterizedShards(flankTestMethods, args)
+    val shardCount = if (args.parameterizedTests.isSingleParameterizedShard()) 1 else parameterizedTests.size
+    val parameterizedShard = calculateParameterizedShards(parameterizedTests, args, shardCount)
+    shards.copy(shards = shards.shards + parameterizedShard.shards)
+} else calculateShardsNormally(args, testFilter)
 
 private fun InstrumentationTestContext.calculateParameterizedShards(
     filteredTests: List<FlankTestMethod>,
@@ -126,9 +121,6 @@ private fun InstrumentationTestContext.calculateParameterizedShards(
     )
 }
 
-private fun String.shouldShardIntoSingle() = (this == "shard-into-single")
-private fun String.shouldShardMultiple() = (this == "shard-into-multiple")
-
 private fun InstrumentationTestContext.calculateDummyShards(
     args: AndroidArgs,
     testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets, args.testTargetsForShard)
@@ -183,6 +175,8 @@ private val ignoredAnnotations = listOf(
     "android.support.test.filters.Suppress"
 )
 
+private fun String.shouldShardSmartly() = (this == "shard-into-single" || this == "shard-into-multiple")
+private fun String.isSingleParameterizedShard() = (this == "shard-into-single")
 private fun String.shouldIgnore(): Boolean = (this == "ignore-all")
 
 @VisibleForTesting

diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
@@ -2832,6 +2832,20 @@ AndroidArgs
         val chunks = runBlocking { parsedYml.runAndroidTests() }.shardChunks
         assertTrue(chunks.size == 1)
     }
+
+    @Test
+    fun `should shard tests into multiple new shards with shard-into-multiple`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testExtremeParameterizedOtherApk
+          parameterized-tests: shard-into-multiple
+        """.trimIndent()
+
+        val parsedYml = AndroidArgs.load(yaml).validate()
+        val chunks = runBlocking { parsedYml.runAndroidTests() }.shardChunks
+        assertTrue(chunks.size == 5)
+    }
 }
 
 private fun AndroidArgs.Companion.load(