fix: Add additional index to matrix when multiple test runs (#1261)

* Update RunAndroidTests.kt * Update RunAndroidTests.kt * Added tests
Flank · Oct 22, 2020 · 2726065 · 2726065
1 parent cc86dac
commit 2726065
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 5 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt
@@ -48,16 +48,16 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
     val additionalApks = args.uploadAdditionalApks(runGcsPath)
 
     args.createAndroidTestContexts().dumpShards(args).upload(args.resultsBucket, runGcsPath)
-        .forEachIndexed { index, context ->
+        .forEachIndexed { contextIndex, context ->
             if (context is InstrumentationTestContext) {
                 ignoredTestsShardChunks += context.ignoredTestCases
                 allTestShardChunks += context.shards
             }
             val androidTestConfig = args.createAndroidTestConfig(context)
-            testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) {
+            testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { runIndex ->
                 GcAndroidTestMatrix.build(
                     androidTestConfig = androidTestConfig,
-                    runGcsPath = "$runGcsPath/matrix_$index/",
+                    runGcsPath = runGcsPath.createGcsPath(contextIndex, runIndex),
                     additionalApkGcsPaths = additionalApks,
                     androidDeviceList = devices,
                     args = args,
@@ -78,6 +78,10 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
     )
 }
 
+private fun String.createGcsPath(contextIndex: Int, runIndex: Int) =
+    if (runIndex == 0) "$this/matrix_$contextIndex/"
+    else "$this/matrix_${contextIndex}_$runIndex/"
+
 private fun List<AndroidTestContext>.dumpShards(config: AndroidArgs) = takeIf { config.isInstrumentationTest }?.apply {
     filterIsInstance<InstrumentationTestContext>().asMatrixTestShards().saveShards(config.obfuscateDumpShards)
     if (config.disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, config.resultsBucket, config.resultsDir)
@@ -92,11 +96,11 @@ private fun AndroidMatrixTestShards.saveShards(obfuscateOutput: Boolean) = saveS
 
 private suspend fun executeAndroidTestMatrix(
     runCount: Int,
-    createTestMatrix: () -> Testing.Projects.TestMatrices.Create
+    createTestMatrix: (runIndex: Int) -> Testing.Projects.TestMatrices.Create
 ): List<Deferred<TestMatrix>> = coroutineScope {
     (0 until runCount).map {
         async(Dispatchers.IO) {
-            createTestMatrix().executeWithRetry()
+            createTestMatrix(it).executeWithRetry()
         }
     }
 }
diff --git a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt
@@ -1,10 +1,14 @@
 package ftl.run.platform
 
 import ftl.args.AndroidArgs
+import ftl.gc.GcAndroidTestMatrix
 import ftl.run.model.TestResult
 import ftl.test.util.FlankTestRunner
 import ftl.test.util.mixedConfigYaml
 import ftl.test.util.should
+import io.mockk.every
+import io.mockk.mockkObject
+import io.mockk.verify
 import kotlinx.coroutines.runBlocking
 import org.junit.Assert.assertEquals
 import org.junit.Test
@@ -34,4 +38,54 @@ class RunAndroidTestsKtTest {
         // then
         assertEquals(expected, actual)
     }
+
+    @Test
+    fun `should add additional index if repeatTests set`() {
+        val androidArgs = AndroidArgs.load(mixedConfigYaml)
+        mockkObject(GcAndroidTestMatrix, androidArgs) {
+
+            every { androidArgs.resultsDir } returns "test_dir"
+            every { androidArgs.resultsBucket } returns "test_bucket"
+            every { androidArgs.repeatTests } returns 2
+
+            runBlocking {
+                runAndroidTests(androidArgs)
+            }
+
+            verify {
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_0/", any(), any(), any(), any())
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_0_1/", any(), any(), any(), any())
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_1/", any(), any(), any(), any())
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_1_1/", any(), any(), any(), any())
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_2/", any(), any(), any(), any())
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_2_1/", any(), any(), any(), any())
+            }
+
+            verify(inverse = true) {
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_0_2/", any(), any(), any(), any())
+            }
+        }
+    }
+
+    @Test
+    fun `shouldn't add additional index if repeatTests not set`() {
+        val androidArgs = AndroidArgs.load(mixedConfigYaml)
+        mockkObject(GcAndroidTestMatrix, androidArgs) {
+
+            every { androidArgs.resultsDir } returns "test_dir"
+            every { androidArgs.resultsBucket } returns "test_bucket"
+
+            runBlocking {
+                runAndroidTests(androidArgs)
+            }
+
+            verify(inverse = true) {
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_0_1/", any(), any(), any(), any())
+            }
+
+            verify {
+                GcAndroidTestMatrix.build(any(), any(), "test_dir/matrix_0/", any(), any(), any(), any())
+            }
+        }
+    }
 }