diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt index 4dc70e67be..260674d1e8 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt @@ -32,6 +32,20 @@ object ArgsToString { fun apksToString(devices: List): String { if (devices.isNullOrEmpty()) return "" - return NEW_LINE + devices.joinToString(System.lineSeparator()) { (app, test) -> " - app: $app\n test: $test" } + + return NEW_LINE + devices.joinToString(System.lineSeparator()) { + val sep = System.lineSeparator() + val environmentVars = if (it.environmentVariables.isNotEmpty()) { + "$sep environment-variables:$sep" + + " ${it.environmentVariables.toList().joinToString("$sep ") { pair -> "${pair.first}: ${pair.second}" }}" + } else "" + + val clientDetails = if (it.clientDetails.isNotEmpty()) { + "$sep client-details:$sep" + + " ${it.clientDetails.toList().joinToString("$sep ") { pair -> "${pair.first}: ${pair.second}" }}" + } else "" + val maxTestShards = if (it.maxTestShards != null) "$sep max-test-shards: ${it.maxTestShards}" else "" + " - app: ${it.app}$sep test: ${it.test}$maxTestShards$clientDetails$environmentVars" + } } } diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index f64bea71a4..bbeb3d7732 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -32,11 +32,19 @@ fun createAndroidArgs( roboScript = gcloud.roboScript?.normalizeFilePath(), // flank - additionalAppTestApks = flank.additionalAppTestApks?.map { (app, test, env) -> + additionalAppTestApks = flank.additionalAppTestApks?.map { + // if additional-pair did not provide certain values, set as top level ones + val mergedClientDetails = mutableMapOf().apply { + // merge additionalAppTestApk's client-details with top-level client-details + putAll(commonArgs.clientDetails ?: emptyMap()) + putAll(it.clientDetails) + } AppTestPair( - app = app?.normalizeFilePath(), - test = test.normalizeFilePath(), - environmentVariables = env + app = it.app?.normalizeFilePath(), + test = it.test.normalizeFilePath(), + environmentVariables = it.environmentVariables, + maxTestShards = it.maxTestShards ?: commonArgs.maxTestShards, + clientDetails = mergedClientDetails ) } ?: emptyList(), useLegacyJUnitResult = flank::useLegacyJUnitResult.require(), diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt index 2912107d3c..0acfbf3ee3 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt @@ -8,5 +8,9 @@ data class AppTestPair( val app: String?, val test: String, @JsonProperty("environment-variables") - val environmentVariables: Map = emptyMap() + val environmentVariables: Map = emptyMap(), + @JsonProperty("max-test-shards") + val maxTestShards: Int? = null, + @JsonProperty("client-details") + var clientDetails: Map = emptyMap() ) diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt index 67c0ab2c73..0699fab719 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt @@ -35,7 +35,8 @@ data class AndroidFlankConfig @JsonIgnore constructor( additionalAppTestApks?.add( AppTestPair( app = appApk, - test = testApk + test = testApk, + maxTestShards = map["max-test-shards"]?.toInt() ) ) } diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index 10e8fc8892..b378a7eb96 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -39,10 +39,15 @@ object GcAndroidTestMatrix { obbFiles: Map ): Testing.Projects.TestMatrices.Create { // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 - val clientInfo = ClientInfo() - .setName("Flank") - .setClientInfoDetails(args.clientDetails?.toClientInfoDetailList()) - + val clientInfo = if (androidTestConfig is AndroidTestConfig.Instrumentation && androidTestConfig.clientDetails.isNotEmpty()) { + ClientInfo() + .setName("Flank") + .setClientInfoDetails(androidTestConfig.clientDetails.toClientInfoDetailList()) + } else { + ClientInfo() + .setName("Flank") + .setClientInfoDetails(args.clientDetails?.toClientInfoDetailList()) + } val matrixGcsPath = join(args.resultsBucket, runGcsPath) // --auto-google-login diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index f26a8f87e7..dc8e71216f 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -1,5 +1,7 @@ package ftl.mock +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import com.google.api.services.toolresults.model.AppStartTime import com.google.api.services.toolresults.model.CPUInfo import com.google.api.services.toolresults.model.Duration @@ -27,6 +29,7 @@ import com.google.gson.GsonBuilder import com.google.gson.LongSerializationPolicy import com.google.testing.model.AndroidDevice import com.google.testing.model.AndroidDeviceCatalog +import com.google.testing.model.ClientInfo import com.google.testing.model.Environment import com.google.testing.model.GoogleCloudStorage import com.google.testing.model.IosDeviceCatalog @@ -37,8 +40,10 @@ import com.google.testing.model.TestExecution import com.google.testing.model.TestMatrix import com.google.testing.model.ToolResultsExecution import com.google.testing.model.ToolResultsStep +import ftl.analytics.objectToMap import ftl.config.FtlConstants import ftl.config.FtlConstants.JSON_FACTORY +import ftl.gc.toClientInfoDetailList import ftl.log.LogbackLogger import ftl.run.exception.FlankGeneralError import ftl.util.Bash @@ -52,6 +57,7 @@ import io.ktor.application.install import io.ktor.features.ContentNegotiation import io.ktor.gson.GsonConverter import io.ktor.http.ContentType +import io.ktor.request.receive import io.ktor.request.uri import io.ktor.response.respond import io.ktor.routing.get @@ -59,10 +65,14 @@ import io.ktor.routing.post import io.ktor.routing.routing import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.net.BindException +import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.Files import java.nio.file.Paths import java.util.concurrent.atomic.AtomicInteger +import java.util.zip.GZIPInputStream object MockServer { @@ -178,7 +188,35 @@ object MockServer { post("/v1/projects/{project}/testMatrices") { println("Responding to POST ${call.request.uri}") val projectId = call.parameters["project"] - + val requestBody = withContext(Dispatchers.IO) { + GZIPInputStream(call.receive().inputStream()).bufferedReader(UTF_8).use { + ObjectMapper().readValue>(it.readText()) + } + } + val clientName = requestBody["clientInfo"]?.objectToMap()?.get("name") as String + val allClientDetails = mutableMapOf() + requestBody["clientInfo"] + ?.objectToMap() + ?.get("clientInfoDetails")?.let { list -> + if (list is List<*>) { + list.forEach { map -> + if (map is Map<*, *>) { + map.toList().chunked(2).forEach { + if (it.size == 2) { + val k = it[0].second + val v = it[1].second + if (k is String && v is String) { + allClientDetails[k] = v + } + } + } + } + } + } + } + val clientInfo = ClientInfo() + .setName(clientName) + .setClientInfoDetails(allClientDetails.toClientInfoDetailList()) val matrixId = matrixIdCounter.incrementAndGet().toString() val resultStorage = ResultStorage().apply { @@ -207,6 +245,7 @@ object MockServer { .setEnvironment(environment) val matrix = TestMatrix() + .setClientInfo(clientInfo) .setProjectId(projectId) .setTestMatrixId("matrix-$matrixId") .setState("FINISHED") diff --git a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt index 151e10455b..860ec4ea8f 100644 --- a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt @@ -13,7 +13,9 @@ data class InstrumentationTestContext( val shards: List = emptyList(), val ignoredTestCases: IgnoredTestCases = emptyList(), val environmentVariables: Map = emptyMap(), - val testTargetsForShard: ShardChunks = emptyList() + val testTargetsForShard: ShardChunks = emptyList(), + val maxTestShards: Int? = null, + val clientDetail: Map = emptyMap(), ) : AndroidTestContext() data class RoboTestContext( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt index 057c9e7702..9f94546159 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt @@ -15,7 +15,8 @@ sealed class AndroidTestConfig { val numUniformShards: Int?, val keepTestTargetsEmpty: Boolean, val environmentVariables: Map = emptyMap(), - val testTargetsForShard: ShardChunks + val testTargetsForShard: ShardChunks, + val clientDetails: Map = emptyMap() ) : AndroidTestConfig() data class Robo( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 3509ac2f15..c476ece63b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -38,14 +38,15 @@ private suspend fun List.setupShards( testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets, args.testTargetsForShard) ): List = coroutineScope { map { testContext -> + val newArgs = if (testContext is InstrumentationTestContext && testContext.maxTestShards != null) { + args.copy(commonArgs = args.commonArgs.copy(maxTestShards = testContext.maxTestShards)) + } else args async { when { testContext !is InstrumentationTestContext -> testContext - args.useCustomSharding -> testContext.userShards(args.customSharding) - args.useTestTargetsForShard -> - testContext.downloadApks() - .calculateDummyShards(args, testFilter) - else -> testContext.downloadApks().calculateShards(args, testFilter) + newArgs.useCustomSharding -> testContext.userShards(newArgs.customSharding) + newArgs.useTestTargetsForShard -> testContext.downloadApks().calculateDummyShards(newArgs, testFilter) + else -> testContext.downloadApks().calculateShards(newArgs, testFilter) } } }.awaitAll().dropEmptyInstrumentationTest() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt index eaed5ce24e..2a8d6a2dd0 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt @@ -16,5 +16,6 @@ internal fun AndroidArgs.createInstrumentationConfig( testShards = testApk.shards.testCases, keepTestTargetsEmpty = disableSharding && testTargets.isEmpty(), environmentVariables = testApk.environmentVariables, - testTargetsForShard = testTargetsForShard + testTargetsForShard = testTargetsForShard, + clientDetails = testApk.clientDetail ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index d8c6010876..a05a7d4c7f 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -40,6 +40,8 @@ private fun AndroidArgs.additionalApksContexts() = additionalAppTestApks.map { app = appApk.asFileReference(), test = it.test.asFileReference(), environmentVariables = it.environmentVariables, - testTargetsForShard = testTargetsForShard + testTargetsForShard = testTargetsForShard, + maxTestShards = it.maxTestShards, + clientDetail = it.clientDetails ) }.toTypedArray() diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index 30ea53ab11..5605ecaac2 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -133,6 +133,42 @@ class AndroidArgsFileTest { } } + @Test + fun `calculateShards additionalAppTestApks with override`() { + val test1 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_1.apk" + val test155 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_155.apk" + val config = createAndroidArgs( + defaultAndroidConfig().apply { + platform.apply { + gcloud.apply { + app = appApkLocal + test = getString(test1) + } + flank.apply { + additionalAppTestApks = mutableListOf( + AppTestPair( + app = appApkLocal, + test = getString(test155), + maxTestShards = 4 + ) + ) + } + } + common.flank.maxTestShards = 3 + } + ) + with(runBlocking { config.getAndroidMatrixShards() }) { + assertEquals(1, get("matrix-0")!!.shards.size) + assertEquals(4, get("matrix-1")!!.shards.size) + assertEquals(1, get("matrix-0")!!.shards["shard-0"]!!.size) + // 155/4 = ~39 + assertEquals(38, get("matrix-1")!!.shards["shard-0"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-1"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-2"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-3"]!!.size) + } + } + @Test fun `calculateShards 0`() = runBlocking { val config = configWithTestMethods(0) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index ec5f45e768..51cef23fb9 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.runBlocking import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Rule @@ -372,6 +373,10 @@ AndroidArgs additional-app-test-apks: - app: $appApkAbsolutePath test: $testErrorApkAbsolutePath + max-test-shards: 7 + client-details: + key1: value1 + key2: value2 run-timeout: 20m legacy-junit-result: false ignore-failed-tests: true @@ -1181,12 +1186,37 @@ AndroidArgs test: $testErrorApk """ assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), + AndroidArgs.load(yaml).validate().additionalAppTestApks + ) + + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, maxTestShards = 1)), + AndroidArgs.load(yaml, cli).validate().additionalAppTestApks + ) + } + + @Test + fun `cli additional-app-test-apks with max-test-shards override`() { + val cli = AndroidRunCommand() + CommandLine(cli).parseArgs("--additional-app-test-apks=app=$appApk,test=$testFlakyApk,max-test-shards=4") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + flank: + additional-app-test-apks: + - app: $appApk + test: $testErrorApk + """ + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), AndroidArgs.load(yaml).validate().additionalAppTestApks ) assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath)), + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, maxTestShards = 4)), AndroidArgs.load(yaml, cli).validate().additionalAppTestApks ) } @@ -1290,6 +1320,116 @@ AndroidArgs assertEquals(4, chunks.size) } + @Test + fun `additional-app-test-apks inherit top level client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!["top-key1"] != null } + ) + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + + @Test + fun `additional-app-test-apks can override top-level client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + client-details: + top-key1: overridden-top-val1 + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!.containsKey("top-key1") } + ) + assertNotNull( + "Matrix did not override `top-key1` client-detail", + matrixMap.map.toList().firstOrNull { + it.second.clientDetails!!["top-key1"] == "overridden-top-val1" + } + ) + + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + + @Test + fun `additional-app-test-apks should pick up client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + client-details: + key1: val1 + key2: val2 + top-key1: overwritten-top-val1 + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!.containsKey("top-key1") } + ) + matrixMap.map + .toList().apply { + // test the module which overrides and adds client details + first { it.second.clientDetails!!.size == 3 } + .apply { + assertEquals("val1", second.clientDetails!!["key1"]) + assertEquals("val2", second.clientDetails!!["key2"]) + assertEquals("overwritten-top-val1", second.clientDetails!!["top-key1"]) + } + // test all other modules got top level client details + first { it.second.clientDetails!!.size == 1 } + .apply { + assertEquals("top-val1", second.clientDetails!!["top-key1"]) + } + last { it.second.clientDetails!!.size == 1 } + .apply { + assertEquals("top-val1", second.clientDetails!!["top-key1"]) + } + } + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + @Test(expected = FlankConfigurationError::class) fun `should fail on missing app apk -- yml file`() { val yaml = """ diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml index 8f8d7890d8..0704054a20 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml @@ -12,5 +12,6 @@ flank: num-test-runs: 1 additional-app-test-apks: - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + max-test-shards: 1 - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk - test: ../test_projects/android/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt index a3c3841b31..d7bedda4df 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt @@ -52,13 +52,15 @@ class CreateAndroidTestContextKtTest { app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-single-success-debug-androidTest.apk") }, shards = should { size == 1 }, - ignoredTestCases = should { size == 2 } + ignoredTestCases = should { size == 2 }, + maxTestShards = 1 ), InstrumentationTestContext( app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-multiple-flaky-debug-androidTest.apk") }, shards = should { size == 2 }, - ignoredTestCases = should { size == 4 } + ignoredTestCases = should { size == 4 }, + maxTestShards = 2 ) )