From 151b0ae3fae1256c53852b5fc0b5233e813b5727 Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Tue, 18 May 2021 12:13:55 -0700 Subject: [PATCH 1/8] implemented new configs under additional-app-tests --- .../main/kotlin/ftl/api/TestAndroidMatrix.kt | 3 +- .../main/kotlin/ftl/args/CreateAndroidArgs.kt | 16 ++++++-- .../main/kotlin/ftl/args/yml/AppTestPair.kt | 6 ++- .../ftl/config/android/AndroidFlankConfig.kt | 3 +- .../src/main/kotlin/ftl/mock/MockServer.kt | 41 ++++++++++++++++++- .../ftl/run/model/AndroidTestContext.kt | 4 +- .../ftl/run/platform/RunAndroidTests.kt | 5 ++- .../android/CreateAndroidTestContext.kt | 11 ++--- .../android/CreateAndroidTestMatrixType.kt | 7 +++- .../ftl/run/platform/android/ResolveApks.kt | 4 +- 10 files changed, 83 insertions(+), 17 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt index 2fb110ae77..f8afe5be92 100644 --- a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt @@ -47,7 +47,8 @@ object TestMatrixAndroid { val numUniformShards: Int?, val keepTestTargetsEmpty: Boolean, val environmentVariables: Map = emptyMap(), - val testTargetsForShard: ShardChunks + val testTargetsForShard: ShardChunks, + val clientDetail: Map = emptyMap(), ) : Type() data class Robo( diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index f64bea71a4..bbeb3d7732 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -32,11 +32,19 @@ fun createAndroidArgs( roboScript = gcloud.roboScript?.normalizeFilePath(), // flank - additionalAppTestApks = flank.additionalAppTestApks?.map { (app, test, env) -> + additionalAppTestApks = flank.additionalAppTestApks?.map { + // if additional-pair did not provide certain values, set as top level ones + val mergedClientDetails = mutableMapOf().apply { + // merge additionalAppTestApk's client-details with top-level client-details + putAll(commonArgs.clientDetails ?: emptyMap()) + putAll(it.clientDetails) + } AppTestPair( - app = app?.normalizeFilePath(), - test = test.normalizeFilePath(), - environmentVariables = env + app = it.app?.normalizeFilePath(), + test = it.test.normalizeFilePath(), + environmentVariables = it.environmentVariables, + maxTestShards = it.maxTestShards ?: commonArgs.maxTestShards, + clientDetails = mergedClientDetails ) } ?: emptyList(), useLegacyJUnitResult = flank::useLegacyJUnitResult.require(), diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt index 2912107d3c..0acfbf3ee3 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt @@ -8,5 +8,9 @@ data class AppTestPair( val app: String?, val test: String, @JsonProperty("environment-variables") - val environmentVariables: Map = emptyMap() + val environmentVariables: Map = emptyMap(), + @JsonProperty("max-test-shards") + val maxTestShards: Int? = null, + @JsonProperty("client-details") + var clientDetails: Map = emptyMap() ) diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt index 67c0ab2c73..0699fab719 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt @@ -35,7 +35,8 @@ data class AndroidFlankConfig @JsonIgnore constructor( additionalAppTestApks?.add( AppTestPair( app = appApk, - test = testApk + test = testApk, + maxTestShards = map["max-test-shards"]?.toInt() ) ) } diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index f26a8f87e7..5a8f2f31c6 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -1,5 +1,7 @@ package ftl.mock +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import com.google.api.services.toolresults.model.AppStartTime import com.google.api.services.toolresults.model.CPUInfo import com.google.api.services.toolresults.model.Duration @@ -27,6 +29,7 @@ import com.google.gson.GsonBuilder import com.google.gson.LongSerializationPolicy import com.google.testing.model.AndroidDevice import com.google.testing.model.AndroidDeviceCatalog +import com.google.testing.model.ClientInfo import com.google.testing.model.Environment import com.google.testing.model.GoogleCloudStorage import com.google.testing.model.IosDeviceCatalog @@ -37,6 +40,8 @@ import com.google.testing.model.TestExecution import com.google.testing.model.TestMatrix import com.google.testing.model.ToolResultsExecution import com.google.testing.model.ToolResultsStep +import ftl.analytics.objectToMap +import ftl.client.google.run.toClientInfoDetailList import ftl.config.FtlConstants import ftl.config.FtlConstants.JSON_FACTORY import ftl.log.LogbackLogger @@ -52,17 +57,21 @@ import io.ktor.application.install import io.ktor.features.ContentNegotiation import io.ktor.gson.GsonConverter import io.ktor.http.ContentType -import io.ktor.request.uri +import io.ktor.request.* import io.ktor.response.respond import io.ktor.routing.get import io.ktor.routing.post import io.ktor.routing.routing import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import java.net.BindException +import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.Files import java.nio.file.Paths import java.util.concurrent.atomic.AtomicInteger +import java.util.zip.GZIPInputStream object MockServer { @@ -180,6 +189,35 @@ object MockServer { val projectId = call.parameters["project"] val matrixId = matrixIdCounter.incrementAndGet().toString() + val requestBody = withContext(Dispatchers.IO) { + GZIPInputStream(call.receive().inputStream()).bufferedReader(UTF_8).use { + ObjectMapper().readValue>(it.readText()) + } + } + val clientName = requestBody["clientInfo"]?.objectToMap()?.get("name") as String + val allClientDetails = mutableMapOf() + requestBody["clientInfo"] + ?.objectToMap() + ?.get("clientInfoDetails")?.let { list -> + if (list is List<*>) { + list.forEach { map -> + if (map is Map<*, *>) { + map.toList().chunked(2).forEach { + if (it.size == 2) { + val k = it[0].second + val v = it[1].second + if (k is String && v is String) { + allClientDetails[k] = v + } + } + } + } + } + } + } + val clientInfo = ClientInfo() + .setName(clientName) + .setClientInfoDetails(allClientDetails.toClientInfoDetailList()) val resultStorage = ResultStorage().apply { toolResultsExecution = ToolResultsExecution() @@ -207,6 +245,7 @@ object MockServer { .setEnvironment(environment) val matrix = TestMatrix() + .setClientInfo(clientInfo) .setProjectId(projectId) .setTestMatrixId("matrix-$matrixId") .setState("FINISHED") diff --git a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt index 80eebecd5b..34aafc88ae 100644 --- a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt @@ -13,7 +13,9 @@ data class InstrumentationTestContext( val shards: List = emptyList(), val ignoredTestCases: IgnoredTestCases = emptyList(), val environmentVariables: Map = emptyMap(), - val testTargetsForShard: ShardChunks = emptyList() + val testTargetsForShard: ShardChunks = emptyList(), + val maxTestShards: Int? = null, + val clientDetail: Map = emptyMap(), ) : AndroidTestContext() data class RoboTestContext( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index c44a7e1614..9c74b96467 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -45,7 +45,10 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope allTestShardChunks += context.shards } } - .map { context -> createAndroidTestMatrixType(context) } + .map { context -> + println("context123") + createAndroidTestMatrixType(context) + } .run { executeTestMatrixAndroid(createAndroidTestConfig(args), toList()) } .takeIf { it.isNotEmpty() } ?: throw FlankGeneralError("There are no tests to run.") diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 3509ac2f15..57dd74a518 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -39,13 +39,14 @@ private suspend fun List.setupShards( ): List = coroutineScope { map { testContext -> async { + val newArgs = if (testContext is InstrumentationTestContext && testContext.maxTestShards != null) { + args.copy(commonArgs = args.commonArgs.copy(maxTestShards = testContext.maxTestShards)) + } else args when { testContext !is InstrumentationTestContext -> testContext - args.useCustomSharding -> testContext.userShards(args.customSharding) - args.useTestTargetsForShard -> - testContext.downloadApks() - .calculateDummyShards(args, testFilter) - else -> testContext.downloadApks().calculateShards(args, testFilter) + newArgs.useCustomSharding -> testContext.userShards(newArgs.customSharding) + newArgs.useTestTargetsForShard -> testContext.downloadApks().calculateDummyShards(newArgs, testFilter) + else -> testContext.downloadApks().calculateShards(newArgs, testFilter) } } }.awaitAll().dropEmptyInstrumentationTest() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt index 11503e0e6c..a6fe27b73b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt @@ -39,7 +39,12 @@ internal fun AndroidArgs.createInstrumentationConfig( testShards = testApk.shards.testCases, keepTestTargetsEmpty = disableSharding && testTargets.isEmpty(), environmentVariables = testApk.environmentVariables, - testTargetsForShard = testTargetsForShard + testTargetsForShard = testTargetsForShard, + clientDetail = testApk.clientDetail.apply { + println("client details") + println(this) + println("/client details") + } ) internal fun AndroidArgs.createRoboConfig( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index d8c6010876..f521ad5462 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -40,6 +40,8 @@ private fun AndroidArgs.additionalApksContexts() = additionalAppTestApks.map { app = appApk.asFileReference(), test = it.test.asFileReference(), environmentVariables = it.environmentVariables, - testTargetsForShard = testTargetsForShard + testTargetsForShard = testTargetsForShard, + maxTestShards = it.maxTestShards, + clientDetail = it.clientDetails, ) }.toTypedArray() From a32530456228b8989361f1b7d70e213110e6e515 Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Tue, 18 May 2021 12:25:10 -0700 Subject: [PATCH 2/8] fixed existing tests --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 4 ++-- .../ftl/fixtures/test_app_cases/flank-multiple-mixed.yml | 1 + .../run/platform/android/CreateAndroidTestContextKtTest.kt | 6 ++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 55bc7be691..24903d03cf 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -1181,12 +1181,12 @@ AndroidArgs test: $testErrorApk """ assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), AndroidArgs.load(yaml).validate().additionalAppTestApks ) assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath)), + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, maxTestShards = 1)), AndroidArgs.load(yaml, cli).validate().additionalAppTestApks ) } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml index 8f8d7890d8..0704054a20 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml @@ -12,5 +12,6 @@ flank: num-test-runs: 1 additional-app-test-apks: - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + max-test-shards: 1 - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk - test: ../test_projects/android/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt index a3c3841b31..d7bedda4df 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt @@ -52,13 +52,15 @@ class CreateAndroidTestContextKtTest { app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-single-success-debug-androidTest.apk") }, shards = should { size == 1 }, - ignoredTestCases = should { size == 2 } + ignoredTestCases = should { size == 2 }, + maxTestShards = 1 ), InstrumentationTestContext( app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-multiple-flaky-debug-androidTest.apk") }, shards = should { size == 2 }, - ignoredTestCases = should { size == 4 } + ignoredTestCases = should { size == 4 }, + maxTestShards = 2 ) ) From 36d3041e6fde19f91f9f9006de236a14e85be20a Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Tue, 18 May 2021 12:52:14 -0700 Subject: [PATCH 3/8] added new tests, fixed client-details --- .../kotlin/ftl/args/AndroidArgsFileTest.kt | 35 +++++ .../test/kotlin/ftl/args/AndroidArgsTest.kt | 138 ++++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index 30ea53ab11..8372c9d4cb 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -132,6 +132,41 @@ class AndroidArgsFileTest { assertEquals(52, get("matrix-1")!!.shards["shard-2"]!!.size) } } + @Test + fun `calculateShards additionalAppTestApks with override`() { + val test1 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_1.apk" + val test155 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_155.apk" + val config = createAndroidArgs( + defaultAndroidConfig().apply { + platform.apply { + gcloud.apply { + app = appApkLocal + test = getString(test1) + } + flank.apply { + additionalAppTestApks = mutableListOf( + AppTestPair( + app = appApkLocal, + test = getString(test155), + maxTestShards = 4 + ) + ) + } + } + common.flank.maxTestShards = 3 + } + ) + with(runBlocking { config.getAndroidMatrixShards() }) { + assertEquals(1, get("matrix-0")!!.shards.size) + assertEquals(4, get("matrix-1")!!.shards.size) + assertEquals(1, get("matrix-0")!!.shards["shard-0"]!!.size) + // 155/4 = ~39 + assertEquals(38, get("matrix-1")!!.shards["shard-0"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-1"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-2"]!!.size) + assertEquals(39, get("matrix-1")!!.shards["shard-3"]!!.size) + } + } @Test fun `calculateShards 0`() = runBlocking { diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 24903d03cf..abff866ca2 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.runBlocking import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Rule @@ -1191,6 +1192,143 @@ AndroidArgs ) } + @Test + fun `cli additional-app-test-apks with max-test-shards override`() { + val cli = AndroidRunCommand() + CommandLine(cli).parseArgs("--additional-app-test-apks=app=$appApk,test=$testFlakyApk,max-test-shards=4") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + flank: + additional-app-test-apks: + - app: $appApk + test: $testErrorApk + """ + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), + AndroidArgs.load(yaml).validate().additionalAppTestApks + ) + + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, maxTestShards = 4)), + AndroidArgs.load(yaml, cli).validate().additionalAppTestApks + ) + } + + @Test + fun `additional-app-test-apks inherit top level client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!["top-key1"] != null } + ) + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + + @Test + fun `additional-app-test-apks can override top-level client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + client-details: + top-key1: overridden-top-val1 + key22: val22 + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!.containsKey("top-key1") } + ) + + assertNotNull( + "Matrix did not override `top-key1` client-detail", + matrixMap.map.toList().firstOrNull { + it.second.clientDetails!!["top-key1"] == "overridden-top-val1" + } + ) + + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + + @Test + fun `additional-app-test-apks should pick up client-details`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + client-details: + top-key1: top-val1 + flank: + additional-app-test-apks: + - test: $testErrorApk + client-details: + key1: val1 + key2: val2 + top-key1: overwritten-top-val1 + - app: null + test: $testErrorApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } + assertTrue( + "Not all matrices have the `top-key1` client-detail", + matrixMap.map.all { it.value.clientDetails!!.containsKey("top-key1") } + ) + matrixMap.map + .toList().apply { + // test the module which overrides and adds client details + first { it.second.clientDetails!!.size == 3 } + .apply { + assertEquals("val1", second.clientDetails!!["key1"]) + assertEquals("val2", second.clientDetails!!["key2"]) + assertEquals("overwritten-top-val1", second.clientDetails!!["top-key1"]) + } + // test all other modules got top level client details + first { it.second.clientDetails!!.size == 1 } + .apply { + assertEquals("top-val1", second.clientDetails!!["top-key1"]) + } + last { it.second.clientDetails!!.size == 1 } + .apply { + assertEquals("top-val1", second.clientDetails!!["top-key1"]) + } + } + assertEquals(3, matrixMap.map.size) + assertEquals(3, chunks.size) + } + @Test fun `cli keep-file-path`() { val cli = AndroidRunCommand() From 6b81a5ce16910788c147e89a34eb923dd5cfea9d Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Tue, 18 May 2021 13:03:58 -0700 Subject: [PATCH 4/8] format fixes, cleanup --- test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt | 2 +- .../ftl/client/google/run/android/GcAndroidTestMatrix.kt | 8 +++++++- test_runner/src/main/kotlin/ftl/mock/MockServer.kt | 3 ++- .../src/main/kotlin/ftl/run/model/AndroidTestContext.kt | 2 +- .../src/main/kotlin/ftl/run/platform/RunAndroidTests.kt | 5 +---- .../run/platform/android/CreateAndroidTestMatrixType.kt | 6 +----- .../main/kotlin/ftl/run/platform/android/ResolveApks.kt | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt index f8afe5be92..fa3bc17608 100644 --- a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt @@ -48,7 +48,7 @@ object TestMatrixAndroid { val keepTestTargetsEmpty: Boolean, val environmentVariables: Map = emptyMap(), val testTargetsForShard: ShardChunks, - val clientDetail: Map = emptyMap(), + val clientDetails: Map = emptyMap(), ) : Type() data class Robo( diff --git a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt index fc2cdaf14d..9fe09c0ee1 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt @@ -58,8 +58,14 @@ private fun createAndroidTestMatrix( runIndex: Int ): Testing.Projects.TestMatrices.Create { + val clientDetails = if (testMatrixType is TestMatrixAndroid.Type.Instrumentation && testMatrixType.clientDetails.isNotEmpty()) { + ClientInfo() + .setName("Flank") + .setClientInfoDetails(testMatrixType.clientDetails.toClientInfoDetailList()) + } else config.clientInfo + val testMatrix = TestMatrix() - .setClientInfo(config.clientInfo) + .setClientInfo(clientDetails) .setTestSpecification(getTestSpecification(testMatrixType, config)) .setResultStorage(config.resultsStorage(contextIndex, runIndex)) .setEnvironmentMatrix(config.environmentMatrix) diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index 5a8f2f31c6..9484f6a3e1 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -57,7 +57,8 @@ import io.ktor.application.install import io.ktor.features.ContentNegotiation import io.ktor.gson.GsonConverter import io.ktor.http.ContentType -import io.ktor.request.* +import io.ktor.request.receive +import io.ktor.request.uri import io.ktor.response.respond import io.ktor.routing.get import io.ktor.routing.post diff --git a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt index 34aafc88ae..f1cb757ee4 100644 --- a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt @@ -15,7 +15,7 @@ data class InstrumentationTestContext( val environmentVariables: Map = emptyMap(), val testTargetsForShard: ShardChunks = emptyList(), val maxTestShards: Int? = null, - val clientDetail: Map = emptyMap(), + val clientDetails: Map = emptyMap(), ) : AndroidTestContext() data class RoboTestContext( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 9c74b96467..c44a7e1614 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -45,10 +45,7 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope allTestShardChunks += context.shards } } - .map { context -> - println("context123") - createAndroidTestMatrixType(context) - } + .map { context -> createAndroidTestMatrixType(context) } .run { executeTestMatrixAndroid(createAndroidTestConfig(args), toList()) } .takeIf { it.isNotEmpty() } ?: throw FlankGeneralError("There are no tests to run.") diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt index a6fe27b73b..064bd7dd31 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt @@ -40,11 +40,7 @@ internal fun AndroidArgs.createInstrumentationConfig( keepTestTargetsEmpty = disableSharding && testTargets.isEmpty(), environmentVariables = testApk.environmentVariables, testTargetsForShard = testTargetsForShard, - clientDetail = testApk.clientDetail.apply { - println("client details") - println(this) - println("/client details") - } + clientDetails = testApk.clientDetails ) internal fun AndroidArgs.createRoboConfig( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index f521ad5462..f5e2a61318 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -42,6 +42,6 @@ private fun AndroidArgs.additionalApksContexts() = additionalAppTestApks.map { environmentVariables = it.environmentVariables, testTargetsForShard = testTargetsForShard, maxTestShards = it.maxTestShards, - clientDetail = it.clientDetails, + clientDetails = it.clientDetails, ) }.toTypedArray() From 841138f8b1fe4c3cb8c4e426172364d14889ec68 Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Wed, 19 May 2021 07:30:47 -0700 Subject: [PATCH 5/8] improved encapsulation --- .../google/run/android/GcAndroidTestMatrix.kt | 23 +++++++++++-------- .../android/CreateAndroidTestContext.kt | 9 +++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt index 9fe09c0ee1..0098f2a2bb 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt @@ -58,11 +58,7 @@ private fun createAndroidTestMatrix( runIndex: Int ): Testing.Projects.TestMatrices.Create { - val clientDetails = if (testMatrixType is TestMatrixAndroid.Type.Instrumentation && testMatrixType.clientDetails.isNotEmpty()) { - ClientInfo() - .setName("Flank") - .setClientInfoDetails(testMatrixType.clientDetails.toClientInfoDetailList()) - } else config.clientInfo + val clientDetails = config.clientInfo(testMatrixType) val testMatrix = TestMatrix() .setClientInfo(clientDetails) @@ -77,11 +73,18 @@ private fun createAndroidTestMatrix( }.getOrElse { e -> throw FlankGeneralError(e) } } -// https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 -private val TestMatrixAndroid.Config.clientInfo - get() = ClientInfo() - .setName("Flank") - .setClientInfoDetails(clientDetails?.toClientInfoDetailList()) +fun TestMatrixAndroid.Config.clientInfo(matrix: TestMatrixAndroid.Type): ClientInfo { + return if (matrix is TestMatrixAndroid.Type.Instrumentation && matrix.clientDetails.isNotEmpty()) { + ClientInfo() + .setName("Flank") + .setClientInfoDetails(matrix.clientDetails.toClientInfoDetailList()) + } else { + // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 + ClientInfo() + .setName("Flank") + .setClientInfoDetails(clientDetails?.toClientInfoDetailList()) + } +} private val TestMatrixAndroid.Config.environmentMatrix get() = EnvironmentMatrix() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 57dd74a518..5277981191 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -39,9 +39,7 @@ private suspend fun List.setupShards( ): List = coroutineScope { map { testContext -> async { - val newArgs = if (testContext is InstrumentationTestContext && testContext.maxTestShards != null) { - args.copy(commonArgs = args.commonArgs.copy(maxTestShards = testContext.maxTestShards)) - } else args + val newArgs = args.prepareArgsForSharding(testContext) when { testContext !is InstrumentationTestContext -> testContext newArgs.useCustomSharding -> testContext.userShards(newArgs.customSharding) @@ -51,6 +49,11 @@ private suspend fun List.setupShards( } }.awaitAll().dropEmptyInstrumentationTest() } +fun AndroidArgs.prepareArgsForSharding(context: AndroidTestContext): AndroidArgs { + return if (context is InstrumentationTestContext && context.maxTestShards != null) { + copy(commonArgs = commonArgs.copy(maxTestShards = context.maxTestShards)) + } else this +} private fun InstrumentationTestContext.userShards(customShardingMap: Map) = customShardingMap .values From e8fd9b0548b1fca7dccec97e030fea70942c79a9 Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Wed, 19 May 2021 07:57:22 -0700 Subject: [PATCH 6/8] improved client details parsing in MockServer --- .../src/main/kotlin/ftl/mock/MockServer.kt | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index 9484f6a3e1..be47e10096 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -195,30 +195,7 @@ object MockServer { ObjectMapper().readValue>(it.readText()) } } - val clientName = requestBody["clientInfo"]?.objectToMap()?.get("name") as String - val allClientDetails = mutableMapOf() - requestBody["clientInfo"] - ?.objectToMap() - ?.get("clientInfoDetails")?.let { list -> - if (list is List<*>) { - list.forEach { map -> - if (map is Map<*, *>) { - map.toList().chunked(2).forEach { - if (it.size == 2) { - val k = it[0].second - val v = it[1].second - if (k is String && v is String) { - allClientDetails[k] = v - } - } - } - } - } - } - } - val clientInfo = ClientInfo() - .setName(clientName) - .setClientInfoDetails(allClientDetails.toClientInfoDetailList()) + val clientInfo = parseClientDetails(requestBody) val resultStorage = ResultStorage().apply { toolResultsExecution = ToolResultsExecution() @@ -406,6 +383,29 @@ object MockServer { } } + private fun parseClientDetails(requestBody: Map): ClientInfo { + val clientName = requestBody["clientInfo"]?.objectToMap()?.get("name") as String + val allClientDetails = mutableMapOf() + requestBody["clientInfo"] + ?.objectToMap() + ?.get("clientInfoDetails") + ?.let { list -> + if (list !is List<*>) + return@let + list.filterIsInstance>() + .forEach { + val k = it["key"] + val v = it["value"] + if (k != null && v != null) { + allClientDetails[k] = v + } + } + } + return ClientInfo() + .setName(clientName) + .setClientInfoDetails(allClientDetails.toClientInfoDetailList()) + } + fun start() { if (isStarted) return val loggingEnabled = LogbackLogger.Root.isEnabled From 9fd6b9d5c096c53a1a8e9094f78f0bd802d8416f Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Wed, 19 May 2021 11:15:24 -0400 Subject: [PATCH 7/8] Update test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt Co-authored-by: piotradamczyk5 <65554637+piotradamczyk5@users.noreply.github.com> --- .../kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 5277981191..7ab65047c2 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -49,7 +49,7 @@ private suspend fun List.setupShards( } }.awaitAll().dropEmptyInstrumentationTest() } -fun AndroidArgs.prepareArgsForSharding(context: AndroidTestContext): AndroidArgs { +private fun AndroidArgs.prepareArgsForSharding(context: AndroidTestContext): AndroidArgs { return if (context is InstrumentationTestContext && context.maxTestShards != null) { copy(commonArgs = commonArgs.copy(maxTestShards = context.maxTestShards)) } else this From 09aaac34ab8c203a1b74eecba8656cc1f4fdde1e Mon Sep 17 00:00:00 2001 From: Asad Salman Date: Wed, 19 May 2021 08:33:08 -0700 Subject: [PATCH 8/8] clean up --- test_runner/src/main/kotlin/ftl/mock/MockServer.kt | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index be47e10096..72d48c2877 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -389,18 +389,14 @@ object MockServer { requestBody["clientInfo"] ?.objectToMap() ?.get("clientInfoDetails") + .run { this as? List<*> } ?.let { list -> - if (list !is List<*>) - return@let list.filterIsInstance>() - .forEach { - val k = it["key"] - val v = it["value"] - if (k != null && v != null) { - allClientDetails[k] = v - } - } + .filter { it.containsKey("key") && it.containsKey("value") } + .map { it.getValue("key") to it.getValue("value") } + .forEach { (key, value) -> allClientDetails[key] = value } } + return ClientInfo() .setName(clientName) .setClientInfoDetails(allClientDetails.toClientInfoDetailList())