diff --git a/docs/index.md b/docs/index.md index a7845562e9..fc9a31462f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -661,13 +661,19 @@ flank: ## Default: false # keep-file-path: false - ### Additional App/Test APKS + ### Additional App/Test APKS ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path! ## If app is omitted, then the top level app is used for that pair. + ## You can overwrite global config per each test pair. + ## Currently supported options are: max-test-shards, test-targets, client-details, environment-variables, device # additional-app-test-apks: # - app: ../test_projects/android/apks/app-debug.apk # test: ../test_projects/android/apks/app1-debug-androidTest.apk + # device: + # - model: Nexus6P + # version: 27 # - test: ../test_projects/android/apks/app2-debug-androidTest.apk + # max-test-shards: 5 ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index d2b2323213..2965f6d9d8 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -3,6 +3,7 @@ package integration import FlankCommand import com.google.common.truth.Truth.assertThat import integration.config.AndroidTest +import org.junit.Assert.assertEquals import org.junit.Test import org.junit.experimental.categories.Category import run @@ -19,9 +20,9 @@ import utils.assertTestResultContainsWebLinks import utils.findTestDirectoryFromOutput import utils.json import utils.loadAsTestSuite -import utils.multipleFailedTests import utils.multipleSuccessfulTests import utils.removeUnicode +import utils.testResults.TestSuite import utils.toJUnitXmlFile import utils.toOutputReportFile @@ -48,10 +49,20 @@ class MultipleApksIT { "MainActivity_robo_script.json" ) - resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { - assertTestResultContainsWebLinks() - assertTestPass(multipleSuccessfulTests) - assertTestFail(multipleFailedTests) + val xmlResult = resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite() + + xmlResult.assertTestResultContainsWebLinks() + xmlResult.assertTestPass(multipleSuccessfulTests) + xmlResult.assertTestFail(listOf("test2")) + + xmlResult.testSuites.groupBy { it.name }.mapValues { it.value.flatMap(TestSuite::testCases) }.run { + assertEquals(20, get("NexusLowRes-28-en-portrait")?.size) + assertEquals(1, get("Pixel2-28-en-portrait")?.size) + assertEquals("com.example.test_app.InstrumentedTest", get("Pixel2-28-en-portrait")?.get(0)?.classname) + assertEquals("test2", get("Pixel2-28-en-portrait")?.get(0)?.name) + assertEquals(1, get("Nexus6P-27-en-portrait")?.size) + assertEquals("com.example.test_app.InstrumentedTest", get("Nexus6P-27-en-portrait")?.get(0)?.classname) + assertEquals("test", get("Nexus6P-27-en-portrait")?.get(0)?.name) } val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport() @@ -69,7 +80,7 @@ class MultipleApksIT { .map { it.testAxises } .flatten() - assertThat(testsResults.sumOf { it.suiteOverview.failures }).isEqualTo(5) - assertThat(testsResults.sumOf { it.suiteOverview.total }).isEqualTo(41) + assertThat(testsResults.sumOf { it.suiteOverview.failures }).isEqualTo(1) + assertThat(testsResults.sumOf { it.suiteOverview.total }).isEqualTo(22) } } diff --git a/integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml b/integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml index d53ee76990..b988e5b073 100644 --- a/integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml +++ b/integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml @@ -9,7 +9,16 @@ flank: output-style: single additional-app-test-apks: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + max-test-shards: 2 - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk + device: + - model: Pixel2 + version: 28 + test-targets: + - class com.example.test_app.InstrumentedTest#test2 - test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk + device: + - model: Nexus6P + version: 27 disable-usage-statistics: true output-report: json diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 27cac1a836..80c3899b83 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -295,10 +295,16 @@ flank: ### Additional App/Test APKS ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path! ## If app is omitted, then the top level app is used for that pair. + ## You can overwrite global config per each test pair. + ## Currently supported options are: max-test-shards, test-targets, client-details, environment-variables, device # additional-app-test-apks: # - app: ../test_projects/android/apks/app-debug.apk # test: ../test_projects/android/apks/app1-debug-androidTest.apk + # device: + # - model: Nexus6P + # version: 27 # - test: ../test_projects/android/apks/app2-debug-androidTest.apk + # max-test-shards: 5 ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). diff --git a/test_runner/src/main/kotlin/ftl/adapter/GoogleTestMatrixAndroid.kt b/test_runner/src/main/kotlin/ftl/adapter/GoogleTestMatrixAndroid.kt index 5320ae2feb..a6cd1f7be3 100644 --- a/test_runner/src/main/kotlin/ftl/adapter/GoogleTestMatrixAndroid.kt +++ b/test_runner/src/main/kotlin/ftl/adapter/GoogleTestMatrixAndroid.kt @@ -9,8 +9,8 @@ import com.google.testing.model.TestMatrix as GoogleTestMatrix object GoogleTestMatrixAndroid : TestMatrixAndroid.Execute, - (TestMatrixAndroid.Config, List) -> List by { config, types -> + (List) -> List by { configTypePairs -> runBlocking { - executeAndroidTests(config, types).map(GoogleTestMatrix::toApiModel) + executeAndroidTests(configTypePairs).map(GoogleTestMatrix::toApiModel) } } diff --git a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt index fa3bc17608..d3aa293e1e 100644 --- a/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/api/TestAndroidMatrix.kt @@ -65,7 +65,12 @@ object TestMatrixAndroid { ) : Type() } - interface Execute : (Config, List) -> List + data class TestSetup( + val config: Config, + val type: Type + ) + + interface Execute : (List) -> List } typealias ShardChunks = List> diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt index 4dc70e67be..8e58fe9a55 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt @@ -3,35 +3,62 @@ package ftl.args import ftl.args.yml.AppTestPair private const val NEW_LINE = '\n' +private const val SPACESx8 = " " +private const val SPACESx10 = " " object ArgsToString { - fun mapToString(map: Map?): String { + fun mapToString( + map: Map?, + transform: (Map.Entry) -> String = { (key, value) -> "$SPACESx8$key: $value" } + ): String { if (map.isNullOrEmpty()) return "" - return NEW_LINE + map.map { (key, value) -> " $key: $value" } + return NEW_LINE + map.map(transform) .joinToString(System.lineSeparator()) } - fun listToString(list: List?): String { + fun listToString(list: List?, transform: (String) -> String = { "$SPACESx8- $it" }): String { if (list.isNullOrEmpty()) return "" return NEW_LINE + list.filterNotNull() - .joinToString(System.lineSeparator()) { dir -> " - $dir" } + .joinToString(System.lineSeparator(), transform = transform) } - fun objectsToString(objects: List?): String { + fun objectsToString(objects: List?, transform: (Any) -> String = { "$it" }): String { if (objects.isNullOrEmpty()) return "" return NEW_LINE + objects.filterNotNull() - .joinToString(System.lineSeparator()) { "$it" } + .joinToString(System.lineSeparator(), transform = transform) } fun listOfListToString(listOfList: List>?): String { if (listOfList.isNullOrEmpty()) return "" return NEW_LINE + listOfList.map { list -> list.joinToString(",") { " $it" } } - .joinToString(System.lineSeparator()) { " - $it" } + .joinToString(System.lineSeparator()) { "$SPACESx8- $it" } } fun apksToString(devices: List): String { if (devices.isNullOrEmpty()) return "" - return NEW_LINE + devices.joinToString(System.lineSeparator()) { (app, test) -> " - app: $app\n test: $test" } + return NEW_LINE + devices.joinToString(System.lineSeparator()) { pair -> + buildString { + if (pair.app == null) appendLine("$SPACESx8- test: ${pair.test}") + else { + appendLine("$SPACESx8- app: ${pair.app}") + appendLine("$SPACESx8 test: ${pair.test}") + } + pair.maxTestShards?.let { appendLine("$SPACESx8 max-test-shards: $it") } + pair.clientDetails?.let { + append("$SPACESx8 client-details:${mapToString(it) { (key, value) -> "$SPACESx10 $key: $value" }}") + } + pair.testTargets?.let { list -> + appendLine("$SPACESx8 test-targets:${listToString(list) { "$SPACESx10- $it" }}") + } + pair.devices?.let { list -> + appendLine("$SPACESx8 device:") + list + .map { " $it" } + .map { it.replace("\n", "\n ") } + .forEach { append(it) } + } + }.trimEnd() + } } } diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index bbeb3d7732..7ce43699b4 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -1,6 +1,5 @@ package ftl.args -import ftl.args.yml.AppTestPair import ftl.config.AndroidConfig import ftl.config.android.AndroidFlankConfig import ftl.config.android.AndroidGcloudConfig @@ -33,18 +32,9 @@ fun createAndroidArgs( // flank additionalAppTestApks = flank.additionalAppTestApks?.map { - // if additional-pair did not provide certain values, set as top level ones - val mergedClientDetails = mutableMapOf().apply { - // merge additionalAppTestApk's client-details with top-level client-details - putAll(commonArgs.clientDetails ?: emptyMap()) - putAll(it.clientDetails) - } - AppTestPair( + it.copy( app = it.app?.normalizeFilePath(), test = it.test.normalizeFilePath(), - environmentVariables = it.environmentVariables, - maxTestShards = it.maxTestShards ?: commonArgs.maxTestShards, - clientDetails = mergedClientDetails ) } ?: emptyList(), useLegacyJUnitResult = flank::useLegacyJUnitResult.require(), diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index b9b0897907..bd3ddc6f33 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -200,6 +200,22 @@ private fun AndroidArgs.assertAdditionalAppTestApks() { .filter { (app, _) -> app == null } .map { File(it.test).name } .run { if (isNotEmpty()) throw FlankConfigurationError("Cannot resolve app apk pair for $this") } + + additionalAppTestApks + .map { + copy( + appApk = it.app ?: appApk, + testApk = it.test, + commonArgs = commonArgs.copy( + maxTestShards = it.maxTestShards ?: maxTestShards, + devices = it.devices ?: devices + ), + additionalAppTestApks = emptyList() + ) + } + .forEach { + it.validate() + } } private fun AndroidArgs.assertApkFilePaths() { diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt index 0acfbf3ee3..541d9a21c1 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt @@ -2,6 +2,7 @@ package ftl.args.yml import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import ftl.config.Device @JsonIgnoreProperties(ignoreUnknown = true) data class AppTestPair( @@ -12,5 +13,9 @@ data class AppTestPair( @JsonProperty("max-test-shards") val maxTestShards: Int? = null, @JsonProperty("client-details") - var clientDetails: Map = emptyMap() + val clientDetails: Map? = null, + @JsonProperty("test-targets") + val testTargets: List? = null, + @JsonProperty("device") + val devices: List? = null ) diff --git a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt index 0098f2a2bb..fd74282151 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt @@ -32,11 +32,10 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope suspend fun executeAndroidTests( - config: TestMatrixAndroid.Config, - testMatrixTypes: List, -): List = testMatrixTypes - .foldIndexed(emptyList>()) { testMatrixTypeIndex, testMatrices, testMatrixType -> - testMatrices + executeAndroidTestMatrix(testMatrixType, testMatrixTypeIndex, config) + testSetups: List +): List = testSetups + .foldIndexed(emptyList>()) { testMatrixTypeIndex, testMatrices, setUp -> + testMatrices + executeAndroidTestMatrix(setUp.type, testMatrixTypeIndex, setUp.config) }.awaitAll() private suspend fun executeAndroidTestMatrix( @@ -58,10 +57,8 @@ private fun createAndroidTestMatrix( runIndex: Int ): Testing.Projects.TestMatrices.Create { - val clientDetails = config.clientInfo(testMatrixType) - val testMatrix = TestMatrix() - .setClientInfo(clientDetails) + .setClientInfo(config.clientInfo) .setTestSpecification(getTestSpecification(testMatrixType, config)) .setResultStorage(config.resultsStorage(contextIndex, runIndex)) .setEnvironmentMatrix(config.environmentMatrix) @@ -73,18 +70,11 @@ private fun createAndroidTestMatrix( }.getOrElse { e -> throw FlankGeneralError(e) } } -fun TestMatrixAndroid.Config.clientInfo(matrix: TestMatrixAndroid.Type): ClientInfo { - return if (matrix is TestMatrixAndroid.Type.Instrumentation && matrix.clientDetails.isNotEmpty()) { - ClientInfo() - .setName("Flank") - .setClientInfoDetails(matrix.clientDetails.toClientInfoDetailList()) - } else { - // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 - ClientInfo() - .setName("Flank") - .setClientInfoDetails(clientDetails?.toClientInfoDetailList()) - } -} +// https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 +private val TestMatrixAndroid.Config.clientInfo + get() = ClientInfo() + .setName("Flank") + .setClientInfoDetails(clientDetails?.toClientInfoDetailList()) private val TestMatrixAndroid.Config.environmentMatrix get() = EnvironmentMatrix() diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt index 0699fab719..c4e82cd8b0 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt @@ -20,8 +20,10 @@ data class AndroidFlankConfig @JsonIgnore constructor( names = ["--additional-app-test-apks"], split = ",", description = [ - "A list of app & test apks to include in the run. " + - "Useful for running multiple module tests within a single Flank run." + "A list of app & test apks to include in the run. Useful for running multiple module tests " + + "within a single Flank run.", + "You can overwrite global config per each test pair. Currently supported options are: " + + "max-test-shards, test-targets, client-details, environment-variables, device" ] ) fun additionalAppTestApks(map: Map?) { @@ -34,9 +36,10 @@ data class AndroidFlankConfig @JsonIgnore constructor( if (testApk != null) { additionalAppTestApks?.add( AppTestPair( - app = appApk, - test = testApk, - maxTestShards = map["max-test-shards"]?.toInt() + app = appApk.toString(), + test = testApk.toString(), + maxTestShards = map["max-test-shards"]?.toInt(), + testTargets = map["test-targets"]?.split(",") ) ) } diff --git a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt index f1cb757ee4..c5bbf269c6 100644 --- a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt @@ -2,10 +2,11 @@ package ftl.run.model import ftl.api.FileReference import ftl.api.ShardChunks +import ftl.args.AndroidArgs import ftl.args.IgnoredTestCases import ftl.shard.Chunk -sealed class AndroidTestContext +sealed class AndroidTestContext(open val args: AndroidArgs) data class InstrumentationTestContext( val app: FileReference, @@ -14,21 +15,23 @@ data class InstrumentationTestContext( val ignoredTestCases: IgnoredTestCases = emptyList(), val environmentVariables: Map = emptyMap(), val testTargetsForShard: ShardChunks = emptyList(), - val maxTestShards: Int? = null, - val clientDetails: Map = emptyMap(), -) : AndroidTestContext() + override val args: AndroidArgs +) : AndroidTestContext(args) data class RoboTestContext( val app: FileReference, - val roboScript: FileReference -) : AndroidTestContext() + val roboScript: FileReference, + override val args: AndroidArgs +) : AndroidTestContext(args) data class GameLoopContext( val app: FileReference, val scenarioLabels: List, val scenarioNumbers: List, -) : AndroidTestContext() + override val args: AndroidArgs +) : AndroidTestContext(args) data class SanityRoboTestContext( - val app: FileReference -) : AndroidTestContext() + val app: FileReference, + override val args: AndroidArgs +) : AndroidTestContext(args) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index a1074bec22..f1ee25d67d 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -3,6 +3,7 @@ package ftl.run.platform import flank.common.join import flank.common.logLn import ftl.api.RemoteStorage +import ftl.api.TestMatrixAndroid import ftl.api.executeTestMatrixAndroid import ftl.api.uploadToRemoteStorage import ftl.args.AndroidArgs @@ -45,8 +46,8 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope allTestShardChunks += context.shards } } - .map { context -> createAndroidTestMatrixType(context) } - .run { executeTestMatrixAndroid(createAndroidTestConfig(args), toList()) } + .map { createTestSetup(it) } + .run { executeTestMatrixAndroid(this) } .takeIf { it.isNotEmpty() } ?: throw FlankGeneralError("There are no Android tests to run.") @@ -83,3 +84,8 @@ private fun AndroidMatrixTestShards.saveShards(config: AndroidArgs) = saveShardC size = size, obfuscatedOutput = config.obfuscateDumpShards ) + +private suspend fun createTestSetup(context: AndroidTestContext) = TestMatrixAndroid.TestSetup( + config = createAndroidTestConfig(context.args), + type = context.args.createAndroidTestMatrixType(context) +) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 57fd79ea82..b0b9e6d04b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -31,31 +31,24 @@ import kotlinx.coroutines.coroutineScope import java.io.File import ftl.shard.TestMethod as ShardTestMethod -suspend fun AndroidArgs.createAndroidTestContexts(): List = resolveApks().setupShards(this) +suspend fun AndroidArgs.createAndroidTestContexts(): List = resolveApks().setupShards() private val customTestAnnotations = listOf("org.junit.experimental.theories.Theory") -private suspend fun List.setupShards( - args: AndroidArgs, - testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets, args.testTargetsForShard) -): List = coroutineScope { +private suspend fun List.setupShards(): List = coroutineScope { map { testContext -> async { - val newArgs = args.prepareArgsForSharding(testContext) + val newArgs = testContext.args + val filters = TestFilters.fromTestTargets(newArgs.testTargets, newArgs.testTargetsForShard) when { testContext !is InstrumentationTestContext -> testContext newArgs.useCustomSharding -> testContext.userShards(newArgs.customSharding) - newArgs.useTestTargetsForShard -> testContext.downloadApks().calculateDummyShards(newArgs, testFilter) - else -> testContext.downloadApks().calculateShards(newArgs, testFilter) + newArgs.useTestTargetsForShard -> testContext.downloadApks().calculateDummyShards(newArgs, filters) + else -> testContext.downloadApks().calculateShards(newArgs, filters) } } }.awaitAll().dropEmptyInstrumentationTest() } -private fun AndroidArgs.prepareArgsForSharding(context: AndroidTestContext): AndroidArgs { - return if (context is InstrumentationTestContext && context.maxTestShards != null) { - copy(commonArgs = commonArgs.copy(maxTestShards = context.maxTestShards)) - } else this -} private fun InstrumentationTestContext.userShards(customShardingMap: Map) = customShardingMap .values diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt index 064bd7dd31..fd317eb1af 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestMatrixType.kt @@ -40,7 +40,7 @@ internal fun AndroidArgs.createInstrumentationConfig( keepTestTargetsEmpty = disableSharding && testTargets.isEmpty(), environmentVariables = testApk.environmentVariables, testTargetsForShard = testTargetsForShard, - clientDetails = testApk.clientDetails + clientDetails = clientDetails.orEmpty() ) internal fun AndroidArgs.createRoboConfig( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index f5e2a61318..5f225cc398 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -24,11 +24,12 @@ private fun AndroidArgs.mainApkContext() = appApk?.let { appApk -> app = appApk.asFileReference(), test = testApk.asFileReference(), environmentVariables = emptyMap(), - testTargetsForShard = testTargetsForShard + testTargetsForShard = testTargetsForShard, + args = this ) - roboScript != null -> RoboTestContext(app = appApk.asFileReference(), roboScript = roboScript.asFileReference()) - isSanityRobo -> SanityRoboTestContext(app = appApk.asFileReference()) - isGameLoop -> GameLoopContext(appApk.asFileReference(), scenarioLabels, scenarioNumbers) + roboScript != null -> RoboTestContext(app = appApk.asFileReference(), roboScript = roboScript.asFileReference(), this) + isSanityRobo -> SanityRoboTestContext(app = appApk.asFileReference(), this) + isGameLoop -> GameLoopContext(appApk.asFileReference(), scenarioLabels, scenarioNumbers, this) else -> null } } @@ -41,7 +42,13 @@ private fun AndroidArgs.additionalApksContexts() = additionalAppTestApks.map { test = it.test.asFileReference(), environmentVariables = it.environmentVariables, testTargetsForShard = testTargetsForShard, - maxTestShards = it.maxTestShards, - clientDetails = it.clientDetails, + args = copy( + commonArgs = commonArgs.copy( + maxTestShards = it.maxTestShards ?: maxTestShards, + devices = it.devices ?: devices, + clientDetails = it.clientDetails ?: clientDetails + ), + testTargets = it.testTargets ?: testTargets + ) ) }.toTypedArray() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt index 734efc5c71..e8b2ab67eb 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt @@ -45,7 +45,7 @@ private fun GameLoopContext.upload(rootGcsBucket: String, runGcsPath: String) = ) private fun SanityRoboTestContext.upload(rootGcsBucket: String, runGcsPath: String) = - SanityRoboTestContext(app.uploadIfNeeded(rootGcsBucket, runGcsPath)) + SanityRoboTestContext(app.uploadIfNeeded(rootGcsBucket, runGcsPath), args) suspend fun AndroidArgs.uploadAdditionalApks() = additionalApks.uploadToGcloudIfNeeded(resultsDir, resultsBucket) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index ba6820f0d8..e487827a2b 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -143,6 +143,12 @@ class AndroidArgsTest { additional-app-test-apks: - app: $appApk test: $testErrorApk + max-test-shards: 123 + test-targets: + - class any.additional.TestClass#test1 + device: + - model: Nexus9 + version: 23 run-timeout: 20m ignore-failed-tests: true output-style: single @@ -374,6 +380,14 @@ AndroidArgs additional-app-test-apks: - app: $appApkAbsolutePath test: $testErrorApkAbsolutePath + max-test-shards: 123 + test-targets: + - class any.additional.TestClass#test1 + device: + - model: Nexus9 + version: 23 + locale: en + orientation: portrait run-timeout: 20m legacy-junit-result: false ignore-failed-tests: true @@ -1183,12 +1197,12 @@ AndroidArgs test: $testErrorApk """ assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), AndroidArgs.load(yaml).validate().additionalAppTestApks ) assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, maxTestShards = 1)), + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath)), AndroidArgs.load(yaml, cli).validate().additionalAppTestApks ) } @@ -1208,7 +1222,7 @@ AndroidArgs test: $testErrorApk """ assertEquals( - listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath, maxTestShards = 1)), + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), AndroidArgs.load(yaml).validate().additionalAppTestApks ) @@ -1218,6 +1232,31 @@ AndroidArgs ) } + @Test + fun `cli additional-app-test-apks with test-targets override`() { + val cli = AndroidRunCommand() + CommandLine(cli).parseArgs("--additional-app-test-apks=app=$appApk,test=$testFlakyApk,test-targets=class any.class.TestClass") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + flank: + additional-app-test-apks: + - app: $appApk + test: $testErrorApk + """ + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), + AndroidArgs.load(yaml).validate().additionalAppTestApks + ) + + assertEquals( + listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath, testTargets = listOf("class any.class.TestClass"))), + AndroidArgs.load(yaml, cli).validate().additionalAppTestApks + ) + } + @Test fun `additional-app-test-apks inherit top level client-details`() { val yaml = """ @@ -1675,7 +1714,8 @@ AndroidArgs shards = listOf( Chunk(listOf(TestMethod(name = "test", time = 0.0))), Chunk(listOf(TestMethod(name = "test", time = 0.0))) - ) + ), + args = args ) ) val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig) @@ -1702,7 +1742,8 @@ AndroidArgs shards = listOf( Chunk(listOf(TestMethod(name = "test", time = 0.0))), Chunk(listOf(TestMethod(name = "test", time = 0.0))) - ) + ), + args = args ) ) val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig) @@ -2412,7 +2453,8 @@ AndroidArgs GameLoopContext( app = "app".asFileReference(), scenarioNumbers = args.scenarioNumbers, - scenarioLabels = args.scenarioLabels + scenarioLabels = args.scenarioLabels, + args = args ) ) val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig) diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml index 0704054a20..026b98f01e 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml @@ -14,4 +14,6 @@ flank: - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk max-test-shards: 1 - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + test-targets: + - class com.example.test_app.InstrumentedTest - test: ../test_projects/android/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index d3518e4c70..d588cbff4a 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -1,6 +1,7 @@ package ftl.gc import com.google.testing.model.TestSetup +import ftl.api.TestMatrixAndroid import ftl.api.TestMatrixAndroid.Type import ftl.args.AndroidArgs import ftl.client.google.run.android.executeAndroidTests @@ -49,7 +50,7 @@ class GcAndroidTestMatrixTest { ) val config = createAndroidTestConfig(androidArgs) - executeAndroidTests(config, listOf(type)) + executeAndroidTests(listOf(TestMatrixAndroid.TestSetup(config, type))) } } @@ -78,7 +79,7 @@ class GcAndroidTestMatrixTest { val config = createAndroidTestConfig(androidArgs) - executeAndroidTests(config, listOf(type)) + executeAndroidTests(listOf(TestMatrixAndroid.TestSetup(config, type))) } } @@ -112,7 +113,7 @@ class GcAndroidTestMatrixTest { val config = createAndroidTestConfig(androidArgs) - executeAndroidTests(config, listOf(type)) + executeAndroidTests(listOf(TestMatrixAndroid.TestSetup(config, type))) } } diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index 10b494d14b..5f9b107673 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -55,24 +55,16 @@ class DumpShardsKtTest { "test": "$path/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", "shards": { "shard-0": [ - "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", - "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized", - "class com.example.test_app.InstrumentedTest#test1", - "class com.example.test_app.InstrumentedTest#test2" + "class com.example.test_app.InstrumentedTest#test0" ], "shard-1": [ - "class com.example.test_app.ParameterizedTest", - "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner", - "class com.example.test_app.InstrumentedTest#test0", - "class com.example.test_app.bar.BarInstrumentedTest#testBar", - "class com.example.test_app.foo.FooInstrumentedTest#testFoo" + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" ] }, "junit-ignored": [ "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", - "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", - "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", - "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore" ] } } @@ -95,8 +87,8 @@ class DumpShardsKtTest { val notExpected = """ { "matrix-0": { - "app": "/pathToApk/app-debug.apk", - "test": "/pathToApk/app-single-success-debug-androidTest.apk", + "app": "pathToApk/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test": "pathToApk/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", "shards": { "shard-0": [ "class com.example.test_app.InstrumentedTest#test" @@ -108,28 +100,20 @@ class DumpShardsKtTest { ] }, "matrix-1": { - "app": "/pathToApk/app-debug.apk", - "test": "/pathToApk/app-multiple-flaky-debug-androidTest.apk", + "app": "pathToApk/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test": "pathToApk/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", "shards": { "shard-0": [ - "class com.example.test_app.InstrumentedTest#test1", - "class com.example.test_app.InstrumentedTest#test2", - "class com.example.test_app.ParameterizedTest", - "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed" + "class com.example.test_app.InstrumentedTest#test0" ], "shard-1": [ - "class com.example.test_app.InstrumentedTest#test0", - "class com.example.test_app.bar.BarInstrumentedTest#testBar", - "class com.example.test_app.foo.FooInstrumentedTest#testFoo", - "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized", - "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner" + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" ] }, "junit-ignored": [ - "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", - "class com.example.test_app.InstrumentedTest#ignoredTestWithSuppress", - "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", - "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore" ] } } diff --git a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt index a87cb5bd55..9f0d8a017b 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt @@ -22,10 +22,10 @@ class RunAndroidTestsKtTest { should { map.size == 3 }, listOf( should { size == 1 }, - should { size == 4 }, - should { size == 5 } + should { size == 1 }, + should { size == 2 } ), - should { size == 6 } + should { size == 4 } ) // when diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt index 9e0ec8dc55..cd04a3a614 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt @@ -37,6 +37,8 @@ class CreateAndroidTestContextKtTest { @get:Rule val root = TemporaryFolder() + private val args: AndroidArgs = AndroidArgs.load(mixedConfigYaml) + @After fun tearDown() = unmockkAll() @@ -46,38 +48,40 @@ class CreateAndroidTestContextKtTest { val expected = listOf( RoboTestContext( app = should { local.endsWith("app-debug.apk") }, - roboScript = should { local.endsWith("MainActivity_robo_script.json") } + roboScript = should { local.endsWith("MainActivity_robo_script.json") }, + args = should { maxTestShards == 2 } ), InstrumentationTestContext( app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-single-success-debug-androidTest.apk") }, shards = should { size == 1 }, ignoredTestCases = should { size == 2 }, - maxTestShards = 1 + args = should { maxTestShards == 1 } ), InstrumentationTestContext( app = should { local.endsWith("app-debug.apk") }, test = should { local.endsWith("app-multiple-flaky-debug-androidTest.apk") }, shards = should { size == 2 }, - ignoredTestCases = should { size == 4 }, - maxTestShards = 2 + ignoredTestCases = should { size == 2 }, + args = should { testTargets == listOf("class com.example.test_app.InstrumentedTest") } ) ) // when val actual: List = runBlocking { - AndroidArgs.load(mixedConfigYaml).createAndroidTestContexts() + args.createAndroidTestContexts() } // then - assertEquals(expected, actual) + actual.forEachIndexed { index, androidTestContext -> assertEquals(expected[index], androidTestContext) } } @Test fun `should pick up @Theory tests`() { val testInstrumentationContext = InstrumentationTestContext( FileReference("", ""), - FileReference("../test_projects/android/apks/app-theory-androidTest.apk", "") + FileReference("../test_projects/android/apks/app-theory-androidTest.apk", ""), + args = args ) val parsedTests = testInstrumentationContext @@ -95,7 +99,8 @@ class CreateAndroidTestContextKtTest { // given val testInstrumentationContext = InstrumentationTestContext( FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", ""), - FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", "") + FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", ""), + args = args ) var actual = 0 @@ -123,7 +128,8 @@ class CreateAndroidTestContextKtTest { fun `should not append all parameterized classes to list of test methods`() { val testInstrumentationContext = InstrumentationTestContext( FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", ""), - FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", "") + FileReference("./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", ""), + args = args ) mockkObject(DexParser) { diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt index 8093baa26d..f5c3369d88 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt @@ -21,46 +21,45 @@ class ResolveApksKtTest { @Test fun `should resolve apks from global app and test`() { + val args = mockk { + every { appApk } returns "app" + every { testApk } returns "test" + every { additionalApks } returns emptyList() + every { additionalAppTestApks } returns emptyList() + every { testTargetsForShard } returns emptyList() + every { customSharding } returns emptyMap() + } assertArrayEquals( arrayOf( InstrumentationTestContext( app = "app".asFileReference(), - test = "test".asFileReference() + test = "test".asFileReference(), + args = args ) ), - mockk { - every { appApk } returns "app" - every { testApk } returns "test" - every { additionalApks } returns emptyList() - every { additionalAppTestApks } returns emptyList() - every { testTargetsForShard } returns emptyList() - every { customSharding } returns emptyMap() - }.resolveApks().toTypedArray() + args.resolveApks().toTypedArray() ) } @Test fun `should resolve apks from additionalAppTestApks`() { + val args = AndroidArgs.default().copy( + additionalAppTestApks = listOf( + AppTestPair( + app = "app", + test = "test" + ) + ) + ) assertArrayEquals( arrayOf( InstrumentationTestContext( app = "app".asFileReference(), - test = "test".asFileReference() + test = "test".asFileReference(), + args = args ) ), - mockk { - every { appApk } returns null - every { testApk } returns null - every { additionalApks } returns emptyList() - every { additionalAppTestApks } returns listOf( - AppTestPair( - app = "app", - test = "test" - ) - ) - every { testTargetsForShard } returns emptyList() - every { customSharding } returns emptyMap() - }.resolveApks().toTypedArray() + args.resolveApks().toTypedArray() ) } @@ -90,7 +89,7 @@ class ResolveApksKtTest { every { type } returns Type.ROBO } assertArrayEquals( - arrayOf(SanityRoboTestContext("app".asFileReference())), + arrayOf(SanityRoboTestContext("app".asFileReference(), androidArgs)), androidArgs.resolveApks().toTypedArray() ) } diff --git a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt index f4bf28694b..756ff6b353 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt @@ -50,9 +50,8 @@ inline fun should(crossinline match: T.() -> Boolean): T = moc val value = slot.captured value.match().also { matches -> matched = matches - if (matches) - println("${this@mockk} match succeed: $value") else - println("${this@mockk} match failed: $value") + if (matches) println("${this@mockk} match succeed: $value") + else println("${this@mockk} match failed: $value") } } every { this@mockk.toString() } answers {