diff --git a/release_notes.md b/release_notes.md index e19e58c41d..726629c7ce 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,6 @@ ## next (unreleased) +- [#817](https://github.com/Flank/flank/pull/817) Add AndroidTestContext as base data for dump shards & test execution. ([jan-gogo](https://github.com/jan-gogo)) - [#801](https://github.com/Flank/flank/pull/801) Omit missing app apk if additional-app-test-apks specified. ([jan-gogo](https://github.com/jan-gogo)) - [#784](https://github.com/Flank/flank/pull/784) Add output-style option. ([jan-gogo](https://github.com/jan-gogo)) - [#779](https://github.com/Flank/flank/pull/779) Print retries & display additional info. ([jan-gogo](https://github.com/jan-gogo)) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index cf2e0bad74..b66bd23475 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -123,7 +123,7 @@ class AndroidArgs( "Option num-uniform-shards cannot be specified along with max-test-shards. Use only one of them." ) - if (!(isRoboTest xor isInstrumentationTest)) throw FlankFatalError( + if (!(isRoboTest or isInstrumentationTest)) throw FlankFatalError( "One of following options must be specified [test, robo-directives, robo-script]." ) @@ -138,8 +138,13 @@ class AndroidArgs( outputStyle } - val isInstrumentationTest get() = testApk != null || additionalAppTestApks.run { isNotEmpty() && all { (app, _) -> app != null } } - val isRoboTest get() = roboDirectives.isNotEmpty() || roboScript != null + val isInstrumentationTest + get() = appApk != null && testApk != null || + additionalAppTestApks.isNotEmpty() && + (appApk != null || additionalAppTestApks.all { (app, _) -> app != null }) + private val isRoboTest + get() = appApk != null && + (roboDirectives.isNotEmpty() || roboScript != null) private fun assertDeviceSupported(device: Device) { when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) { diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt index 8686c05820..73d150526a 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt @@ -8,18 +8,6 @@ data class AppTestPair( val test: String ) -data class ResolvedApks( - val app: String, - val test: String?, - val additionalApks: List = emptyList() -) - -data class UploadedApks( - val app: String, - val test: String?, - val additionalApks: List = emptyList() -) - /** Flank specific parameters for Android */ @JsonIgnoreProperties(ignoreUnknown = true) class AndroidFlankYmlParams( diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 3cc9e619ad..2049ac3aed 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -42,10 +42,10 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { val config = AndroidArgs.load(Paths.get(configPath), cli = this) - if (dumpShards) { - dumpShards(config) - } else runBlocking { - newTestRun(config) + runBlocking { + if (dumpShards) + dumpShards(config) else + newTestRun(config) } } diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 06a62c5593..354e90b195 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -9,21 +9,27 @@ import ftl.util.FlankFatalError import java.nio.file.Files import java.nio.file.Paths -fun dumpShards(args: AndroidArgs) { +suspend fun dumpShards( + args: AndroidArgs, + shardFilePath: String = ANDROID_SHARD_FILE +) { if (!args.isInstrumentationTest) throw FlankFatalError( "Cannot dump shards for non instrumentation test, ensure test apk has been set." ) - val shards: AndroidMatrixTestShards = getAndroidMatrixShards(args) + val shards: AndroidMatrixTestShards = args.getAndroidMatrixShards() saveShardChunks( - shardFilePath = ANDROID_SHARD_FILE, + shardFilePath = shardFilePath, shards = shards, size = shards.size ) } -fun dumpShards(args: IosArgs) { +fun dumpShards( + args: IosArgs, + shardFilePath: String = IOS_SHARD_FILE +) { saveShardChunks( - shardFilePath = IOS_SHARD_FILE, + shardFilePath = shardFilePath, shards = args.testShardChunks, size = args.testShardChunks.size ) diff --git a/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt new file mode 100644 index 0000000000..44b1af46bb --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/model/AndroidTestContext.kt @@ -0,0 +1,17 @@ +package ftl.run.model + +import ftl.args.ShardChunks +import ftl.util.FileReference + +sealed class AndroidTestContext + +data class InstrumentationTestContext( + val app: FileReference, + val test: FileReference, + val shards: ShardChunks = emptyList() +) : AndroidTestContext() + +data class RoboTestContext( + val app: FileReference, + val roboScript: FileReference +) : AndroidTestContext() diff --git a/test_runner/src/main/kotlin/ftl/run/model/InstrumentationTestApk.kt b/test_runner/src/main/kotlin/ftl/run/model/InstrumentationTestApk.kt deleted file mode 100644 index 13966d94a8..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/model/InstrumentationTestApk.kt +++ /dev/null @@ -1,8 +0,0 @@ -package ftl.run.model - -import ftl.util.FileReference - -data class InstrumentationTestApk( - val app: FileReference = FileReference(), - val test: FileReference = FileReference() -) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index b3d1e43457..55e435fc2c 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -3,17 +3,16 @@ package ftl.run.platform import com.google.api.services.testing.Testing import com.google.api.services.testing.model.TestMatrix import ftl.args.AndroidArgs -import ftl.args.ShardChunks -import ftl.run.platform.android.getAndroidShardChunks -import ftl.args.yml.ResolvedApks import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix import ftl.gc.GcToolResults import ftl.http.executeWithRetry +import ftl.run.model.InstrumentationTestContext import ftl.run.model.TestResult import ftl.run.platform.android.createAndroidTestConfig -import ftl.run.platform.android.resolveApks -import ftl.run.platform.android.uploadApks +import ftl.run.platform.android.createAndroidTestContexts +import ftl.run.platform.android.upload +import ftl.run.platform.android.uploadAdditionalApks import ftl.run.platform.android.uploadOtherFiles import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage @@ -37,47 +36,26 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val history = GcToolResults.createToolResultsHistory(args) val otherGcsFiles = args.uploadOtherFiles(runGcsPath) + val additionalApks = args.uploadAdditionalApks(runGcsPath) - args.resolveApks().forEachIndexed { index: Int, apks: ResolvedApks -> - val testShards = apks.test?.let { test -> - getAndroidShardChunks(args, test) - } - // We can't return if testShards is null since it can be a robo test. - testShards?.let { - val shardsWithAtLeastOneTest = testShards.filterAtLeastOneTest() - if (shardsWithAtLeastOneTest.isEmpty()) { - // No tests to run, skipping the execution. - return@forEachIndexed + args.createAndroidTestContexts() + .upload(args.resultsBucket, runGcsPath) + .forEachIndexed { index, context -> + if (context is InstrumentationTestContext) allTestShardChunks += context.shards + val androidTestConfig = args.createAndroidTestConfig(context) + testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { + GcAndroidTestMatrix.build( + androidTestConfig = androidTestConfig, + runGcsPath = "$runGcsPath/matrix_$index/", + additionalApkGcsPaths = additionalApks, + androidDeviceList = androidDeviceList, + args = args, + otherFiles = otherGcsFiles, + toolResultsHistory = history + ) } - allTestShardChunks += shardsWithAtLeastOneTest } - val uploadedApks = uploadApks( - apks = apks, - args = args, - runGcsPath = runGcsPath - ) - - val androidTestConfig = args.createAndroidTestConfig( - uploadedApks = uploadedApks, - testShards = testShards, - runGcsPath = runGcsPath, - keepTestTargetsEmpty = args.disableSharding && args.testTargets.isEmpty() - ) - - testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { - GcAndroidTestMatrix.build( - androidTestConfig = androidTestConfig, - runGcsPath = "$runGcsPath/matrix_$index/", - additionalApkGcsPaths = uploadedApks.additionalApks, - androidDeviceList = androidDeviceList, - args = args, - otherFiles = otherGcsFiles, - toolResultsHistory = history - ) - } - } - if (testMatrices.isEmpty()) throw FlankCommonException("There are no tests to run.") println(beforeRunMessage(args, allTestShardChunks)) @@ -95,5 +73,3 @@ private suspend fun executeAndroidTestMatrix( } } } - -private fun ShardChunks.filterAtLeastOneTest(): ShardChunks = filter { chunk -> chunk.isNotEmpty() } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt index 0c717b1c8e..c78926224a 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestConfig.kt @@ -1,25 +1,13 @@ package ftl.run.platform.android import ftl.args.AndroidArgs -import ftl.args.ShardChunks -import ftl.args.yml.UploadedApks -import ftl.util.FlankFatalError +import ftl.run.model.AndroidTestContext +import ftl.run.model.InstrumentationTestContext +import ftl.run.model.RoboTestContext internal fun AndroidArgs.createAndroidTestConfig( - uploadedApks: UploadedApks, - testShards: ShardChunks? = null, - runGcsPath: String? = null, - keepTestTargetsEmpty: Boolean = false -): AndroidTestConfig = when { - isInstrumentationTest -> createInstrumentationConfig( - uploadedApks = uploadedApks, - keepTestTargetsEmpty = keepTestTargetsEmpty, - testShards = testShards ?: throw FlankFatalError("Arg testShards is required for instrumentation test.") - ) - isRoboTest - -> createRoboConfig( - uploadedApks = uploadedApks, - runGcsPath = runGcsPath ?: throw FlankFatalError("Arg runGcsPath is required for robo test.") - ) - else -> throw FlankFatalError("Cannot create AndroidTestConfig, invalid AndroidArgs.") + testContext: AndroidTestContext +): AndroidTestConfig = when (testContext) { + is InstrumentationTestContext -> createInstrumentationConfig(testContext) + is RoboTestContext -> createRoboConfig(testContext) } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt new file mode 100644 index 0000000000..b8527dd01f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -0,0 +1,72 @@ +package ftl.run.platform.android + +import com.linkedin.dex.parser.DexParser +import ftl.args.AndroidArgs +import ftl.args.ArgsHelper +import ftl.config.FtlConstants +import ftl.filter.TestFilter +import ftl.filter.TestFilters +import ftl.run.model.AndroidTestContext +import ftl.run.model.InstrumentationTestContext +import ftl.util.FlankTestMethod +import ftl.util.downloadIfNeeded +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import java.io.File + +suspend fun AndroidArgs.createAndroidTestContexts(): List = resolveApks().setupShards(this) + +private suspend fun List.setupShards( + args: AndroidArgs, + testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets) +): List = coroutineScope { + map { testContext -> + async { + if (testContext !is InstrumentationTestContext) testContext + else testContext.downloadApks().calculateShards( + args = args, + testFilter = testFilter + ) + } + }.awaitAll().dropEmptyInstrumentationTest() +} + +private fun InstrumentationTestContext.downloadApks(): InstrumentationTestContext = copy( + app = app.downloadIfNeeded(), + test = test.downloadIfNeeded() +) + +private fun InstrumentationTestContext.calculateShards( + args: AndroidArgs, + testFilter: TestFilter = TestFilters.fromTestTargets(args.testTargets) +): InstrumentationTestContext = copy( + shards = ArgsHelper.calculateShards( + filteredTests = getFlankTestMethods(testFilter), + args = args, + forcedShardCount = args.numUniformShards + ).filter { it.isNotEmpty() } +) + +private fun InstrumentationTestContext.getFlankTestMethods( + testFilter: TestFilter +): List = + DexParser.findTestMethods(test.local).asSequence().distinct().filter(testFilter.shouldRun).map { testMethod -> + FlankTestMethod( + testName = "class ${testMethod.testName}", + ignored = testMethod.annotations.any { it.name == "org.junit.Ignore" } + ) + }.toList() + +private fun List.dropEmptyInstrumentationTest(): List = + filterIsInstance().filter { it.shards.isEmpty() }.let { withoutTests -> + if (withoutTests.isNotEmpty()) + printNoTests(withoutTests) + minus(withoutTests) + } + +private fun printNoTests(testApks: List) { + val testApkNames = testApks.joinToString(", ") { pathname -> File(pathname.test.local).name } + println("${FtlConstants.indent}No tests for $testApkNames") + println() +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt index 4f0a632c0b..4761b2e78f 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateInstrumentationConfig.kt @@ -1,20 +1,17 @@ package ftl.run.platform.android import ftl.args.AndroidArgs -import ftl.args.ShardChunks -import ftl.args.yml.UploadedApks +import ftl.run.model.InstrumentationTestContext internal fun AndroidArgs.createInstrumentationConfig( - uploadedApks: UploadedApks, - keepTestTargetsEmpty: Boolean, - testShards: ShardChunks + testApk: InstrumentationTestContext ) = AndroidTestConfig.Instrumentation( - appApkGcsPath = uploadedApks.app, - testApkGcsPath = uploadedApks.test!!, + appApkGcsPath = testApk.app.gcs, + testApkGcsPath = testApk.test.gcs, testRunnerClass = testRunnerClass, orchestratorOption = "USE_ORCHESTRATOR".takeIf { useOrchestrator }, disableSharding = disableSharding, numUniformShards = numUniformShards, - testShards = testShards, - keepTestTargetsEmpty = keepTestTargetsEmpty + testShards = testApk.shards, + keepTestTargetsEmpty = disableSharding && testTargets.isEmpty() ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt index 0ff66bc640..1aabef65fe 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateRoboConfig.kt @@ -1,16 +1,12 @@ package ftl.run.platform.android import ftl.args.AndroidArgs -import ftl.args.yml.UploadedApks -import ftl.gc.GcStorage +import ftl.run.model.RoboTestContext internal fun AndroidArgs.createRoboConfig( - uploadedApks: UploadedApks, - runGcsPath: String + testApk: RoboTestContext ) = AndroidTestConfig.Robo( - appApkGcsPath = uploadedApks.app, + appApkGcsPath = testApk.app.gcs, flankRoboDirectives = roboDirectives, - roboScriptGcsPath = roboScript?.let { - GcStorage.upload(it, resultsBucket, runGcsPath) - } + roboScriptGcsPath = testApk.roboScript.gcs ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt index f6a2d480cb..093ced7bb3 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt @@ -1,42 +1,21 @@ package ftl.run.platform.android import ftl.args.AndroidArgs -import ftl.args.ShardChunks import ftl.run.model.AndroidMatrixTestShards import ftl.run.model.AndroidTestShards -import ftl.run.model.InstrumentationTestApk -import ftl.util.FlankFatalError -import ftl.util.asFileReference +import ftl.run.model.InstrumentationTestContext -fun getAndroidMatrixShards( - args: AndroidArgs -): AndroidMatrixTestShards = - getInstrumentationShardChunks( - args = args, - testApks = args.createInstrumentationTestApks() - ).asMatrixTestShards() +suspend fun AndroidArgs.getAndroidMatrixShards(): AndroidMatrixTestShards = this + .createAndroidTestContexts() + .filterIsInstance() + .asMatrixTestShards() -private fun AndroidArgs.createInstrumentationTestApks(): List = - listOfNotNull( - testApk?.let { testApk -> - InstrumentationTestApk( - app = appApk?.asFileReference() ?: throw FlankFatalError("Cannot resolve app apk for $testApk"), - test = testApk.asFileReference() - ) - } - ) + additionalAppTestApks.map { - InstrumentationTestApk( - app = (it.app ?: appApk)?.asFileReference() ?: throw FlankFatalError("Cannot resolve app apk for $testApk"), - test = it.test.asFileReference() - ) - } - -private fun Map.asMatrixTestShards(): AndroidMatrixTestShards = - map { (testApks, shards: List>) -> +private fun List.asMatrixTestShards(): AndroidMatrixTestShards = + map { testApks -> AndroidTestShards( app = testApks.app.local, test = testApks.test.local, - shards = shards.mapIndexed { index, testCases -> + shards = testApks.shards.mapIndexed { index, testCases -> "shard-$index" to testCases }.toMap() ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidShardChunks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidShardChunks.kt deleted file mode 100644 index 709c1f51fb..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidShardChunks.kt +++ /dev/null @@ -1,15 +0,0 @@ -package ftl.run.platform.android - -import ftl.args.AndroidArgs -import ftl.args.ShardChunks -import ftl.run.model.InstrumentationTestApk -import ftl.util.asFileReference - -fun getAndroidShardChunks( - args: AndroidArgs, - testApk: String -): ShardChunks = - getInstrumentationShardChunks( - args = args, - testApks = listOf(InstrumentationTestApk(test = testApk.asFileReference())) - ).flatMap { (_, shardChunks) -> shardChunks } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/GetInstrumentationShardChunks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/GetInstrumentationShardChunks.kt deleted file mode 100644 index a15a8ea113..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/GetInstrumentationShardChunks.kt +++ /dev/null @@ -1,55 +0,0 @@ -package ftl.run.platform.android - -import com.linkedin.dex.parser.DexParser -import ftl.args.AndroidArgs -import ftl.args.ArgsHelper -import ftl.args.ShardChunks -import ftl.config.FtlConstants -import ftl.filter.TestFilter -import ftl.filter.TestFilters -import ftl.run.model.InstrumentationTestApk -import ftl.util.FlankTestMethod -import ftl.util.downloadIfNeeded -import java.io.File - -fun getInstrumentationShardChunks( - args: AndroidArgs, - testApks: List -): Map = - getFlankTestMethods( - testApks = testApks.download(), - testFilter = TestFilters.fromTestTargets(args.testTargets) - ).mapValues { (_, testMethods: List) -> - if (testMethods.isNotEmpty()) { - ArgsHelper.calculateShards(testMethods, args, args.numUniformShards) - } else { - printNoTests(testApks) - emptyList() - } - } - -private fun getFlankTestMethods( - testApks: List, - testFilter: TestFilter -): Map> = - testApks.associateWith { testApk -> - DexParser.findTestMethods(testApk.test.local).asSequence().distinct().filter(testFilter.shouldRun).map { testMethod -> - FlankTestMethod( - testName = "class ${testMethod.testName}", - ignored = testMethod.annotations.any { it.name == "org.junit.Ignore" } - ) - }.toList() - } - -private fun List.download(): List = - map { reference -> - reference.copy( - app = reference.app.downloadIfNeeded(), - test = reference.test.downloadIfNeeded() - ) - } - -private fun printNoTests(testApks: List) { - val testApkNames = testApks.joinToString(", ") { pathname -> File(pathname.test.local).name } - println("${FtlConstants.indent}No tests for $testApkNames") -} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index 3ec93abdbe..3f9e32be44 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -1,23 +1,35 @@ package ftl.run.platform.android +import com.google.common.annotations.VisibleForTesting import ftl.args.AndroidArgs -import ftl.args.yml.ResolvedApks +import ftl.run.model.AndroidTestContext +import ftl.run.model.InstrumentationTestContext +import ftl.run.model.RoboTestContext import ftl.util.FlankFatalError +import ftl.util.asFileReference -internal fun AndroidArgs.resolveApks() = listOfNotNull( - element = appApk?.let { appApk -> - ResolvedApks( - app = appApk, - test = testApk, - additionalApks = additionalApks - ) +@VisibleForTesting +internal fun AndroidArgs.resolveApks(): List = listOfNotNull( + appApk?.let { appApk -> + testApk?.let { testApk -> + InstrumentationTestContext( + app = appApk.asFileReference(), + test = testApk.asFileReference() + ) + } ?: roboScript?.let { roboScript -> + RoboTestContext( + app = appApk.asFileReference(), + roboScript = roboScript.asFileReference() + ) + } } ).plus( elements = additionalAppTestApks.map { - ResolvedApks( - app = it.app ?: appApk ?: throw FlankFatalError("Cannot resolve app apk for ${it.test}"), - test = it.test, - additionalApks = additionalApks + InstrumentationTestContext( + app = (it.app ?: appApk) + ?.asFileReference() + ?: throw FlankFatalError("Cannot create app-test apks pair for instrumentation tests, missing app apk for test ${it.test}"), + test = it.test.asFileReference() ) } ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt index 51a1371591..612a3b7d2b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt @@ -1,9 +1,11 @@ package ftl.run.platform.android import ftl.args.AndroidArgs -import ftl.args.yml.ResolvedApks -import ftl.args.yml.UploadedApks -import ftl.gc.GcStorage +import ftl.run.model.AndroidTestContext +import ftl.run.model.InstrumentationTestContext +import ftl.run.model.RoboTestContext +import ftl.util.asFileReference +import ftl.util.uploadIfNeeded import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -14,21 +16,27 @@ import kotlinx.coroutines.coroutineScope * * @return AppTestPair with their GCS paths */ -internal suspend fun uploadApks( - apks: ResolvedApks, - args: AndroidArgs, - runGcsPath: String -): UploadedApks = coroutineScope { - val gcsBucket = args.resultsBucket +suspend fun List.upload(rootGcsBucket: String, runGcsPath: String) = coroutineScope { + map { context -> async(Dispatchers.IO) { context.upload(rootGcsBucket, runGcsPath) } }.awaitAll() +} + +private fun AndroidTestContext.upload(rootGcsBucket: String, runGcsPath: String) = when (this) { + is InstrumentationTestContext -> upload(rootGcsBucket, runGcsPath) + is RoboTestContext -> upload(rootGcsBucket, runGcsPath) +} + +private fun InstrumentationTestContext.upload(rootGcsBucket: String, runGcsPath: String) = copy( + app = app.uploadIfNeeded(rootGcsBucket, runGcsPath), + test = test.uploadIfNeeded(rootGcsBucket, runGcsPath) +) - val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app, gcsBucket, runGcsPath) } - val testApkGcsPath = apks.test?.let { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } - val additionalApkGcsPaths = - apks.additionalApks.map { async(Dispatchers.IO) { GcStorage.upload(it, gcsBucket, runGcsPath) } } +private fun RoboTestContext.upload(rootGcsBucket: String, runGcsPath: String) = copy( + app = app.uploadIfNeeded(rootGcsBucket, runGcsPath), + roboScript = roboScript.uploadIfNeeded(rootGcsBucket, runGcsPath) +) - UploadedApks( - app = appApkGcsPath.await(), - test = testApkGcsPath?.await(), - additionalApks = additionalApkGcsPaths.awaitAll() - ) +suspend fun AndroidArgs.uploadAdditionalApks(runGcsPath: String) = coroutineScope { + additionalApks.map { + async { it.asFileReference().uploadIfNeeded(resultsBucket, runGcsPath).gcs } + }.awaitAll() } diff --git a/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusPrinter.kt b/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusPrinter.kt index dbf749736f..31a364836c 100644 --- a/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusPrinter.kt +++ b/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusPrinter.kt @@ -17,9 +17,11 @@ fun createExecutionStatusPrinter( @VisibleForTesting internal class SingleLinePrinter : (List) -> Unit { private var previousLineSize = 0 + private val cache = mutableMapOf() override fun invoke(changes: List) { + cache += changes.associateBy(ExecutionStatus.Change::name) val time = changes.firstOrNull()?.time ?: return - changes.fold(emptyMap()) { acc, (_, _, current: ExecutionStatus) -> + cache.values.fold(emptyMap()) { acc, (_, _, current: ExecutionStatus) -> acc + Pair( first = current.state, second = acc.getOrDefault(current.state, 0) + 1 diff --git a/test_runner/src/main/kotlin/ftl/util/FileReference.kt b/test_runner/src/main/kotlin/ftl/util/FileReference.kt index d66a97a9d9..800f12cc48 100644 --- a/test_runner/src/main/kotlin/ftl/util/FileReference.kt +++ b/test_runner/src/main/kotlin/ftl/util/FileReference.kt @@ -6,15 +6,26 @@ import ftl.gc.GcStorage data class FileReference( val local: String = "", val gcs: String = "" -) +) { + init { + assertNotEmpty() + } +} + +private fun FileReference.assertNotEmpty() { + if (local.isBlank() && gcs.isBlank()) + throw FlankFatalError("Cannot create empty FileReference") +} fun String.asFileReference(): FileReference = if (startsWith(FtlConstants.GCS_PREFIX)) FileReference(gcs = this) else FileReference(local = this) -fun FileReference.downloadIfNeeded() = when { - local.isNotBlank() -> this - gcs.isNotBlank() -> copy(local = GcStorage.download(gcs)) - else -> this -} +fun FileReference.downloadIfNeeded() = + if (local.isNotBlank()) this + else copy(local = GcStorage.download(gcs)) + +fun FileReference.uploadIfNeeded(rootGcsBucket: String, runGcsPath: String) = + if (gcs.isNotBlank()) this + else copy(gcs = GcStorage.upload(local, rootGcsBucket, runGcsPath)) diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index 8cddfa8604..a32987d08f 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -21,6 +21,8 @@ fun main() { "firebase", "test", "android", "run", // "--dry", +// "--dump-shards", + "--output-style=single", "--full-junit-result", "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", "--project=$projectId" diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index 09a14f0146..695436a1d5 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -11,14 +11,15 @@ import ftl.args.yml.FlankYmlParams import ftl.args.yml.GcloudYml import ftl.args.yml.GcloudYmlParams import ftl.config.Device +import ftl.run.platform.android.createAndroidTestContexts import ftl.run.platform.android.getAndroidMatrixShards -import ftl.run.platform.android.getAndroidShardChunks import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath import ftl.test.util.TestHelper.getString +import kotlinx.coroutines.runBlocking import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test @@ -142,7 +143,7 @@ class AndroidArgsFileTest { ), "" ) - with(getAndroidMatrixShards(config)) { + with(runBlocking { config.getAndroidMatrixShards() }) { assertEquals(1, get("matrix-0")!!.shards["shard-0"]!!.size) assertEquals(51, get("matrix-1")!!.shards["shard-0"]!!.size) assertEquals(52, get("matrix-1")!!.shards["shard-1"]!!.size) @@ -151,9 +152,9 @@ class AndroidArgsFileTest { } @Test - fun `calculateShards 0`() { + fun `calculateShards 0`() = runBlocking { val config = configWithTestMethods(0) - val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + val testShardChunks = config.createAndroidTestContexts() with(config) { assert(maxTestShards, 1) assert(testShardChunks.size, 0) @@ -163,7 +164,7 @@ class AndroidArgsFileTest { @Test fun `calculateShards 1`() { val config = configWithTestMethods(1) - val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + val testShardChunks = getAndroidShardChunks(config) with(config) { assert(maxTestShards, 1) assert(testShardChunks.size, 1) @@ -174,7 +175,7 @@ class AndroidArgsFileTest { @Test fun `calculateShards 155`() { val config = configWithTestMethods(155) - val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + val testShardChunks = getAndroidShardChunks(config) with(config) { assert(maxTestShards, 1) assert(testShardChunks.size, 1) @@ -185,7 +186,7 @@ class AndroidArgsFileTest { @Test fun `calculateShards 155 40`() { val config = configWithTestMethods(155, maxTestShards = 40) - val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + val testShardChunks = getAndroidShardChunks(config) with(config) { assert(maxTestShards, 40) assert(testShardChunks.size, 40) @@ -196,7 +197,7 @@ class AndroidArgsFileTest { @Test fun `should distribute equally to shards`() { val config = configWithTestMethods(155, maxTestShards = 40) - val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + val testShardChunks = getAndroidShardChunks(config) with(config) { assert(maxTestShards, 40) assert(testShardChunks.size, 40) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 4091ce30e3..d628bfb03d 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -3,16 +3,16 @@ package ftl.args import com.google.api.services.testing.model.TestSpecification import com.google.common.truth.Truth.assertThat import ftl.args.yml.AppTestPair -import ftl.args.yml.UploadedApks import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FlankRoboDirective import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion import ftl.gc.android.setupAndroidTest +import ftl.run.model.InstrumentationTestContext import ftl.run.platform.android.createAndroidTestConfig +import ftl.run.platform.android.createAndroidTestContexts import ftl.run.platform.runAndroidTests -import ftl.run.platform.android.getAndroidShardChunks import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.absolutePath @@ -20,6 +20,7 @@ import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath import ftl.util.FlankCommonException import ftl.util.FlankFatalError +import ftl.util.asFileReference import io.mockk.every import io.mockk.mockkStatic import io.mockk.unmockkAll @@ -459,7 +460,7 @@ AndroidArgs """ ) - val testShardChunks = getAndroidShardChunks(androidArgs, androidArgs.testApk!!) + val testShardChunks = getAndroidShardChunks(androidArgs) with(androidArgs) { assert(maxTestShards, -1) assert(testShardChunks.size, 2) @@ -493,7 +494,7 @@ AndroidArgs disable-sharding: true """ val androidArgs = AndroidArgs.load(yaml) - val testShardChunks = getAndroidShardChunks(androidArgs, androidArgs.testApk!!) + val testShardChunks = runBlocking { androidArgs.createAndroidTestContexts() } assertThat(testShardChunks).hasSize(0) } @@ -505,7 +506,7 @@ AndroidArgs test: $invalidApk """ val androidArgs = AndroidArgs.load(yaml) - val testShardChunks = getAndroidShardChunks(androidArgs, androidArgs.testApk!!) + val testShardChunks = runBlocking { androidArgs.createAndroidTestContexts() } assertThat(testShardChunks).hasSize(0) } @@ -1313,8 +1314,8 @@ AndroidArgs test: $testApk """.trimIndent() - mockkStatic("ftl.run.platform.android.GetAndroidShardChunksKt") - every { getAndroidShardChunks(any(), any()) } returns listOf() + mockkStatic("ftl.run.platform.android.CreateAndroidTestContextKt") + every { runBlocking { any().createAndroidTestContexts() } } returns listOf() val parsedYml = AndroidArgs.load(yaml) runBlocking { runAndroidTests(parsedYml) } @@ -1379,7 +1380,13 @@ AndroidArgs disable-sharding: true """.trimIndent() val args = AndroidArgs.load(yaml) - val androidTestConfig = args.createAndroidTestConfig(UploadedApks("", ""), listOf(listOf("test")), null, true) + val androidTestConfig = args.createAndroidTestConfig( + InstrumentationTestContext( + app = "app".asFileReference(), + test = "test".asFileReference(), + shards = listOf(listOf("test"), listOf("test")) + ) + ) val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig) assertTrue(testSpecification.androidInstrumentationTest.testTargets.isEmpty()) } @@ -1397,10 +1404,18 @@ AndroidArgs disable-sharding: true """.trimIndent() val args = AndroidArgs.load(yaml) - val androidTestConfig = args.createAndroidTestConfig(UploadedApks("", ""), listOf(listOf("test"), listOf("test"))) + val androidTestConfig = args.createAndroidTestConfig( + InstrumentationTestContext( + app = "app".asFileReference(), + test = "test".asFileReference(), + shards = listOf(listOf("test"), listOf("test")) + ) + ) val testSpecification = TestSpecification().setupAndroidTest(androidTestConfig) assertTrue(testSpecification.androidInstrumentationTest.testTargets.isNotEmpty()) } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = load(StringReader(yamlData), cli) + +fun getAndroidShardChunks(args: AndroidArgs): ShardChunks = runBlocking { (args.createAndroidTestContexts().first() as InstrumentationTestContext).shards } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml index 0d58b7e8fc..483bbf9c25 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-apk.yml @@ -9,4 +9,4 @@ flank: num-test-runs: 1 additional-app-test-apks: - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk -# - test: ../test_app/apks/invalid.apk + - test: ../test_app/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml new file mode 100644 index 0000000000..92b15179cd --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml @@ -0,0 +1,13 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + robo-script: ./src/test/kotlin/ftl/fixtures/test_app_cases/MainActivity_robo_script.json + num-flaky-test-attempts: 2 + +flank: + disable-sharding: false + max-test-shards: 2 + num-test-runs: 1 + additional-app-test-apks: + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + - test: ../test_app/apks/invalid.apk diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt new file mode 100644 index 0000000000..3f23be45be --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -0,0 +1,87 @@ +package ftl.run + +import ftl.args.AndroidArgs +import ftl.args.IosArgs +import ftl.test.util.ios2ConfigYaml +import ftl.test.util.mixedConfigYaml +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Test +import java.io.File + +class DumpShardsKtTest { + + @Test + fun `dump shards android`() { + // given + val path = File("").absolutePath + val expected = """ +{ + "matrix-0": { + "app": "$path/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test": "$path/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", + "shards": { + "shard-0": [ + "class com.example.test_app.InstrumentedTest#test" + ] + } + }, + "matrix-1": { + "app": "$path/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test": "$path/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", + "shards": { + "shard-0": [ + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" + ], + "shard-1": [ + "class com.example.test_app.InstrumentedTest#test0", + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo" + ] + } + } +} + """.trimIndent() + + // when + val actual = runBlocking { + dumpShards(AndroidArgs.load(mixedConfigYaml), TEST_SHARD_FILE) + File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() + } + + // then + assertEquals(expected, actual) + } + + @Test + fun `dump shards ios`() { + // given + val expected = """ +[ + [ + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testWithInRoot", + "EarlGreyExampleSwiftTests/testWithCondition", + "EarlGreyExampleSwiftTests/testWithCustomFailureHandler" + ], + [ + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testWithCustomMatcher", + "EarlGreyExampleSwiftTests/testWithCustomAssertion" + ] +] + """.trimIndent() + + // when + val actual = runBlocking { + dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) + File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() + } + + // then + assertEquals(expected, actual) + } +} + +private const val TEST_SHARD_FILE = "test_dump_shard_file.json" diff --git a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt new file mode 100644 index 0000000000..54b8d25940 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt @@ -0,0 +1,35 @@ +package ftl.run.platform + +import ftl.args.AndroidArgs +import ftl.json.MatrixMap +import ftl.test.util.FlankTestRunner +import ftl.test.util.mixedConfigYaml +import ftl.test.util.should +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(FlankTestRunner::class) +class RunAndroidTestsKtTest { + + @Test + fun `run android tests for mixed contexts`() { + // given + val expected = should { + map.size == 3 + } to listOf>( + should { size == 1 }, + should { size == 2 }, + should { size == 3 } + ) + + // when + val actual = runBlocking { + runAndroidTests(AndroidArgs.load(mixedConfigYaml)) + } + + // then + assertEquals(expected, actual) + } +} diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt new file mode 100644 index 0000000000..fa9ffd43c7 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt @@ -0,0 +1,48 @@ +package ftl.run.platform.android + +import ftl.args.AndroidArgs +import ftl.run.model.AndroidTestContext +import ftl.run.model.InstrumentationTestContext +import ftl.run.model.RoboTestContext +import ftl.test.util.mixedConfigYaml +import ftl.test.util.should +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Test + +class CreateAndroidTestContextKtTest { + + @Test + fun `create AndroidTestConfig for mixed tests`() { + // given + val expected = listOf( + RoboTestContext( + app = should { local.endsWith("app-debug.apk") }, + roboScript = should { local.endsWith("MainActivity_robo_script.json") } + ), + InstrumentationTestContext( + app = should { local.endsWith("app-debug.apk") }, + test = should { local.endsWith("app-single-success-debug-androidTest.apk") }, + shards = listOf( + should { size == 1 } + ) + ), + InstrumentationTestContext( + app = should { local.endsWith("app-debug.apk") }, + test = should { local.endsWith("app-multiple-flaky-debug-androidTest.apk") }, + shards = listOf( + should { size == 2 }, + should { size == 3 } + ) + ) + ) + + // when + val actual: List = runBlocking { + AndroidArgs.load(mixedConfigYaml).createAndroidTestContexts() + } + + // then + assertEquals(expected, actual) + } +} diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt index 91e0c50e98..9c00c11cf1 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt @@ -2,8 +2,9 @@ package ftl.run.platform.android import ftl.args.AndroidArgs import ftl.args.yml.AppTestPair -import ftl.args.yml.ResolvedApks +import ftl.run.model.InstrumentationTestContext import ftl.util.FlankFatalError +import ftl.util.asFileReference import io.mockk.every import io.mockk.mockk import org.junit.Assert.assertArrayEquals @@ -15,9 +16,9 @@ class ResolveApksKtTest { fun `should resolve apks from global app and test`() { assertArrayEquals( arrayOf( - ResolvedApks( - app = "app", - test = "test" + InstrumentationTestContext( + app = "app".asFileReference(), + test = "test".asFileReference() ) ), mockk { @@ -33,9 +34,9 @@ class ResolveApksKtTest { fun `should resolve apks from additionalAppTestApks`() { assertArrayEquals( arrayOf( - ResolvedApks( - app = "app", - test = "test" + InstrumentationTestContext( + app = "app".asFileReference(), + test = "test".asFileReference() ) ), mockk { diff --git a/test_runner/src/test/kotlin/ftl/test/util/Constants.kt b/test_runner/src/test/kotlin/ftl/test/util/Constants.kt new file mode 100644 index 0000000000..8364627d5f --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/test/util/Constants.kt @@ -0,0 +1,6 @@ +package ftl.test.util + +import ftl.test.util.TestHelper.getPath + +val mixedConfigYaml = getPath("src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml") +val ios2ConfigYaml = getPath("src/test/kotlin/ftl/fixtures/flank2.ios.yml") diff --git a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt index e580490fff..7ed1af23bc 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt @@ -1,5 +1,10 @@ +@file:Suppress("RemoveCurlyBracesFromTemplate") + package ftl.test.util +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot import org.junit.Assert import java.nio.file.Path import java.nio.file.Paths @@ -29,3 +34,30 @@ object TestHelper { exception } } + +inline fun ignore(): T = mockk(relaxed = true) { + val slot = slot() + every { this@mockk == capture(slot) } answers { + println("${this@mockk} match ignored: ${slot.captured}") + true + } +} + +inline fun should(crossinline match: T.() -> Boolean): T = mockk(relaxed = true) { + val slot = slot() + var matched = false + every { this@mockk == capture(slot) } answers { + val value = slot.captured + value.match().also { matches -> + matched = matches + if (matches) + println("${this@mockk} match succeed: $value") else + println("${this@mockk} match failed: $value") + } + } + every { this@mockk.toString() } answers { + if (matched && slot.isCaptured) + slot.captured.toString() else + callOriginal() + } +}