From 480cb7449d2885bda5d76fb8eec0cd5937cd331d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Tue, 10 Mar 2020 08:39:21 +0100 Subject: [PATCH] Refactor ftl.run (#661) --- .../main/kotlin/ftl/args/AndroidTestShard.kt | 2 +- .../src/main/kotlin/ftl/args/ArgsHelper.kt | 2 +- .../src/main/kotlin/ftl/args/IosArgs.kt | 2 +- .../src/main/kotlin/ftl/args/ShardChunks.kt | 3 + .../kotlin/ftl/cli/firebase/CancelCommand.kt | 4 +- .../kotlin/ftl/cli/firebase/RefreshCommand.kt | 7 +- .../test/android/AndroidRunCommand.kt | 16 +- .../cli/firebase/test/ios/IosRunCommand.kt | 13 +- .../main/kotlin/ftl/gc/GcAndroidTestMatrix.kt | 5 +- .../kotlin/ftl/reports/util/ReportManager.kt | 9 +- .../main/kotlin/ftl/run/AndroidTestRunner.kt | 80 ---- .../src/main/kotlin/ftl/run/CancelLastRun.kt | 45 +++ .../main/kotlin/ftl/run/GenericTestRunner.kt | 91 ----- .../src/main/kotlin/ftl/run/IosTestRunner.kt | 65 ---- .../src/main/kotlin/ftl/run/NewTestRun.kt | 33 ++ .../src/main/kotlin/ftl/run/RefreshLastRun.kt | 70 ++++ .../src/main/kotlin/ftl/run/TestRunner.kt | 355 ------------------ .../kotlin/ftl/run/common/FetchArtifacts.kt | 83 ++++ .../main/kotlin/ftl/run/common/GetLastArgs.kt | 28 ++ .../kotlin/ftl/run/common/GetLastGcsPath.kt | 16 + .../kotlin/ftl/run/common/GetLastMatrices.kt | 36 ++ .../kotlin/ftl/run/common/PollMatrices.kt | 63 ++++ .../main/kotlin/ftl/run/common/PrettyPrint.kt | 6 + .../kotlin/ftl/run/common/UpdateMatrixFile.kt | 19 + .../ftl/run/{ => model}/RunningDevices.kt | 2 +- .../main/kotlin/ftl/run/model/TestResult.kt | 6 + .../ftl/run/platform/RunAndroidTests.kt | 81 ++++ .../kotlin/ftl/run/platform/RunIosTests.kt | 71 ++++ .../ftl/run/platform/common/AfterRunTests.kt | 48 +++ .../run/platform/common/BeforeRunMessage.kt | 35 ++ .../ftl/run/platform/common/BeforeRunTests.kt | 47 +++ test_runner/src/test/kotlin/Debug.kt | 22 +- .../kotlin/ftl/gc/GcAndroidTestMatrixTest.kt | 6 +- .../ftl/reports/utils/ReportManagerTest.kt | 6 +- .../kotlin/ftl/run/GenericTestRunnerTest.kt | 16 +- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 17 +- 36 files changed, 762 insertions(+), 648 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/args/ShardChunks.kt delete mode 100644 test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt delete mode 100644 test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt delete mode 100644 test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/NewTestRun.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt delete mode 100644 test_runner/src/main/kotlin/ftl/run/TestRunner.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/GetLastGcsPath.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/PrettyPrint.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt rename test_runner/src/main/kotlin/ftl/run/{ => model}/RunningDevices.kt (99%) create mode 100644 test_runner/src/main/kotlin/ftl/run/model/TestResult.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt create mode 100644 test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt index 17db1fcb96..cc6b9bec2e 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt @@ -13,7 +13,7 @@ import kotlinx.coroutines.runBlocking object AndroidTestShard { // computed properties not specified in yaml - fun getTestShardChunks(args: AndroidArgs, testApk: String): List> { + fun getTestShardChunks(args: AndroidArgs, testApk: String): ShardChunks { // Download test APK if necessary so it can be used to validate test methods var testLocalApk = testApk if (testApk.startsWith(FtlConstants.GCS_PREFIX)) { diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index 83079d60cb..016a0af196 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -222,7 +222,7 @@ object ArgsHelper { return ArgsFileVisitor("glob:$filePath").walk(searchDir) } - fun calculateShards(filteredTests: List, args: IArgs): List> { + fun calculateShards(filteredTests: List, args: IArgs): ShardChunks { val shards = if (args.disableSharding) { mutableListOf(filteredTests.map { it.testName } as MutableList) } else { diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 471755baaf..1d3e7950b7 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -66,7 +66,7 @@ class IosArgs( val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull() // computed properties not specified in yaml - val testShardChunks: List> by lazy { + val testShardChunks: ShardChunks by lazy { if (disableSharding) return@lazy listOf(emptyList()) val validTestMethods = Xctestrun.findTestNames(xctestrunFile) diff --git a/test_runner/src/main/kotlin/ftl/args/ShardChunks.kt b/test_runner/src/main/kotlin/ftl/args/ShardChunks.kt new file mode 100644 index 0000000000..7b88d53c9c --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/ShardChunks.kt @@ -0,0 +1,3 @@ +package ftl.args + +typealias ShardChunks = List> diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt index 14a62c3aae..e11852f46d 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt @@ -1,7 +1,7 @@ package ftl.cli.firebase import ftl.args.AndroidArgs -import ftl.run.TestRunner +import ftl.run.cancelLastRun import picocli.CommandLine @CommandLine.Command( @@ -20,7 +20,7 @@ Reads in the matrix_ids.json file. Cancels any incomplete matrices. ) class CancelCommand : Runnable { override fun run() { - TestRunner.cancelLastRun(AndroidArgs.default()) + cancelLastRun(AndroidArgs.default()) } @CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["Prints this help message"]) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt index b9e90a4b40..0cd76c97f1 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt @@ -1,7 +1,7 @@ package ftl.cli.firebase import ftl.args.AndroidArgs -import ftl.run.TestRunner +import ftl.run.refreshLastRun import kotlinx.coroutines.runBlocking import picocli.CommandLine.Command import picocli.CommandLine.Option @@ -23,7 +23,10 @@ Reads in the matrix_ids.json file. Refreshes any incomplete matrices. class RefreshCommand : Runnable { override fun run() { runBlocking { - TestRunner.refreshLastRun(AndroidArgs.default(), emptyList()) + refreshLastRun( + currentArgs = AndroidArgs.default(), + testShardChunks = emptyList() + ) } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 89db2b15a9..317d032f87 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -2,6 +2,7 @@ package ftl.cli.firebase.test.android import ftl.args.AndroidArgs import ftl.args.AndroidTestShard +import ftl.args.ShardChunks import ftl.args.yml.AppTestPair import ftl.config.Device import ftl.config.FtlConstants @@ -10,13 +11,14 @@ import ftl.config.FtlConstants.defaultAndroidVersion import ftl.config.FtlConstants.defaultLocale import ftl.config.FtlConstants.defaultOrientation import ftl.mock.MockServer -import ftl.run.TestRunner -import java.nio.file.Files -import java.nio.file.Paths -import kotlin.system.exitProcess +import ftl.run.common.prettyPrint +import ftl.run.newTestRun import kotlinx.coroutines.runBlocking import picocli.CommandLine.Command import picocli.CommandLine.Option +import java.nio.file.Files +import java.nio.file.Paths +import kotlin.system.exitProcess @Command( name = "run", @@ -43,8 +45,8 @@ class AndroidRunCommand : Runnable { val config = AndroidArgs.load(Paths.get(configPath), cli = this) if (dumpShards) { - val testShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) - val testShardChunksJson = TestRunner.gson.toJson(testShardChunks) + val testShardChunks: ShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk) + val testShardChunksJson: String = prettyPrint.toJson(testShardChunks) Files.write(Paths.get(shardFile), testShardChunksJson.toByteArray()) println("Saved shards to $shardFile") @@ -52,7 +54,7 @@ class AndroidRunCommand : Runnable { } runBlocking { - TestRunner.newRun(config) + newTestRun(config) } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 1140c7d727..e6af8b7443 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -6,13 +6,14 @@ import ftl.config.FtlConstants import ftl.config.FtlConstants.defaultIosModel import ftl.config.FtlConstants.defaultIosVersion import ftl.mock.MockServer -import ftl.run.TestRunner -import java.nio.file.Files -import java.nio.file.Paths -import kotlin.system.exitProcess +import ftl.run.common.prettyPrint +import ftl.run.newTestRun import kotlinx.coroutines.runBlocking import picocli.CommandLine.Command import picocli.CommandLine.Option +import java.nio.file.Files +import java.nio.file.Paths +import kotlin.system.exitProcess @Command( name = "run", @@ -38,14 +39,14 @@ class IosRunCommand : Runnable { val config = IosArgs.load(Paths.get(configPath), cli = this) if (dumpShards) { - val testShardChunksJson = TestRunner.gson.toJson(config.testShardChunks) + val testShardChunksJson: String = prettyPrint.toJson(config.testShardChunks) Files.write(Paths.get(shardFile), testShardChunksJson.toByteArray()) println("Saved shards to $shardFile") exitProcess(0) } runBlocking { - TestRunner.newRun(config) + newTestRun(config) } } diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index e3a466bf64..c0b99200b1 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -19,6 +19,7 @@ import com.google.api.services.testing.model.TestSpecification import com.google.api.services.testing.model.TestTargetsForShard import com.google.api.services.testing.model.ToolResultsHistory import ftl.args.AndroidArgs +import ftl.args.ShardChunks import ftl.util.fatalError import ftl.util.join import ftl.util.testTimeoutToSeconds @@ -35,7 +36,7 @@ object GcAndroidTestMatrix { testApkGcsPath: String, runGcsPath: String, androidDeviceList: AndroidDeviceList, - testTargets: List>, + testShards: ShardChunks, args: AndroidArgs, toolResultsHistory: ToolResultsHistory ): Testing.Projects.TestMatrices.Create { @@ -46,7 +47,7 @@ object GcAndroidTestMatrix { val matrixGcsPath = join(args.resultsBucket, runGcsPath) // ShardingOption().setUniformSharding(UniformSharding().setNumShards()) - val testTargetsForShard: List = testTargets.map { + val testTargetsForShard: List = testShards.map { TestTargetsForShard().setTestTargets(it) } val manualSharding = ManualSharding().setTestTargetsForShard(testTargetsForShard) diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 5e50156509..f63d7935ab 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -2,6 +2,7 @@ package ftl.reports.util import ftl.args.IArgs import ftl.args.IosArgs +import ftl.args.ShardChunks import ftl.gc.GcStorage import ftl.json.MatrixMap import ftl.reports.CostReport @@ -91,7 +92,7 @@ object ReportManager { } /** Returns true if there were no test failures */ - fun generate(matrices: MatrixMap, args: IArgs, testShardChunks: List>): Int { + fun generate(matrices: MatrixMap, args: IArgs, testShardChunks: ShardChunks): Int { val testSuite = parseTestSuite(matrices, args) val useFlakyTests = args.flakyTestAttempts > 0 @@ -127,7 +128,7 @@ object ReportManager { oldResult: JUnitTestResult, newResult: JUnitTestResult, args: IArgs, - testShardChunks: List> + testShardChunks: ShardChunks ): List { val oldDurations = Shard.createTestMethodDurationMap(oldResult, args) @@ -154,7 +155,7 @@ object ReportManager { oldResult: JUnitTestResult, newResult: JUnitTestResult, args: IArgs, - testShardChunks: List> + testShardChunks: ShardChunks ) { val list = createShardEfficiencyList(oldResult, newResult, args, testShardChunks) @@ -166,7 +167,7 @@ object ReportManager { private fun processJunitXml( newTestResult: JUnitTestResult?, args: IArgs, - testShardChunks: List> + testShardChunks: ShardChunks ) { if (newTestResult == null) return diff --git a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt deleted file mode 100644 index 9f532e48d5..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt +++ /dev/null @@ -1,80 +0,0 @@ -package ftl.run - -import com.google.api.services.testing.model.TestMatrix -import ftl.args.AndroidArgs -import ftl.args.AndroidTestShard -import ftl.args.yml.AppTestPair -import ftl.gc.GcAndroidDevice -import ftl.gc.GcAndroidTestMatrix -import ftl.gc.GcStorage -import ftl.gc.GcToolResults -import ftl.http.executeWithRetry -import ftl.json.MatrixMap -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope - -object AndroidTestRunner { - - suspend fun runTests(args: AndroidArgs): Pair>> = coroutineScope { - val (stopwatch, runGcsPath) = GenericTestRunner.beforeRunTests(args) - - // GcAndroidMatrix => GcAndroidTestMatrix - // GcAndroidTestMatrix.execute() 3x retry => matrix id (string) - val androidDeviceList = GcAndroidDevice.build(args.devices) - - val jobs = arrayListOf>() - val runCount = args.repeatTests - val history = GcToolResults.createToolResultsHistory(args) - val apkPairsInArgs = listOf(AppTestPair(app = args.appApk, test = args.testApk)) + args.additionalAppTestApks - val allTestShardChunks: List> = apkPairsInArgs.map { unresolvedApkPair -> - val resolvedApkPair = resolveApkPair(unresolvedApkPair, args, runGcsPath) - // Ensure we only shard tests that are part of the test apk. Use the unresolved test apk path to make sure - // we don't re-download an apk it is on the local file system. - AndroidTestShard.getTestShardChunks(args, unresolvedApkPair.test).also { testShards -> - repeat(runCount) { - // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread - // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html - jobs += async(Dispatchers.IO) { - GcAndroidTestMatrix.build( - appApkGcsPath = resolvedApkPair.app, - testApkGcsPath = resolvedApkPair.test, - runGcsPath = runGcsPath, - androidDeviceList = androidDeviceList, - testTargets = testShards, - args = args, - toolResultsHistory = history - ).executeWithRetry() - } - } - } - }.flatten() - - println(GenericTestRunner.beforeRunMessage(args, allTestShardChunks)) - val matrixMap = GenericTestRunner.afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, args) - matrixMap to allTestShardChunks - } - - /** - * Upload an APK pair if the path given is local - * - * @return AppTestPair with their GCS paths - */ - private suspend fun resolveApkPair( - apk: AppTestPair, - args: AndroidArgs, - runGcsPath: String - ): AppTestPair = coroutineScope { - val gcsBucket = args.resultsBucket - - val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.app, gcsBucket, runGcsPath) } - val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.test, gcsBucket, runGcsPath) } - - AppTestPair( - app = appApkGcsPath.await(), - test = testApkGcsPath.await() - ) - } -} diff --git a/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt new file mode 100644 index 0000000000..e79af89737 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt @@ -0,0 +1,45 @@ +package ftl.run + +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.gc.GcTestMatrix +import ftl.json.MatrixMap +import ftl.run.common.getLastArgs +import ftl.run.common.getLastMatrices +import ftl.util.MatrixState +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking + +// used to cancel and update results from an async run +fun cancelLastRun(args: IArgs) { + val matrixMap = getLastMatrices(args) + val lastArgs = getLastArgs(args) + + cancelMatrices(matrixMap, lastArgs) +} + +/** Cancel all in progress matrices in parallel **/ +private fun cancelMatrices(matrixMap: MatrixMap, args: IArgs) { + println("CancelMatrices") + + val map = matrixMap.map + var matrixCount = 0 + + runBlocking { + map.forEach { matrix -> + // Only cancel unfinished + if (MatrixState.inProgress(matrix.value.state)) { + matrixCount += 1 + launch { GcTestMatrix.cancel(matrix.key, args) } + } + } + } + + if (matrixCount == 0) { + println(FtlConstants.indent + "No matrices to cancel") + } else { + println(FtlConstants.indent + "Cancelling ${matrixCount}x matrices") + } + + println() +} diff --git a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt deleted file mode 100644 index 84891dc00e..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/GenericTestRunner.kt +++ /dev/null @@ -1,91 +0,0 @@ -package ftl.run - -import com.google.api.services.testing.model.TestMatrix -import ftl.args.IArgs -import ftl.config.FtlConstants -import ftl.config.FtlConstants.useMock -import ftl.json.MatrixMap -import ftl.json.SavedMatrix -import ftl.util.StopWatch -import ftl.util.uniqueObjectName -import java.io.File - -object GenericTestRunner { - fun beforeRunTests(args: IArgs): Pair { - println("RunTests") - val stopwatch = StopWatch().start() - TestRunner.assertMockUrl() - - val runGcsPath = args.resultsDir ?: uniqueObjectName() - - // Avoid spamming the results/ dir with temporary files from running the test suite. - if (useMock) { - Runtime.getRuntime().addShutdownHook(Thread { - File(args.localResultDir, runGcsPath).deleteRecursively() - }) - } - - if (args.useLocalResultDir()) { - // Only one result is stored when using --local-result-dir - // Delete any old results if they exist before storing new ones. - File(args.localResultDir).deleteRecursively() - } - - return stopwatch to runGcsPath - } - - fun afterRunTests( - jobs: List, - runGcsPath: String, - stopwatch: StopWatch, - config: IArgs - ): MatrixMap { - val savedMatrices = mutableMapOf() - - jobs.forEach { matrix -> - val matrixId = matrix.testMatrixId - savedMatrices[matrixId] = SavedMatrix(matrix) - } - - val matrixMap = MatrixMap(savedMatrices, runGcsPath) - TestRunner.updateMatrixFile(matrixMap, config) - TestRunner.saveConfigFile(matrixMap, config) - - println(FtlConstants.indent + "${savedMatrices.size} matrix ids created in ${stopwatch.check()}") - val gcsBucket = "https://console.developers.google.com/storage/browser/" + - config.resultsBucket + "/" + matrixMap.runPath - println(FtlConstants.indent + gcsBucket) - println() - - return matrixMap - } - - private fun s(amount: Int): String { - return if (amount > 1) { - "s" - } else { - "" - } - } - - fun beforeRunMessage(args: IArgs, testShardChunks: List>): String { - val runCount = args.repeatTests - val shardCount = testShardChunks.size - val testsCount = testShardChunks.sumBy { it.size } - - val result = StringBuilder() - result.appendln( - " $testsCount test${s(testsCount)} / $shardCount shard${s(shardCount)}" - ) - - if (runCount > 1) { - result.appendln(" Running ${runCount}x") - val runDevices = runCount * shardCount - val runTests = runCount * testsCount - result.appendln(" $runDevices total shard${s(runDevices)}") - result.appendln(" $runTests total test${s(runTests)}") - } - - return result.toString() - } -} diff --git a/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt deleted file mode 100644 index b0e8bf3fe5..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/IosTestRunner.kt +++ /dev/null @@ -1,65 +0,0 @@ -package ftl.run - -import com.google.api.services.testing.model.TestMatrix -import ftl.args.IosArgs -import ftl.config.FtlConstants -import ftl.gc.GcIosMatrix -import ftl.gc.GcIosTestMatrix -import ftl.gc.GcStorage -import ftl.gc.GcToolResults -import ftl.http.executeWithRetry -import ftl.ios.Xctestrun -import ftl.json.MatrixMap -import ftl.run.GenericTestRunner.beforeRunMessage -import ftl.util.ShardCounter -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope - -object IosTestRunner { - // https://github.com/bootstraponline/gcloud_cli/blob/5bcba57e825fc98e690281cf69484b7ba4eb668a/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py#L109 - // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run - // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/ - suspend fun runTests(iosArgs: IosArgs): Pair>> = coroutineScope { - val (stopwatch, runGcsPath) = GenericTestRunner.beforeRunTests(iosArgs) - - val iosDeviceList = GcIosMatrix.build(iosArgs.devices) - - val xcTestParsed = Xctestrun.parse(iosArgs.xctestrunFile) - - val jobs = arrayListOf>() - val runCount = iosArgs.repeatTests - val shardCounter = ShardCounter() - val history = GcToolResults.createToolResultsHistory(iosArgs) - - // Upload only after parsing shards to detect missing methods early. - val xcTestGcsPath = if (iosArgs.xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { - iosArgs.xctestrunZip - } else { - GcStorage.uploadXCTestZip(iosArgs, runGcsPath) - } - - println(beforeRunMessage(iosArgs, iosArgs.testShardChunks)) - repeat(runCount) { - iosArgs.testShardChunks.forEach { testTargets -> - jobs += async(Dispatchers.IO) { - GcIosTestMatrix.build( - iosDeviceList = iosDeviceList, - testZipGcsPath = xcTestGcsPath, - runGcsPath = runGcsPath, - testTargets = testTargets, - xcTestParsed = xcTestParsed, - args = iosArgs, - shardCounter = shardCounter, - toolResultsHistory = history - ).executeWithRetry() - } - } - } - - val matrixMap = GenericTestRunner.afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, iosArgs) - matrixMap to iosArgs.testShardChunks - } -} diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt new file mode 100644 index 0000000000..9c4599a566 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt @@ -0,0 +1,33 @@ +package ftl.run + +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.args.IosArgs +import ftl.reports.util.ReportManager +import ftl.run.model.TestResult +import ftl.run.common.fetchArtifacts +import ftl.run.common.pollMatrices +import ftl.run.platform.runAndroidTests +import ftl.run.platform.runIosTests +import kotlin.system.exitProcess + +suspend fun newTestRun(args: IArgs) { + println(args) + val (matrixMap, testShardChunks) = runTests(args) + + if (!args.async) { + pollMatrices(matrixMap, args) + fetchArtifacts(matrixMap, args) + + val exitCode = ReportManager.generate(matrixMap, args, testShardChunks) + exitProcess(exitCode) + } +} + +private suspend fun runTests(args: IArgs): TestResult { + return when (args) { + is AndroidArgs -> runAndroidTests(args) + is IosArgs -> runIosTests(args) + else -> throw RuntimeException("Unknown config type") + } +} diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt new file mode 100644 index 0000000000..5710750302 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt @@ -0,0 +1,70 @@ +package ftl.run + +import com.google.api.services.testing.model.TestMatrix +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.gc.GcTestMatrix +import ftl.json.MatrixMap +import ftl.reports.util.ReportManager +import ftl.run.common.fetchArtifacts +import ftl.run.common.getLastArgs +import ftl.run.common.getLastMatrices +import ftl.run.common.pollMatrices +import ftl.run.common.updateMatrixFile +import ftl.args.ShardChunks +import ftl.util.MatrixState +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlin.system.exitProcess + +// used to update and poll the results from an async run +suspend fun refreshLastRun(currentArgs: IArgs, testShardChunks: ShardChunks) { + val matrixMap = getLastMatrices(currentArgs) + val lastArgs = getLastArgs(currentArgs) + + refreshMatrices(matrixMap, lastArgs) + pollMatrices(matrixMap, lastArgs) + fetchArtifacts(matrixMap, lastArgs) + + // Must generate reports *after* fetching xml artifacts since reports require xml + val exitCode = ReportManager.generate(matrixMap, lastArgs, testShardChunks) + exitProcess(exitCode) +} + +/** Refresh all in progress matrices in parallel **/ +private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = coroutineScope { + println("RefreshMatrices") + + val jobs = arrayListOf>() + val map = matrixMap.map + var matrixCount = 0 + map.forEach { matrix -> + // Only refresh unfinished + if (MatrixState.inProgress(matrix.value.state)) { + matrixCount += 1 + jobs += async(Dispatchers.IO) { GcTestMatrix.refresh(matrix.key, args) } + } + } + + if (matrixCount != 0) { + println(FtlConstants.indent + "Refreshing ${matrixCount}x matrices") + } + + var dirty = false + jobs.awaitAll().forEach { matrix -> + val matrixId = matrix.testMatrixId + + println(FtlConstants.indent + "${matrix.state} $matrixId") + + if (map[matrixId]?.update(matrix) == true) dirty = true + } + + if (dirty) { + println(FtlConstants.indent + "Updating matrix file") + updateMatrixFile(matrixMap, args) + } + println() +} diff --git a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt deleted file mode 100644 index 1694a62ebf..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt +++ /dev/null @@ -1,355 +0,0 @@ -package ftl.run - -import com.google.api.services.testing.model.TestMatrix -import com.google.cloud.storage.Storage -import com.google.gson.GsonBuilder -import com.google.gson.reflect.TypeToken -import ftl.args.AndroidArgs -import ftl.args.IArgs -import ftl.args.IosArgs -import ftl.config.FtlConstants -import ftl.config.FtlConstants.defaultAndroidConfig -import ftl.config.FtlConstants.defaultIosConfig -import ftl.config.FtlConstants.indent -import ftl.config.FtlConstants.localhost -import ftl.config.FtlConstants.matrixIdsFile -import ftl.gc.GcStorage -import ftl.gc.GcTestMatrix -import ftl.gc.GcTesting -import ftl.gc.GcToolResults -import ftl.json.MatrixMap -import ftl.json.SavedMatrix -import ftl.reports.util.ReportManager -import ftl.util.Artifacts -import ftl.util.MatrixState -import ftl.util.ObjPath -import ftl.util.StopWatch -import ftl.util.StopWatchMatrix -import ftl.util.completed -import ftl.util.fatalError -import ftl.util.sleep -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlin.system.exitProcess - -object TestRunner { - val gson = GsonBuilder().setPrettyPrinting().create()!! - - fun assertMockUrl() { - if (!FtlConstants.useMock) return - if (!GcTesting.get.rootUrl.contains(localhost)) throw RuntimeException("expected localhost in GcTesting") - if (!GcStorage.storageOptions.host.contains(localhost)) throw RuntimeException("expected localhost in GcStorage") - if (!GcToolResults.service.rootUrl.contains(localhost)) throw RuntimeException("expected localhost in GcToolResults") - } - - private suspend fun runTests(args: IArgs): Pair>> { - return when (args) { - is AndroidArgs -> AndroidTestRunner.runTests(args) - is IosArgs -> IosTestRunner.runTests(args) - else -> throw RuntimeException("Unknown config type") - } - } - - fun updateMatrixFile(matrixMap: MatrixMap, args: IArgs): Path { - val matrixIdsPath = if (args.useLocalResultDir()) { - Paths.get(args.localResultDir, matrixIdsFile) - } else { - Paths.get(args.localResultDir, matrixMap.runPath, matrixIdsFile) - } - matrixIdsPath.parent.toFile().mkdirs() - Files.write(matrixIdsPath, gson.toJson(matrixMap.map).toByteArray()) - return matrixIdsPath - } - - fun saveConfigFile(matrixMap: MatrixMap, args: IArgs): Path? { - val configFilePath = if (args.useLocalResultDir()) { - Paths.get(args.localResultDir, FtlConstants.configFileName(args)) - } else { - Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.configFileName(args)) - } - - configFilePath.parent.toFile().mkdirs() - Files.write(configFilePath, args.data.toByteArray()) - return configFilePath - } - - /** Refresh all in progress matrices in parallel **/ - private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = coroutineScope { - println("RefreshMatrices") - - val jobs = arrayListOf>() - val map = matrixMap.map - var matrixCount = 0 - map.forEach { matrix -> - // Only refresh unfinished - if (MatrixState.inProgress(matrix.value.state)) { - matrixCount += 1 - jobs += async(Dispatchers.IO) { GcTestMatrix.refresh(matrix.key, args) } - } - } - - if (matrixCount != 0) { - println(indent + "Refreshing ${matrixCount}x matrices") - } - - var dirty = false - jobs.awaitAll().forEach { matrix -> - val matrixId = matrix.testMatrixId - - println(indent + "${matrix.state} $matrixId") - - if (map[matrixId]?.update(matrix) == true) dirty = true - } - - if (dirty) { - println(indent + "Updating matrix file") - updateMatrixFile(matrixMap, args) - } - println() - } - - /** Cancel all in progress matrices in parallel **/ - private fun cancelMatrices(matrixMap: MatrixMap, args: IArgs) { - println("CancelMatrices") - - val map = matrixMap.map - var matrixCount = 0 - - runBlocking { - map.forEach { matrix -> - // Only cancel unfinished - if (MatrixState.inProgress(matrix.value.state)) { - matrixCount += 1 - launch { GcTestMatrix.cancel(matrix.key, args) } - } - } - } - - if (matrixCount == 0) { - println(indent + "No matrices to cancel") - } else { - println(indent + "Cancelling ${matrixCount}x matrices") - } - - println() - } - - private fun lastGcsPath(args: IArgs): String? { - val resultsFile = Paths.get(args.localResultDir).toFile() - if (!resultsFile.exists()) return null - - val scheduledRuns = resultsFile.listFiles() - .filter { it.isDirectory } - .sortedByDescending { it.lastModified() } - if (scheduledRuns.isEmpty()) return null - - return scheduledRuns.first().name - } - - /** Reads in the last matrices from the localResultDir folder **/ - private fun lastArgs(args: IArgs): IArgs { - val lastRun = lastGcsPath(args) - - if (lastRun == null) { - fatalError("no runs found in results/ folder") - } - - val iosConfig = Paths.get(args.localResultDir, lastRun, defaultIosConfig) - val androidConfig = Paths.get(args.localResultDir, lastRun, defaultAndroidConfig) - - when { - iosConfig.toFile().exists() -> return IosArgs.load(iosConfig) - androidConfig.toFile().exists() -> return AndroidArgs.load(androidConfig) - else -> fatalError("No config file found in the last run folder: $lastRun") - } - - throw RuntimeException("should not happen") - } - - /** Reads in the last matrices from the localResultDir folder **/ - private fun lastMatrices(args: IArgs): MatrixMap { - val lastRun = lastGcsPath(args) - - if (lastRun == null) { - fatalError("no runs found in results/ folder") - throw RuntimeException("fatalError failed to exit the process") - } - - println("Loading run $lastRun") - return matrixPathToObj(lastRun, args) - } - - /** Creates MatrixMap from matrix_ids.json file */ - fun matrixPathToObj(path: String, args: IArgs): MatrixMap { - var filePath = Paths.get(path, matrixIdsFile).toFile() - if (!filePath.exists()) { - filePath = Paths.get(args.localResultDir, path, matrixIdsFile).toFile() - } - val json = filePath.readText() - - val listOfSavedMatrix = object : TypeToken>() {}.type - val map: MutableMap = gson.fromJson(json, listOfSavedMatrix) - - return MatrixMap(map, path) - } - - fun getDownloadPath(args: IArgs, blobPath: String): Path { - val localDir = args.localResultDir - val p = if (args is AndroidArgs) - ObjPath.parse(blobPath) else - ObjPath.legacyParse(blobPath) - - // Store downloaded artifacts at device root. - return if (args.useLocalResultDir()) { - if (args is AndroidArgs && args.keepFilePath) - Paths.get(localDir, p.shardName, p.deviceName, p.filePathName, p.fileName) - else - Paths.get(localDir, p.shardName, p.deviceName, p.fileName) - } else { - if (args is AndroidArgs && args.keepFilePath) - Paths.get(localDir, p.objName, p.shardName, p.deviceName, p.filePathName, p.fileName) - else - Paths.get(localDir, p.objName, p.shardName, p.deviceName, p.fileName) - } - } - - private fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) { - println("FetchArtifacts") - val fields = Storage.BlobListOption.fields(Storage.BlobField.NAME) - - var dirty = false - val filtered = matrixMap.map.values.filter { - val finished = it.state == MatrixState.FINISHED - val notDownloaded = !it.downloaded - finished && notDownloaded - } - - print(indent) - runBlocking { - filtered.forEach { matrix -> - launch { - val prefix = Storage.BlobListOption.prefix(matrix.gcsPathWithoutRootBucket) - val result = GcStorage.storage.list(matrix.gcsRootBucket, prefix, fields) - val artifactsList = Artifacts.regexList(args) - - result.iterateAll().forEach { blob -> - val blobPath = blob.blobId.name - val matched = artifactsList.any { blobPath.matches(it) } - if (matched) { - val downloadFile = getDownloadPath(args, blobPath) - - print(".") - if (!downloadFile.toFile().exists()) { - val parentFile = downloadFile.parent.toFile() - parentFile.mkdirs() - blob.downloadTo(downloadFile) - } - } - } - - dirty = true - matrix.downloaded = true - } - } - } - println() - - if (dirty) { - println(indent + "Updating matrix file") - updateMatrixFile(matrixMap, args) - println() - } - } - - /** Synchronously poll all matrix ids until they complete. Returns true if test run passed. **/ - private fun pollMatrices(matrices: MatrixMap, args: IArgs) { - println("PollMatrices") - val poll = matrices.map.values.filter { - MatrixState.inProgress(it.state) - } - - val stopwatch = StopWatch().start() - poll.forEach { - val matrixId = it.matrixId - pollMatrix(matrixId, stopwatch, args, matrices) - } - println() - - updateMatrixFile(matrices, args) - } - - // Used for when the matrix has exactly one test. Polls for detailed progress - // - // Port of MonitorTestExecutionProgress - // gcloud-cli/googlecloudsdk/api_lib/firebase/test/matrix_ops.py - private fun pollMatrix(matrixId: String, stopwatch: StopWatch, args: IArgs, matrices: MatrixMap) { - var refreshedMatrix = GcTestMatrix.refresh(matrixId, args) - val watch = StopWatchMatrix(stopwatch, matrixId) - val runningDevices = RunningDevices(stopwatch, refreshedMatrix.testExecutions) - - while (true) { - if (matrices.map[matrixId]?.update(refreshedMatrix) == true) updateMatrixFile(matrices, args) - - runningDevices.allRunning().forEach { nextDevice -> - nextDevice.poll(refreshedMatrix) - } - - // Matrix has 0 or more devices (test executions) - // Verify all executions are complete & the matrix itself is marked as complete. - if (runningDevices.allComplete() && refreshedMatrix.completed()) { - break - } - - // GetTestMatrix is not designed to handle many requests per second. - // Sleep to avoid overloading the system. - sleep(5) - refreshedMatrix = GcTestMatrix.refresh(matrixId, args) - } - - // Print final matrix state with timestamp. May be many minutes after the 'Done.' progress message. - watch.puts(refreshedMatrix.state) - } - - // used to update and poll the results from an async run - suspend fun refreshLastRun(currentArgs: IArgs, testShardChunks: List>) { - val matrixMap = lastMatrices(currentArgs) - val lastArgs = lastArgs(currentArgs) - - refreshMatrices(matrixMap, lastArgs) - pollMatrices(matrixMap, lastArgs) - fetchArtifacts(matrixMap, lastArgs) - - // Must generate reports *after* fetching xml artifacts since reports require xml - val exitCode = ReportManager.generate(matrixMap, lastArgs, testShardChunks) - exitProcess(exitCode) - } - - // used to cancel and update results from an async run - fun cancelLastRun(args: IArgs) { - val matrixMap = lastMatrices(args) - val lastArgs = lastArgs(args) - - cancelMatrices(matrixMap, lastArgs) - } - - suspend fun newRun(args: IArgs) { - println(args) - val (matrixMap, testShardChunks) = runTests(args) - - if (!args.async) { - pollMatrices(matrixMap, args) - fetchArtifacts(matrixMap, args) - - val exitCode = ReportManager.generate(matrixMap, args, testShardChunks) - exitProcess(exitCode) - } - } -} diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt new file mode 100644 index 0000000000..92c6d7e267 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt @@ -0,0 +1,83 @@ +package ftl.run.common + +import com.google.cloud.storage.Storage +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.gc.GcStorage +import ftl.json.MatrixMap +import ftl.util.Artifacts +import ftl.util.MatrixState +import ftl.util.ObjPath +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import java.nio.file.Path +import java.nio.file.Paths + +internal fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) { + println("FetchArtifacts") + val fields = Storage.BlobListOption.fields(Storage.BlobField.NAME) + + var dirty = false + val filtered = matrixMap.map.values.filter { + val finished = it.state == MatrixState.FINISHED + val notDownloaded = !it.downloaded + finished && notDownloaded + } + + print(FtlConstants.indent) + runBlocking { + filtered.forEach { matrix -> + launch { + val prefix = Storage.BlobListOption.prefix(matrix.gcsPathWithoutRootBucket) + val result = GcStorage.storage.list(matrix.gcsRootBucket, prefix, fields) + val artifactsList = Artifacts.regexList(args) + + result.iterateAll().forEach { blob -> + val blobPath = blob.blobId.name + val matched = artifactsList.any { blobPath.matches(it) } + if (matched) { + val downloadFile = getDownloadPath(args, blobPath) + + print(".") + if (!downloadFile.toFile().exists()) { + val parentFile = downloadFile.parent.toFile() + parentFile.mkdirs() + blob.downloadTo(downloadFile) + } + } + } + + dirty = true + matrix.downloaded = true + } + } + } + println() + + if (dirty) { + println(FtlConstants.indent + "Updating matrix file") + updateMatrixFile(matrixMap, args) + println() + } +} + +internal fun getDownloadPath(args: IArgs, blobPath: String): Path { + val localDir = args.localResultDir + val p = if (args is AndroidArgs) + ObjPath.parse(blobPath) else + ObjPath.legacyParse(blobPath) + + // Store downloaded artifacts at device root. + return if (args.useLocalResultDir()) { + if (args is AndroidArgs && args.keepFilePath) + Paths.get(localDir, p.shardName, p.deviceName, p.filePathName, p.fileName) + else + Paths.get(localDir, p.shardName, p.deviceName, p.fileName) + } else { + if (args is AndroidArgs && args.keepFilePath) + Paths.get(localDir, p.objName, p.shardName, p.deviceName, p.filePathName, p.fileName) + else + Paths.get(localDir, p.objName, p.shardName, p.deviceName, p.fileName) + } +} diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt new file mode 100644 index 0000000000..b570da7601 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt @@ -0,0 +1,28 @@ +package ftl.run.common + +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.args.IosArgs +import ftl.config.FtlConstants +import ftl.util.fatalError +import java.nio.file.Paths + +/** Reads in the last matrices from the localResultDir folder **/ +internal fun getLastArgs(args: IArgs): IArgs { + val lastRun = args.getLastGcsPath() + + if (lastRun == null) { + fatalError("no runs found in results/ folder") + } + + val iosConfig = Paths.get(args.localResultDir, lastRun, FtlConstants.defaultIosConfig) + val androidConfig = Paths.get(args.localResultDir, lastRun, FtlConstants.defaultAndroidConfig) + + when { + iosConfig.toFile().exists() -> return IosArgs.load(iosConfig) + androidConfig.toFile().exists() -> return AndroidArgs.load(androidConfig) + else -> fatalError("No config file found in the last run folder: $lastRun") + } + + throw RuntimeException("should not happen") +} diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastGcsPath.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastGcsPath.kt new file mode 100644 index 0000000000..8ba9a4e1bd --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastGcsPath.kt @@ -0,0 +1,16 @@ +package ftl.run.common + +import ftl.args.IArgs +import java.io.File +import java.nio.file.Paths + +internal fun IArgs.getLastGcsPath(): String? = Paths + .get(localResultDir) + .toFile() + .listFiles() + ?.getLastModifiedDirectory() + ?.name + +private fun Array.getLastModifiedDirectory(): File? = this + .filter(File::isDirectory) + .maxBy(File::lastModified) diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt new file mode 100644 index 0000000000..f87c806362 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt @@ -0,0 +1,36 @@ +package ftl.run.common + +import com.google.gson.reflect.TypeToken +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.json.MatrixMap +import ftl.json.SavedMatrix +import ftl.util.fatalError +import java.nio.file.Paths + +/** Reads in the last matrices from the localResultDir folder **/ +internal fun getLastMatrices(args: IArgs): MatrixMap { + val lastRun = args.getLastGcsPath() + + if (lastRun == null) { + fatalError("no runs found in results/ folder") + throw RuntimeException("fatalError failed to exit the process") + } + + println("Loading run $lastRun") + return matrixPathToObj(lastRun, args) +} + +/** Creates MatrixMap from matrix_ids.json file */ +internal fun matrixPathToObj(path: String, args: IArgs): MatrixMap { + var filePath = Paths.get(path, FtlConstants.matrixIdsFile).toFile() + if (!filePath.exists()) { + filePath = Paths.get(args.localResultDir, path, FtlConstants.matrixIdsFile).toFile() + } + val json = filePath.readText() + + val listOfSavedMatrix = object : TypeToken>() {}.type + val map: MutableMap = prettyPrint.fromJson(json, listOfSavedMatrix) + + return MatrixMap(map, path) +} diff --git a/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt new file mode 100644 index 0000000000..a133e8ae44 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt @@ -0,0 +1,63 @@ +package ftl.run.common + +import ftl.args.IArgs +import ftl.gc.GcTestMatrix +import ftl.json.MatrixMap +import ftl.run.model.RunningDevices +import ftl.util.MatrixState +import ftl.util.StopWatch +import ftl.util.StopWatchMatrix +import ftl.util.completed +import ftl.util.sleep + +/** Synchronously poll all matrix ids until they complete. Returns true if test run passed. **/ +internal fun pollMatrices(matrices: MatrixMap, args: IArgs) { + println("PollMatrices") + val poll = matrices.map.values.filter { + MatrixState.inProgress(it.state) + } + + val stopwatch = StopWatch().start() + poll.forEach { + val matrixId = it.matrixId + pollMatrix(matrixId, stopwatch, args, matrices) + } + println() + + updateMatrixFile(matrices, args) +} + +// Used for when the matrix has exactly one test. Polls for detailed progress +// +// Port of MonitorTestExecutionProgress +// gcloud-cli/googlecloudsdk/api_lib/firebase/test/matrix_ops.py +private fun pollMatrix(matrixId: String, stopwatch: StopWatch, args: IArgs, matrices: MatrixMap) { + var refreshedMatrix = GcTestMatrix.refresh(matrixId, args) + val watch = StopWatchMatrix(stopwatch, matrixId) + val runningDevices = RunningDevices(stopwatch, refreshedMatrix.testExecutions) + + while (true) { + if (matrices.map[matrixId]?.update(refreshedMatrix) == true) updateMatrixFile( + matrices, + args + ) + + runningDevices.allRunning().forEach { nextDevice -> + nextDevice.poll(refreshedMatrix) + } + + // Matrix has 0 or more devices (test executions) + // Verify all executions are complete & the matrix itself is marked as complete. + if (runningDevices.allComplete() && refreshedMatrix.completed()) { + break + } + + // GetTestMatrix is not designed to handle many requests per second. + // Sleep to avoid overloading the system. + sleep(5) + refreshedMatrix = GcTestMatrix.refresh(matrixId, args) + } + + // Print final matrix state with timestamp. May be many minutes after the 'Done.' progress message. + watch.puts(refreshedMatrix.state) +} diff --git a/test_runner/src/main/kotlin/ftl/run/common/PrettyPrint.kt b/test_runner/src/main/kotlin/ftl/run/common/PrettyPrint.kt new file mode 100644 index 0000000000..0182afe36b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/PrettyPrint.kt @@ -0,0 +1,6 @@ +package ftl.run.common + +import com.google.gson.Gson +import com.google.gson.GsonBuilder + +val prettyPrint: Gson = GsonBuilder().setPrettyPrinting().create()!! diff --git a/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt b/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt new file mode 100644 index 0000000000..289a5abda9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt @@ -0,0 +1,19 @@ +package ftl.run.common + +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.json.MatrixMap +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +internal fun updateMatrixFile(matrixMap: MatrixMap, args: IArgs): Path { + val matrixIdsPath = if (args.useLocalResultDir()) { + Paths.get(args.localResultDir, FtlConstants.matrixIdsFile) + } else { + Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.matrixIdsFile) + } + matrixIdsPath.parent.toFile().mkdirs() + Files.write(matrixIdsPath, prettyPrint.toJson(matrixMap.map).toByteArray()) + return matrixIdsPath +} diff --git a/test_runner/src/main/kotlin/ftl/run/RunningDevices.kt b/test_runner/src/main/kotlin/ftl/run/model/RunningDevices.kt similarity index 99% rename from test_runner/src/main/kotlin/ftl/run/RunningDevices.kt rename to test_runner/src/main/kotlin/ftl/run/model/RunningDevices.kt index e3c802897c..2cebe63eeb 100644 --- a/test_runner/src/main/kotlin/ftl/run/RunningDevices.kt +++ b/test_runner/src/main/kotlin/ftl/run/model/RunningDevices.kt @@ -1,4 +1,4 @@ -package ftl.run +package ftl.run.model import com.google.api.services.testing.model.Environment import com.google.api.services.testing.model.TestDetails diff --git a/test_runner/src/main/kotlin/ftl/run/model/TestResult.kt b/test_runner/src/main/kotlin/ftl/run/model/TestResult.kt new file mode 100644 index 0000000000..865775095b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/model/TestResult.kt @@ -0,0 +1,6 @@ +package ftl.run.model + +import ftl.json.MatrixMap +import ftl.args.ShardChunks + +internal typealias TestResult = Pair diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt new file mode 100644 index 0000000000..6f7bf31aea --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -0,0 +1,81 @@ +package ftl.run.platform + +import com.google.api.services.testing.model.TestMatrix +import ftl.args.AndroidArgs +import ftl.args.AndroidTestShard +import ftl.args.yml.AppTestPair +import ftl.gc.GcAndroidDevice +import ftl.gc.GcAndroidTestMatrix +import ftl.gc.GcStorage +import ftl.gc.GcToolResults +import ftl.http.executeWithRetry +import ftl.args.ShardChunks +import ftl.run.model.TestResult +import ftl.run.platform.common.afterRunTests +import ftl.run.platform.common.beforeRunMessage +import ftl.run.platform.common.beforeRunTests +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineScope { + val (stopwatch, runGcsPath) = beforeRunTests(args) + + // GcAndroidMatrix => GcAndroidTestMatrix + // GcAndroidTestMatrix.execute() 3x retry => matrix id (string) + val androidDeviceList = GcAndroidDevice.build(args.devices) + + val jobs = arrayListOf>() + val runCount = args.repeatTests + val history = GcToolResults.createToolResultsHistory(args) + val apkPairsInArgs = listOf(AppTestPair(app = args.appApk, test = args.testApk)) + args.additionalAppTestApks + val allTestShardChunks: ShardChunks = apkPairsInArgs.map { unresolvedApkPair -> + val resolvedApkPair = resolveApkPair(unresolvedApkPair, args, runGcsPath) + // Ensure we only shard tests that are part of the test apk. Use the unresolved test apk path to make sure + // we don't re-download an apk it is on the local file system. + AndroidTestShard.getTestShardChunks(args, unresolvedApkPair.test).also { testShards -> + repeat(runCount) { + // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread + // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html + jobs += async(Dispatchers.IO) { + GcAndroidTestMatrix.build( + appApkGcsPath = resolvedApkPair.app, + testApkGcsPath = resolvedApkPair.test, + runGcsPath = runGcsPath, + androidDeviceList = androidDeviceList, + testShards = testShards, + args = args, + toolResultsHistory = history + ).executeWithRetry() + } + } + } + }.flatten() + + println(beforeRunMessage(args, allTestShardChunks)) + val matrixMap = afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, args) + matrixMap to allTestShardChunks +} + +/** + * Upload an APK pair if the path given is local + * + * @return AppTestPair with their GCS paths + */ +private suspend fun resolveApkPair( + apk: AppTestPair, + args: AndroidArgs, + runGcsPath: String +): AppTestPair = coroutineScope { + val gcsBucket = args.resultsBucket + + val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.app, gcsBucket, runGcsPath) } + val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apk.test, gcsBucket, runGcsPath) } + + AppTestPair( + app = appApkGcsPath.await(), + test = testApkGcsPath.await() + ) +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt new file mode 100644 index 0000000000..f43f3fdab5 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -0,0 +1,71 @@ +package ftl.run.platform + +import com.google.api.services.testing.model.TestMatrix +import ftl.args.IosArgs +import ftl.config.FtlConstants +import ftl.gc.GcIosMatrix +import ftl.gc.GcIosTestMatrix +import ftl.gc.GcStorage +import ftl.gc.GcToolResults +import ftl.http.executeWithRetry +import ftl.ios.Xctestrun +import ftl.run.model.TestResult +import ftl.run.platform.common.afterRunTests +import ftl.run.platform.common.beforeRunMessage +import ftl.run.platform.common.beforeRunTests +import ftl.util.ShardCounter +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +// https://github.com/bootstraponline/gcloud_cli/blob/5bcba57e825fc98e690281cf69484b7ba4eb668a/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py#L109 +// https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run +// https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/ +internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope { + val (stopwatch, runGcsPath) = beforeRunTests(iosArgs) + + val iosDeviceList = GcIosMatrix.build(iosArgs.devices) + + val xcTestParsed = Xctestrun.parse(iosArgs.xctestrunFile) + + val jobs = arrayListOf>() + val runCount = iosArgs.repeatTests + val shardCounter = ShardCounter() + val history = GcToolResults.createToolResultsHistory(iosArgs) + + // Upload only after parsing shards to detect missing methods early. + val xcTestGcsPath = getXcTestGcPath(iosArgs, runGcsPath) + + println(beforeRunMessage(iosArgs, iosArgs.testShardChunks)) + + repeat(runCount) { + jobs += iosArgs.testShardChunks.map { testTargets -> + async(Dispatchers.IO) { + GcIosTestMatrix.build( + iosDeviceList = iosDeviceList, + testZipGcsPath = xcTestGcsPath, + runGcsPath = runGcsPath, + testTargets = testTargets, + xcTestParsed = xcTestParsed, + args = iosArgs, + shardCounter = shardCounter, + toolResultsHistory = history + ).executeWithRetry() + } + } + } + + val matrixMap = afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, iosArgs) + matrixMap to iosArgs.testShardChunks +} + +private fun getXcTestGcPath( + iosArgs: IosArgs, + runGcsPath: String +) = if (iosArgs.xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { + iosArgs.xctestrunZip +} else { + GcStorage.uploadXCTestZip(iosArgs, runGcsPath) +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt new file mode 100644 index 0000000000..266e235d2b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -0,0 +1,48 @@ +package ftl.run.platform.common + +import com.google.api.services.testing.model.TestMatrix +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.json.MatrixMap +import ftl.json.SavedMatrix +import ftl.run.common.updateMatrixFile +import ftl.util.StopWatch +import java.nio.file.Files +import java.nio.file.Paths + +internal fun afterRunTests( + jobs: List, + runGcsPath: String, + stopwatch: StopWatch, + config: IArgs +): MatrixMap { + val savedMatrices = mutableMapOf() + + jobs.forEach { matrix -> + val matrixId = matrix.testMatrixId + savedMatrices[matrixId] = SavedMatrix(matrix) + } + + val matrixMap = MatrixMap(savedMatrices, runGcsPath) + updateMatrixFile(matrixMap, config) + saveConfigFile(matrixMap, config) + + println(FtlConstants.indent + "${savedMatrices.size} matrix ids created in ${stopwatch.check()}") + val gcsBucket = "https://console.developers.google.com/storage/browser/" + + config.resultsBucket + "/" + matrixMap.runPath + println(FtlConstants.indent + gcsBucket) + println() + + return matrixMap +} + +private fun saveConfigFile(matrixMap: MatrixMap, args: IArgs) { + val configFilePath = if (args.useLocalResultDir()) { + Paths.get(args.localResultDir, FtlConstants.configFileName(args)) + } else { + Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.configFileName(args)) + } + + configFilePath.parent.toFile().mkdirs() + Files.write(configFilePath, args.data.toByteArray()) +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt new file mode 100644 index 0000000000..6b71befb30 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt @@ -0,0 +1,35 @@ +package ftl.run.platform.common + +import ftl.args.IArgs +import ftl.args.ShardChunks + +internal fun beforeRunMessage(args: IArgs, testShardChunks: ShardChunks): String { + val runCount = args.repeatTests + val shardCount = testShardChunks.size + val testsCount = testShardChunks.sumBy { it.size } + + val result = StringBuilder() + result.appendln( + " $testsCount test${s(testsCount)} / $shardCount shard${s( + shardCount + )}" + ) + + if (runCount > 1) { + result.appendln(" Running ${runCount}x") + val runDevices = runCount * shardCount + val runTests = runCount * testsCount + result.appendln(" $runDevices total shard${s(runDevices)}") + result.appendln(" $runTests total test${s(runTests)}") + } + + return result.toString() +} + +private fun s(amount: Int): String { + return if (amount > 1) { + "s" + } else { + "" + } +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt new file mode 100644 index 0000000000..ef9e7b74f7 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt @@ -0,0 +1,47 @@ +package ftl.run.platform.common + +import ftl.args.IArgs +import ftl.config.FtlConstants +import ftl.gc.GcStorage +import ftl.gc.GcTesting +import ftl.gc.GcToolResults +import ftl.util.StopWatch +import ftl.util.uniqueObjectName +import java.io.File + +internal fun beforeRunTests(args: IArgs): Pair { + println("RunTests") + assertMockUrl() + + val stopwatch = StopWatch().start() + val runGcsPath = args.resultsDir ?: uniqueObjectName() + + // Avoid spamming the results/ dir with temporary files from running the test suite. + if (FtlConstants.useMock) + deleteMockResultDirOnShotDown(args, runGcsPath) + + if (args.useLocalResultDir()) { + // Only one result is stored when using --local-result-dir + // Delete any old results if they exist before storing new ones. + deleteLocalResultDir(args) + } + + return stopwatch to runGcsPath +} + +private fun assertMockUrl() { + if (!FtlConstants.useMock) return + if (!GcTesting.get.rootUrl.contains(FtlConstants.localhost)) throw RuntimeException("expected localhost in GcTesting") + if (!GcStorage.storageOptions.host.contains(FtlConstants.localhost)) throw RuntimeException("expected localhost in GcStorage") + if (!GcToolResults.service.rootUrl.contains(FtlConstants.localhost)) throw RuntimeException("expected localhost in GcToolResults") +} + +private fun deleteMockResultDirOnShotDown(args: IArgs, runGcsPath: String) { + Runtime.getRuntime().addShutdownHook(Thread { + File(args.localResultDir, runGcsPath).deleteRecursively() + }) +} + +private fun deleteLocalResultDir(args: IArgs) { + File(args.localResultDir).deleteRecursively() +} diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index 470b74439d..4515e663f1 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -10,17 +10,19 @@ fun main() { // run "gradle check" to generate required fixtures val projectId = System.getenv("FLANK_PROJECT_ID") ?: "YOUR PROJECT ID" - val quantity = "single" - val type = "success" + val quantity = "multiple" + val type = "flaky" // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 - jvmHangingSafe { CommandLine(Main()).execute( - "--debug", - "firebase", "test", - "android", "run", - "--dry", - "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", - "--project=$projectId" - ) } + jvmHangingSafe { + CommandLine(Main()).execute( +// "--debug", + "firebase", "test", + "android", "run", +// "--dry", + "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", + "--project=$projectId" + ) + } } diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index 4f89ad7828..536cb5b6b9 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -21,7 +21,7 @@ class GcAndroidTestMatrixTest { testApkGcsPath = "", runGcsPath = "", androidDeviceList = AndroidDeviceList(), - testTargets = emptyList(), + testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs) ) @@ -35,7 +35,7 @@ class GcAndroidTestMatrixTest { testApkGcsPath = "", runGcsPath = "", androidDeviceList = AndroidDeviceList(), - testTargets = listOf(listOf("")), + testShards = listOf(listOf("")), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs) ) @@ -53,7 +53,7 @@ class GcAndroidTestMatrixTest { testApkGcsPath = "", runGcsPath = "", androidDeviceList = AndroidDeviceList(), - testTargets = emptyList(), + testShards = emptyList(), args = androidArgs, toolResultsHistory = createToolResultsHistory(androidArgs) ) diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index d9ccd60997..28fd7be820 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -6,7 +6,7 @@ import ftl.reports.util.ReportManager import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestResult import ftl.reports.xml.model.JUnitTestSuite -import ftl.run.TestRunner +import ftl.run.common.matrixPathToObj import ftl.test.util.FlankTestRunner import org.junit.Rule import org.junit.Test @@ -30,7 +30,7 @@ class ReportManagerTest { @Test fun `generate fromErrorResult`() { // TODO: NPE on Windows - val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result", AndroidArgs.default()) + val matrix = matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result", AndroidArgs.default()) val mockArgs = mock(AndroidArgs::class.java) `when`(mockArgs.smartFlankGcsPath).thenReturn("") ReportManager.generate(matrix, mockArgs, emptyList()) @@ -38,7 +38,7 @@ class ReportManagerTest { @Test fun `generate fromSuccessResult`() { - val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result", AndroidArgs.default()) + val matrix = matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result", AndroidArgs.default()) val mockArgs = mock(AndroidArgs::class.java) `when`(mockArgs.smartFlankGcsPath).thenReturn("") ReportManager.generate(matrix, mockArgs, emptyList()) diff --git a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt index ba643ada4d..21a91e7310 100644 --- a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt @@ -1,7 +1,7 @@ package ftl.run import ftl.args.IArgs -import ftl.run.GenericTestRunner.beforeRunMessage +import ftl.run.platform.common.beforeRunMessage import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.assert import org.junit.Test @@ -43,8 +43,8 @@ class GenericTestRunnerTest { fun testBeforeRunMessage3() { val result = beforeRunMessage( createMock(2), - listOf(listOf(""), listOf(""), listOf(""), listOf(""), listOf(""), listOf("")) - ).normalizeLineEnding() + listOf(listOf(""), listOf(""), listOf(""), listOf(""), listOf(""), listOf("")) + ).normalizeLineEnding() assert( result, """ 6 tests / 6 shards @@ -57,9 +57,13 @@ class GenericTestRunnerTest { @Test fun testBeforeRunMessage4() { - val result = beforeRunMessage(createMock(100), - listOf(listOf("", "", "", "", ""), - listOf("", "", "", "", ""))).normalizeLineEnding() + val result = beforeRunMessage( + createMock(100), + listOf( + listOf("", "", "", "", ""), + listOf("", "", "", "", "") + ) + ).normalizeLineEnding() assert( result, """ 10 tests / 2 shards diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 5e8ffdd32d..590844c3f3 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -4,6 +4,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.AndroidArgs import ftl.args.IosArgs import ftl.config.FtlConstants.isWindows +import ftl.run.common.getDownloadPath import ftl.test.util.FlankTestRunner import ftl.util.ObjPath import java.nio.file.Paths @@ -30,7 +31,7 @@ class TestRunnerTest { `when`(iosArgs.localResultDir).thenReturn(localResultDir) `when`(iosArgs.useLocalResultDir()).thenReturn(false) - val downloadFile = TestRunner.getDownloadPath(iosArgs, gcsIosPath) + val downloadFile = getDownloadPath(iosArgs, gcsIosPath) assertThat(downloadFile).isEqualTo( Paths.get( localResultDir, @@ -49,7 +50,7 @@ class TestRunnerTest { `when`(iosArgs.localResultDir).thenReturn(localResultDir) `when`(iosArgs.useLocalResultDir()).thenReturn(true) - val downloadFile = TestRunner.getDownloadPath(iosArgs, gcsIosPath) + val downloadFile = getDownloadPath(iosArgs, gcsIosPath) assertThat(downloadFile).isEqualTo( Paths.get( localResultDir, @@ -68,7 +69,7 @@ class TestRunnerTest { `when`(androidArgs.useLocalResultDir()).thenReturn(true) `when`(androidArgs.keepFilePath).thenReturn(true) - val downloadFile = TestRunner.getDownloadPath(androidArgs, gcsAndroidPath) + val downloadFile = getDownloadPath(androidArgs, gcsAndroidPath) assertThat(downloadFile).isEqualTo( Paths.get( localResultDir, @@ -88,7 +89,7 @@ class TestRunnerTest { `when`(androidArgs.useLocalResultDir()).thenReturn(false) `when`(androidArgs.keepFilePath).thenReturn(true) - val downloadFile = TestRunner.getDownloadPath(androidArgs, gcsAndroidPath) + val downloadFile = getDownloadPath(androidArgs, gcsAndroidPath) assertThat(downloadFile).isEqualTo( Paths.get( localResultDir, @@ -105,7 +106,7 @@ class TestRunnerTest { fun `mockedAndroidTestRun local`() { val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml")) runBlocking { - TestRunner.newRun(localConfig) + newTestRun(localConfig) } } @@ -113,7 +114,7 @@ class TestRunnerTest { fun `mockedAndroidTestRun gcsAndHistoryName`() { val gcsConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.gcs.yml")) runBlocking { - TestRunner.newRun(gcsConfig) + newTestRun(gcsConfig) } } @@ -123,7 +124,7 @@ class TestRunnerTest { val config = IosArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.ios.yml")) runBlocking { - TestRunner.newRun(config) + newTestRun(config) } } @@ -133,7 +134,7 @@ class TestRunnerTest { val config = IosArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml")) runBlocking { - TestRunner.newRun(config) + newTestRun(config) } } }