From f6c6d1301ad0150c9cfd0c6c7302cde1d5eeed9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 21 Nov 2020 01:34:49 +0100 Subject: [PATCH] * Simplify beforeRunTests return signature * Treat Args as context --- .../test/android/AndroidRunCommand.kt | 9 ++-- .../cli/firebase/test/ios/IosRunCommand.kt | 9 ++-- .../src/main/kotlin/ftl/run/DumpShards.kt | 21 ++++---- .../src/main/kotlin/ftl/run/NewTestRun.kt | 19 +++---- .../src/main/kotlin/ftl/run/RefreshLastRun.kt | 2 +- .../kotlin/ftl/run/common/FetchArtifacts.kt | 2 +- .../kotlin/ftl/run/common/UpdateMatrixFile.kt | 10 ++-- .../ftl/run/platform/RunAndroidTests.kt | 31 ++++++------ .../kotlin/ftl/run/platform/RunIosTests.kt | 49 ++++++++----------- .../ftl/run/platform/android/UploadApks.kt | 8 +-- .../run/platform/android/UploadOtherFiles.kt | 10 ++-- .../ftl/run/platform/common/AfterRunTests.kt | 24 +++++---- .../run/platform/common/BeforeRunMessage.kt | 4 +- .../ftl/run/platform/common/BeforeRunTests.kt | 30 +++++------- .../src/main/kotlin/ftl/util/FileReference.kt | 12 ++++- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 4 +- .../firebase/test/ios/IosRunCommandTest.kt | 4 +- .../test/kotlin/ftl/run/DumpShardsKtTest.kt | 25 +++++++--- .../kotlin/ftl/run/GenericTestRunnerTest.kt | 26 +++++----- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 12 ++--- .../ftl/run/platform/RunAndroidTestsKtTest.kt | 6 +-- 21 files changed, 157 insertions(+), 160 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index b99f6985a8..7da4f7478c 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -44,10 +44,11 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() - runBlocking { - if (dumpShards) dumpShards(args = config) - else newTestRun(config) + AndroidArgs.load(Paths.get(configPath), cli = this).validate().run { + runBlocking { + if (dumpShards) dumpShards() + else newTestRun() + } } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 966f7d3bdf..ef478fac98 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -44,12 +44,9 @@ class IosRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = IosArgs.load(Paths.get(configPath), cli = this).validate() - - if (dumpShards) { - dumpShards(args = config) - } else runBlocking { - newTestRun(config) + IosArgs.load(Paths.get(configPath), cli = this).validate().run { + if (dumpShards) dumpShards() + else runBlocking { newTestRun() } } } diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 840625208a..67e152a132 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -1,5 +1,6 @@ package ftl.run +import com.google.common.annotations.VisibleForTesting import ftl.args.AndroidArgs import ftl.args.IosArgs import ftl.args.isInstrumentationTest @@ -12,31 +13,31 @@ import ftl.util.obfuscatePrettyPrinter import java.nio.file.Files import java.nio.file.Paths -suspend fun dumpShards( - args: AndroidArgs, +suspend fun AndroidArgs.dumpShards( + @VisibleForTesting shardFilePath: String = ANDROID_SHARD_FILE, ) { - if (!args.isInstrumentationTest) throw FlankConfigurationError( + if (!isInstrumentationTest) throw FlankConfigurationError( "Cannot dump shards for non instrumentation test, ensure test apk has been set." ) - val shards: AndroidMatrixTestShards = args.getAndroidMatrixShards() + val shards: AndroidMatrixTestShards = getAndroidMatrixShards() saveShardChunks( shardFilePath = shardFilePath, shards = shards, size = shards.flatMap { it.value.shards.values }.count(), - obfuscatedOutput = args.obfuscateDumpShards + obfuscatedOutput = obfuscateDumpShards ) } -fun dumpShards( - args: IosArgs, +fun IosArgs.dumpShards( + @VisibleForTesting shardFilePath: String = IOS_SHARD_FILE, ) { saveShardChunks( shardFilePath = shardFilePath, - shards = args.testShardChunks.testCases, - size = args.testShardChunks.size, - obfuscatedOutput = args.obfuscateDumpShards + shards = testShardChunks.testCases, + size = testShardChunks.size, + obfuscatedOutput = obfuscateDumpShards ) } diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt index bbdb85b9e2..db360bc220 100644 --- a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt @@ -18,9 +18,11 @@ import ftl.run.platform.runIosTests import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeoutOrNull -suspend fun newTestRun(args: IArgs) = withTimeoutOrNull(args.parsedTimeout) { +suspend fun IArgs.newTestRun() = withTimeoutOrNull(parsedTimeout) { + val args: IArgs = this@newTestRun println(args) - val (matrixMap, testShardChunks, ignoredTests) = cancelTestsOnTimeout(args.project) { runTests(args) } + val (matrixMap, testShardChunks, ignoredTests) = + cancelTestsOnTimeout(project) { runTests() } if (!args.async) { cancelTestsOnTimeout(args.project, matrixMap.map) { pollMatrices(matrixMap.map.keys, args).updateMatrixMap(matrixMap) } @@ -28,19 +30,18 @@ suspend fun newTestRun(args: IArgs) = withTimeoutOrNull(args.parsedTimeout) { cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) } println() - matrixMap.printMatricesWebLinks(args.project) + matrixMap.printMatricesWebLinks(project) - matrixMap.validate(args.ignoreFailedTests) + matrixMap.validate(ignoreFailedTests) } } -private suspend fun runTests(args: IArgs): TestResult { - return when (args) { - is AndroidArgs -> runAndroidTests(args) - is IosArgs -> runIosTests(args) +private suspend fun IArgs.runTests(): TestResult = + when (this) { + is AndroidArgs -> runAndroidTests() + is IosArgs -> runIosTests() else -> throw FlankGeneralError("Unknown config type") } -} private suspend fun cancelTestsOnTimeout( projectId: String, diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt index 8fa79f55ff..84ce15cee2 100644 --- a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt @@ -73,7 +73,7 @@ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = corouti if (dirty) { println(FtlConstants.indent + "Updating matrix file") - updateMatrixFile(matrixMap, args) + args.updateMatrixFile(matrixMap) } println() } diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt index 25b57b8e67..84c93a14ef 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt @@ -57,7 +57,7 @@ internal suspend fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) = corouti if (dirty) { println(FtlConstants.indent + "Updating matrix file") - updateMatrixFile(matrixMap, args) + args.updateMatrixFile(matrixMap) println() } } diff --git a/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt b/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt index 289a5abda9..1f08642c99 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/UpdateMatrixFile.kt @@ -7,12 +7,10 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -internal fun updateMatrixFile(matrixMap: MatrixMap, args: IArgs): Path { - val matrixIdsPath = if (args.useLocalResultDir()) { - Paths.get(args.localResultDir, FtlConstants.matrixIdsFile) - } else { - Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.matrixIdsFile) - } +internal fun IArgs.updateMatrixFile(matrixMap: MatrixMap): Path { + val matrixIdsPath = if (useLocalResultDir()) + Paths.get(localResultDir, FtlConstants.matrixIdsFile) else + Paths.get(localResultDir, matrixMap.runPath, FtlConstants.matrixIdsFile) matrixIdsPath.parent.toFile().mkdirs() Files.write(matrixIdsPath, prettyPrint.toJson(matrixMap.map).toByteArray()) return matrixIdsPath diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 06d1fee067..2bfa2a108f 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -34,32 +34,31 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineScope { - val (stopwatch, runGcsPath) = beforeRunTests(args) - - // GcAndroidMatrix => GcAndroidTestMatrix - // GcAndroidTestMatrix.execute() 3x retry => matrix id (string) - val devices = GcAndroidDevice.build(args.devices) +internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope { + val args = this@runAndroidTests + val stopwatch = beforeRunTests() + val devices = GcAndroidDevice.build(devices) val testMatrices = mutableListOf>() val allTestShardChunks = mutableListOf() val ignoredTestsShardChunks = mutableListOf>() - val history = GcToolResults.createToolResultsHistory(args) - val otherGcsFiles = args.uploadOtherFiles(runGcsPath) - val additionalApks = args.uploadAdditionalApks(runGcsPath) - val obbFiles = args.uploadObbFiles(runGcsPath) + val otherGcsFiles = uploadOtherFiles() + val additionalApks = uploadAdditionalApks() + val obbFiles = uploadObbFiles() - args.createAndroidTestContexts().dumpShards(args).upload(args.resultsBucket, runGcsPath) + createAndroidTestContexts() + .dumpShards(args) + .upload(resultsBucket, resultsDir) .forEachIndexed { contextIndex, context -> if (context is InstrumentationTestContext) { ignoredTestsShardChunks += context.ignoredTestCases allTestShardChunks += context.shards } - val androidTestConfig = args.createAndroidTestConfig(context) - testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { runIndex -> + val androidTestConfig = createAndroidTestConfig(context) + testMatrices += executeAndroidTestMatrix(runCount = repeatTests) { runIndex -> GcAndroidTestMatrix.build( androidTestConfig = androidTestConfig, - runGcsPath = runGcsPath.createGcsPath(contextIndex, runIndex), + runGcsPath = resultsDir.createGcsPath(contextIndex, runIndex), additionalApkGcsPaths = additionalApks, androidDeviceList = devices, args = args, @@ -72,10 +71,10 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS if (testMatrices.isEmpty()) throw FlankGeneralError("There are no tests to run.") - println(beforeRunMessage(args, allTestShardChunks)) + println(beforeRunMessage(allTestShardChunks)) TestResult( - matrixMap = afterRunTests(testMatrices.awaitAll(), runGcsPath, stopwatch, args), + matrixMap = afterRunTests(testMatrices.awaitAll(), stopwatch), shardChunks = allTestShardChunks.testCases, ignoredTests = ignoredTestsShardChunks.flatten() ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index 7c0ae26cbd..93b9ba0a42 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -2,7 +2,6 @@ package ftl.run.platform import com.google.api.services.testing.model.TestMatrix import ftl.args.IosArgs -import ftl.config.FtlConstants import ftl.gc.GcIosMatrix import ftl.gc.GcIosTestMatrix import ftl.gc.GcStorage @@ -19,6 +18,8 @@ import ftl.run.platform.common.beforeRunMessage import ftl.run.platform.common.beforeRunTests import ftl.shard.testCases import ftl.util.ShardCounter +import ftl.util.asFileReference +import ftl.util.uploadIfNeeded import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async @@ -28,38 +29,39 @@ import kotlinx.coroutines.coroutineScope // https://github.com/bootstraponline/gcloud_cli/blob/5bcba57e825fc98e690281cf69484b7ba4eb668a/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py#L109 // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/ -internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope { - val (stopwatch, runGcsPath) = beforeRunTests(iosArgs) +internal suspend fun IosArgs.runIosTests(): TestResult = coroutineScope { + val args = this@runIosTests + val stopwatch = beforeRunTests() - val iosDeviceList = GcIosMatrix.build(iosArgs.devices) - val xcTestParsed = parseToNSDictionary(iosArgs.xctestrunFile) + val iosDeviceList = GcIosMatrix.build(devices) + val xcTestParsed = parseToNSDictionary(xctestrunFile) val jobs = arrayListOf>() - val runCount = iosArgs.repeatTests + val runCount = repeatTests val shardCounter = ShardCounter() - val history = GcToolResults.createToolResultsHistory(iosArgs) - val otherGcsFiles = iosArgs.uploadOtherFiles(runGcsPath) - val additionalIpasGcsFiles = iosArgs.uploadAdditionalIpas(runGcsPath) + val history = GcToolResults.createToolResultsHistory(args) + val otherGcsFiles = uploadOtherFiles() + val additionalIpasGcsFiles = uploadAdditionalIpas() - dumpShards(iosArgs) - if (iosArgs.disableResultsUpload.not()) - GcStorage.upload(IOS_SHARD_FILE, iosArgs.resultsBucket, iosArgs.resultsDir) + dumpShards() + if (disableResultsUpload.not()) + GcStorage.upload(IOS_SHARD_FILE, resultsBucket, resultsDir) // Upload only after parsing shards to detect missing methods early. - val xcTestGcsPath = getXcTestGcPath(iosArgs, runGcsPath) + val xcTestGcsPath = uploadIfNeeded(xctestrunZip.asFileReference()).gcs - println(beforeRunMessage(iosArgs, iosArgs.testShardChunks)) + println(beforeRunMessage(testShardChunks)) repeat(runCount) { - jobs += iosArgs.testShardChunks.map { testTargets -> + jobs += testShardChunks.map { testTargets -> async(Dispatchers.IO) { GcIosTestMatrix.build( iosDeviceList = iosDeviceList, testZipGcsPath = xcTestGcsPath, - runGcsPath = runGcsPath, + runGcsPath = resultsDir, testTargets = testTargets.testMethodNames, xcTestParsed = xcTestParsed, - args = iosArgs, + args = args, shardCounter = shardCounter, toolResultsHistory = history, otherFiles = otherGcsFiles, @@ -70,16 +72,7 @@ internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope } TestResult( - matrixMap = afterRunTests(jobs.awaitAll(), runGcsPath, stopwatch, iosArgs), - shardChunks = iosArgs.testShardChunks.testCases + matrixMap = afterRunTests(jobs.awaitAll(), stopwatch), + shardChunks = testShardChunks.testCases ) } - -private fun getXcTestGcPath( - iosArgs: IosArgs, - runGcsPath: String -) = if (iosArgs.xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { - iosArgs.xctestrunZip -} else { - GcStorage.uploadXCTestZip(iosArgs, runGcsPath) -} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt index 14195dec96..f3299d898d 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadApks.kt @@ -47,11 +47,11 @@ private fun GameLoopContext.upload(rootGcsBucket: String, runGcsPath: String) = private fun SanityRoboTestContext.upload(rootGcsBucket: String, runGcsPath: String) = SanityRoboTestContext(app.uploadIfNeeded(rootGcsBucket, runGcsPath)) -suspend fun AndroidArgs.uploadAdditionalApks(runGcsPath: String) = - additionalApks.uploadToGcloudIfNeeded(runGcsPath, resultsBucket) +suspend fun AndroidArgs.uploadAdditionalApks() = + additionalApks.uploadToGcloudIfNeeded(resultsDir, resultsBucket) -suspend fun IosArgs.uploadAdditionalIpas(runGcsPath: String) = - additionalIpas.uploadToGcloudIfNeeded(runGcsPath, resultsBucket) +suspend fun IosArgs.uploadAdditionalIpas() = + additionalIpas.uploadToGcloudIfNeeded(resultsDir, resultsBucket) private suspend fun List.uploadToGcloudIfNeeded( runGcsPath: String, diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt index a91744c2d1..9a3eb7b314 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/UploadOtherFiles.kt @@ -8,17 +8,15 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -internal suspend fun IArgs.uploadOtherFiles( - runGcsPath: String -): Map = coroutineScope { +internal suspend fun IArgs.uploadOtherFiles(): Map = coroutineScope { otherFiles .map { (devicePath: String, filePath: String) -> - async(Dispatchers.IO) { devicePath to GcStorage.upload(filePath, resultsBucket, runGcsPath) } + async(Dispatchers.IO) { devicePath to GcStorage.upload(filePath, resultsBucket, resultsDir) } }.awaitAll().toMap() } -internal suspend fun AndroidArgs.uploadObbFiles(runGcsPath: String): Map = coroutineScope { +internal suspend fun AndroidArgs.uploadObbFiles(): Map = coroutineScope { obbFiles.map { - async(Dispatchers.IO) { it to GcStorage.upload(it, resultsBucket, runGcsPath) } + async(Dispatchers.IO) { it to GcStorage.upload(it, resultsBucket, resultsDir) } }.awaitAll().toMap() } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index 92d41fae03..27dd20d9f9 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -17,36 +17,34 @@ import kotlinx.coroutines.launch import java.nio.file.Files import java.nio.file.Paths -internal suspend fun afterRunTests( +internal suspend fun IArgs.afterRunTests( testMatrices: List, - runGcsPath: String, stopwatch: StopWatch, - config: IArgs ) = MatrixMap( map = testMatrices.toSavedMatrixMap(), - runPath = runGcsPath + runPath = resultsDir ).also { matrixMap -> - updateMatrixFile(matrixMap, config) - saveConfigFile(matrixMap, config) + updateMatrixFile(matrixMap) + saveConfigFile(matrixMap) println(FtlConstants.indent + "${matrixMap.map.size} matrix ids created in ${stopwatch.check()}") val gcsBucket = "https://console.developers.google.com/storage/browser/" + - config.resultsBucket + "/" + matrixMap.runPath + resultsBucket + "/" + matrixMap.runPath println(FtlConstants.indent + gcsBucket) println() - matrixMap.printMatricesWebLinks(config.project) + matrixMap.printMatricesWebLinks(project) } private fun List.toSavedMatrixMap() = associate { matrix -> matrix.testMatrixId to createSavedMatrix(matrix) } -private fun saveConfigFile(matrixMap: MatrixMap, args: IArgs) { - val configFilePath = if (args.useLocalResultDir()) - Paths.get(args.localResultDir, FtlConstants.configFileName(args)) else - Paths.get(args.localResultDir, matrixMap.runPath, FtlConstants.configFileName(args)) +private fun IArgs.saveConfigFile(matrixMap: MatrixMap) { + val configFilePath = if (useLocalResultDir()) + Paths.get(localResultDir, FtlConstants.configFileName(this)) else + Paths.get(localResultDir, matrixMap.runPath, FtlConstants.configFileName(this)) configFilePath.parent.toFile().mkdirs() - Files.write(configFilePath, args.data.toByteArray()) + Files.write(configFilePath, data.toByteArray()) } internal suspend inline fun MatrixMap.printMatricesWebLinks(project: String) = coroutineScope { diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt index 865bf4740c..4fc66f8d4d 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt @@ -5,8 +5,8 @@ import ftl.config.FtlConstants import ftl.shard.Chunk import ftl.shard.TestMethod -internal fun beforeRunMessage(args: IArgs, testShardChunks: List): String { - val runCount = args.repeatTests +internal fun IArgs.beforeRunMessage(testShardChunks: List): String { + val runCount = repeatTests val shardCount = testShardChunks.size val (classesCount, testsCount) = testShardChunks.partitionedTestCases.testAndClassesCount diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt index 860ab83311..7716ad1117 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt @@ -9,24 +9,20 @@ import ftl.run.exception.FlankGeneralError import ftl.util.StopWatch import java.io.File -internal fun beforeRunTests(args: IArgs): Pair { +internal fun IArgs.beforeRunTests() = StopWatch().also { watch -> println("\nRunTests") assertMockUrl() - val stopwatch = StopWatch().start() - val runGcsPath = args.resultsDir + watch.start() // Avoid spamming the results/ dir with temporary files from running the test suite. if (FtlConstants.useMock) - deleteMockResultDirOnShutDown(args, runGcsPath) + deleteMockResultDirOnShutDown() - if (args.useLocalResultDir()) { - // Only one result is stored when using --local-result-dir - // Delete any old results if they exist before storing new ones. - deleteLocalResultDir(args) - } - - return stopwatch to runGcsPath + // Only one result is stored when using --local-result-dir + // Delete any old results if they exist before storing new ones. + if (useLocalResultDir()) + deleteLocalResultDir() } private fun assertMockUrl() { @@ -36,14 +32,10 @@ private fun assertMockUrl() { if (!GcToolResults.service.rootUrl.contains(FtlConstants.localhost)) throw FlankGeneralError("expected localhost in GcToolResults") } -private fun deleteMockResultDirOnShutDown(args: IArgs, runGcsPath: String) { - Runtime.getRuntime().addShutdownHook( - Thread { - File(args.localResultDir, runGcsPath).deleteRecursively() - } - ) +private fun IArgs.deleteMockResultDirOnShutDown() { + File(localResultDir, resultsDir).deleteOnExit() } -private fun deleteLocalResultDir(args: IArgs) { - File(args.localResultDir).deleteRecursively() +private fun IArgs.deleteLocalResultDir() { + File(localResultDir).deleteRecursively() } diff --git a/test_runner/src/main/kotlin/ftl/util/FileReference.kt b/test_runner/src/main/kotlin/ftl/util/FileReference.kt index 2cd5a77e73..01641ed6ae 100644 --- a/test_runner/src/main/kotlin/ftl/util/FileReference.kt +++ b/test_runner/src/main/kotlin/ftl/util/FileReference.kt @@ -1,5 +1,6 @@ package ftl.util +import ftl.args.IArgs import ftl.config.FtlConstants import ftl.gc.GcStorage import ftl.run.exception.FlankGeneralError @@ -27,6 +28,15 @@ fun FileReference.downloadIfNeeded() = if (local.isNotBlank()) this else copy(local = GcStorage.download(gcs)) -fun FileReference.uploadIfNeeded(rootGcsBucket: String, runGcsPath: String) = +fun IArgs.uploadIfNeeded(file: FileReference): FileReference = + file.uploadIfNeeded( + rootGcsBucket = resultsBucket, + runGcsPath = resultsDir + ) + +fun FileReference.uploadIfNeeded( + rootGcsBucket: String, + runGcsPath: String +): FileReference = if (gcs.isNotBlank()) this else copy(gcs = GcStorage.upload(local, rootGcsBucket, runGcsPath)) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index b2dadb16bd..e534b86c90 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -1243,7 +1243,7 @@ AndroidArgs """.trimIndent() val parsedYml = AndroidArgs.load(yaml).validate() - val (matrixMap, chunks) = runBlocking { runAndroidTests(parsedYml) } + val (matrixMap, chunks) = runBlocking { parsedYml.runAndroidTests() } assertEquals(4, matrixMap.map.size) assertEquals(4, chunks.size) } @@ -1381,7 +1381,7 @@ AndroidArgs every { runBlocking { any().createAndroidTestContexts() } } returns listOf() val parsedYml = AndroidArgs.load(yaml).validate() - runBlocking { runAndroidTests(parsedYml) } + runBlocking { parsedYml.runAndroidTests() } } @Test diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 32d69dc69a..ee2f3c78ae 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -355,7 +355,7 @@ class IosRunCommandTest { val runCmd = IosRunCommand() runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" runCmd.run() - verify { dumpShards(any(), any()) } + verify { any().dumpShards(any()) } } @Test @@ -368,7 +368,7 @@ class IosRunCommandTest { runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" CommandLine(runCmd).parseArgs("--disable-results-upload") runCmd.run() - verify { dumpShards(any(), any()) } + verify { any().dumpShards(any()) } verify(inverse = true) { GcStorage.upload(IOS_SHARD_FILE, any(), any()) } } } diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index 2e1ad7b0d1..77a0d65ec2 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -24,7 +24,8 @@ import org.junit.contrib.java.lang.system.SystemOutRule class DumpShardsKtTest { @get:Rule - val output: SystemOutRule = SystemOutRule().enableLog().muteForSuccessfulTests() + val output: SystemOutRule = + SystemOutRule().enableLog().muteForSuccessfulTests() @After fun tearDown() = output.clearLog() @@ -78,7 +79,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(AndroidArgs.load(mixedConfigYaml), TEST_SHARD_FILE) + AndroidArgs.load(mixedConfigYaml).dumpShards(TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } @@ -135,13 +136,18 @@ class DumpShardsKtTest { // when val actual = runBlocking { - dumpShards(AndroidArgs.load(mixedConfigYaml, AndroidRunCommand().apply { obfuscate = true }), TEST_SHARD_FILE) + AndroidArgs.load( + yamlPath = mixedConfigYaml, + cli = AndroidRunCommand().apply { obfuscate = true } + ).dumpShards(TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } // then assertNotEquals(notExpected, actual) - assertThat(notExpected.split(System.lineSeparator()).size).isEqualTo(actual.split(System.lineSeparator()).size) // same line count + assertThat(notExpected.split(System.lineSeparator()).size).isEqualTo( + actual.split(System.lineSeparator()).size + ) // same line count assertThat(output.log).contains("Saved 3 shards to $TEST_SHARD_FILE") } @@ -166,7 +172,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) + IosArgs.load(ios2ConfigYaml).dumpShards(TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } @@ -197,13 +203,18 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml, IosRunCommand().apply { obfuscate = true }), TEST_SHARD_FILE) + IosArgs.load( + yamlPath = ios2ConfigYaml, + cli = IosRunCommand().apply { obfuscate = true } + ).dumpShards(TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } // then assertNotEquals(notExpected, actual) - assertThat(notExpected.split(System.lineSeparator()).size).isEqualTo(actual.split(System.lineSeparator()).size) // same line count + assertThat(notExpected.split(System.lineSeparator()).size).isEqualTo( + actual.split(System.lineSeparator()).size + ) // same line count assertThat(output.log).contains("Saved 2 shards to $TEST_SHARD_FILE") } } diff --git a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt index 10b31321d7..8fb22b9a19 100644 --- a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt @@ -28,13 +28,17 @@ class GenericTestRunnerTest { @Test fun testBeforeRunMessage1() { - val result = beforeRunMessage(createMock(1), listOf(Chunk(listOf(TestMethod(name = "", time = 0.0))))).normalizeLineEnding() + val result = createMock(1).beforeRunMessage( + listOf(Chunk(listOf(TestMethod(name = "", time = 0.0)))) + ).normalizeLineEnding() assert(result, " 1 test / 1 shard\n") } @Test fun testBeforeRunMessage2() { - val result = beforeRunMessage(createMock(2), listOf(Chunk(listOf(TestMethod(name = "", time = 0.0))))).normalizeLineEnding() + val result = createMock(2).beforeRunMessage( + listOf(Chunk(listOf(TestMethod(name = "", time = 0.0)))) + ).normalizeLineEnding() assert( result, """ @@ -48,8 +52,7 @@ class GenericTestRunnerTest { @Test fun testBeforeRunMessage3() { - val result = beforeRunMessage( - createMock(2), + val result = createMock(2).beforeRunMessage( List(6) { Chunk(listOf(TestMethod(name = "", time = 0.0))) } ).normalizeLineEnding() assert( @@ -65,8 +68,7 @@ class GenericTestRunnerTest { @Test fun testBeforeRunMessage4() { - val result = beforeRunMessage( - createMock(100), + val result = createMock(100).beforeRunMessage( List(2) { Chunk(List(5) { TestMethod(name = "", time = 0.0) }) } ).normalizeLineEnding() assert( @@ -86,8 +88,7 @@ class GenericTestRunnerTest { 10 tests + 3 parameterized classes / 2 shards """.trimIndent() - val result = beforeRunMessage( - createMock(1), + val result = createMock(1).beforeRunMessage( listOf( Chunk( listOf( @@ -122,8 +123,7 @@ class GenericTestRunnerTest { 3 parameterized classes / 2 shards """.trimIndent() - val result = beforeRunMessage( - createMock(1), + val result = createMock(1).beforeRunMessage( listOf( Chunk( listOf( @@ -152,8 +152,7 @@ class GenericTestRunnerTest { 30 total parameterized classes """.trimIndent() - val result = beforeRunMessage( - createMock(10), + val result = createMock(10).beforeRunMessage( listOf( Chunk( listOf( @@ -191,8 +190,7 @@ class GenericTestRunnerTest { 30 total parameterized classes """.trimIndent() - val result = beforeRunMessage( - createMock(10), + val result = createMock(10).beforeRunMessage( listOf( Chunk( listOf( diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 2e12eb01f3..6efc2c2671 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -114,7 +114,7 @@ class TestRunnerTest { fun `mockedAndroidTestRun local`() { val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml")) runBlocking { - newTestRun(localConfig) + localConfig.newTestRun() } } @@ -122,7 +122,7 @@ class TestRunnerTest { fun `mockedAndroidTestRun gcsAndHistoryName`() { val gcsConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.gcs.yml")) runBlocking { - newTestRun(gcsConfig) + gcsConfig.newTestRun() } } @@ -132,7 +132,7 @@ class TestRunnerTest { val config = IosArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.ios.yml")) runBlocking { - newTestRun(config) + config.newTestRun() } } @@ -142,7 +142,7 @@ class TestRunnerTest { val config = IosArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml")) runBlocking { - newTestRun(config) + config.newTestRun() } } @@ -150,7 +150,7 @@ class TestRunnerTest { fun `matrix webLink should be printed before polling matrices`() { val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml")) runBlocking { - newTestRun(localConfig) + localConfig.newTestRun() } val matrixWebLinkHeader = "Matrices webLink" val matrixLink = Regex("(matrix-\\d+ https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?") @@ -171,7 +171,7 @@ class TestRunnerTest { getMockedTestMatrix() ) runBlocking { - newTestRun(localConfig) + localConfig.newTestRun() } val matrixWebLinkHeader = "Matrices webLink" val message = "Unable to get web link" diff --git a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt index c3fe08afe5..6591639671 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/RunAndroidTestsKtTest.kt @@ -32,7 +32,7 @@ class RunAndroidTestsKtTest { // when val actual = runBlocking { - runAndroidTests(AndroidArgs.load(mixedConfigYaml)) + AndroidArgs.load(mixedConfigYaml).runAndroidTests() } // then @@ -49,7 +49,7 @@ class RunAndroidTestsKtTest { every { androidArgs.repeatTests } returns 2 runBlocking { - runAndroidTests(androidArgs) + androidArgs.runAndroidTests() } verify { @@ -76,7 +76,7 @@ class RunAndroidTestsKtTest { every { androidArgs.resultsBucket } returns "test_bucket" runBlocking { - runAndroidTests(androidArgs) + androidArgs.runAndroidTests() } verify(inverse = true) {