diff --git a/README.md b/README.md index 6b0a5bb2ee..928d45e7aa 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,9 @@ flank: ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + + ## The max time this test run can execute before it is cancelled (default: unlimited). + # run-timeout: 60m ``` ### Android example @@ -322,6 +325,9 @@ flank: # - app: ../test_app/apks/app-debug.apk # test: ../test_app/apks/app1-debug-androidTest.apk # - test: ../test_app/apks/app2-debug-androidTest.apk + + ## The max time this test run can execute before it is cancelled (default: unlimited). + # run-timeout: 60m ``` ### Android code coverage diff --git a/release_notes.md b/release_notes.md index 22adc79473..9c6e602503 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,6 +1,7 @@ ## next (unreleased) -- [#657](https://github.com/Flank/flank/pull/657) Fix execution hangs. ([pawelpasterz](https://github.com/pawelpasterz)) +- [#672](https://github.com/Flank/flank/pull/672) Flank timeout feature. ([pawelpasterz](https://github.com/pawelpasterz)) +- [#657](https://github.com/Flank/flank/pull/657) Fix execution hangs. ([pawelpasterz](https://github.com/pawelpasterz)) - [#654](https://github.com/Flank/flank/pull/654) Fix test filters when using both notPackage and notClass. ([jan-gogo](https://github.com/jan-gogo)) - [#648](https://github.com/Flank/flank/pull/648) Include @Ignore JUnit tests in JUnit XML. ([pawelpasterz](https://github.com/pawelpasterz)) - [#646](https://github.com/Flank/flank/pull/646) Adopt kotlin-logging as a logging framework. ([jan-gogo](https://github.com/jan-gogo)) diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index 616d756937..dcc213ea23 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -85,6 +85,7 @@ tasks.jacocoTestReport { classDirectories.setFrom( fileTree("build/classes/kotlin/main").apply { exclude("**/*\$run$1.class") + exclude("**/ftl/mock/*") }) reports { diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index f30f711048..4d9134dd03 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -109,3 +109,6 @@ flank: ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + + ## The max time this test run can execute before it is cancelled (default: unlimited). + # run-timeout: 60m diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 858602509a..21220495a7 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -142,3 +142,6 @@ flank: # - app: ../test_app/apks/app-debug.apk # test: ../test_app/apks/app-debug-androidTest.apk # - test: ../test_app/apks/app-debug-androidTest.apk + + ## The max time this test run can execute before it is cancelled (default: unlimited). + # run-timeout: 60m diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 9500696f4c..bb31062d01 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -72,6 +72,7 @@ class AndroidArgs( override val disableSharding = cli?.disableSharding ?: flank.disableSharding override val project = cli?.project ?: flank.project override val localResultDir = cli?.localResultDir ?: flank.localResultDir + override val runTimeout = cli?.runTimeout ?: flank.runTimeout private val androidFlank = androidFlankYml.flank val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks @@ -126,14 +127,11 @@ AndroidArgs test: $testApk auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator - directories-to-pull: -${listToString(directoriesToPull)} + directories-to-pull:${listToString(directoriesToPull)} performance-metrics: $performanceMetrics test-runner-class: $testRunnerClass - test-targets: -${listToString(testTargets)} - device: -${devicesToString(devices)} + test-targets:${listToString(testTargets)} + device:${devicesToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: @@ -142,17 +140,15 @@ ${devicesToString(devices)} num-test-runs: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath smart-flank-disable-upload: $smartFlankDisableUpload - files-to-download: -${listToString(filesToDownload)} - test-targets-always-run: -${listToString(testTargetsAlwaysRun)} + files-to-download:${listToString(filesToDownload)} + test-targets-always-run:${listToString(testTargetsAlwaysRun)} disable-sharding: $disableSharding project: $project local-result-dir: $localResultDir # Android Flank Yml keep-file-path: $keepFilePath - additional-app-test-apks: -${apksToString(additionalAppTestApks)} + additional-app-test-apks:${apksToString(additionalAppTestApks)} + run-timeout: $runTimeout """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt index 83ad49e860..06c66bf415 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt @@ -3,27 +3,30 @@ package ftl.args import ftl.args.yml.AppTestPair import ftl.config.Device +private const val NEW_LINE = '\n' + object ArgsToString { fun mapToString(map: Map?): String { if (map.isNullOrEmpty()) return "" - return map.map { (key, value) -> " $key: $value" } + return NEW_LINE + map.map { (key, value) -> " $key: $value" } .joinToString("\n") } fun listToString(list: List?): String { if (list.isNullOrEmpty()) return "" - return list.filterNotNull() + return NEW_LINE + list.filterNotNull() .joinToString("\n") { dir -> " - $dir" } } fun devicesToString(devices: List?): String { if (devices.isNullOrEmpty()) return "" - return devices.filterNotNull() + return NEW_LINE + devices.filterNotNull() .joinToString("\n") { "$it" } } fun apksToString(devices: List): String { - return devices.joinToString("\n") { (app, test) -> " - app: $app\n test: $test" } + if (devices.isNullOrEmpty()) return "" + return NEW_LINE + devices.joinToString("\n") { (app, test) -> " - app: $app\n test: $test" } } } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 48c3583610..89533c7acd 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -1,6 +1,7 @@ package ftl.args import ftl.args.yml.FlankYmlParams +import ftl.util.timeoutToMils // Properties common to both Android and iOS interface IArgs { @@ -27,6 +28,12 @@ interface IArgs { val filesToDownload: List val disableSharding: Boolean val localResultDir: String + val runTimeout: String + val parsedTimeout: Long + get() = timeoutToMils(runTimeout).let { + if (it < 0) Long.MAX_VALUE + else it + } fun useLocalResultDir() = localResultDir != FlankYmlParams.defaultLocalResultDir diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 1d3e7950b7..066a273281 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -61,6 +61,7 @@ class IosArgs( override val disableSharding = cli?.disableSharding ?: flank.disableSharding override val project = cli?.project ?: flank.project override val localResultDir = cli?.localResultsDir ?: flank.localResultDir + override val runTimeout = cli?.runTimeout ?: flank.runTimeout private val iosFlank = iosFlankYml.flank val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull() @@ -121,8 +122,7 @@ IosArgs test: $xctestrunZip xctestrun-file: $xctestrunFile xcode-version: $xcodeVersion - device: -${devicesToString(devices)} + device:${devicesToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: @@ -131,16 +131,14 @@ ${devicesToString(devices)} num-test-runs: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath smart-flank-disable-upload: $smartFlankDisableUpload - test-targets-always-run: -${listToString(testTargetsAlwaysRun)} - files-to-download: -${listToString(filesToDownload)} + test-targets-always-run:${listToString(testTargetsAlwaysRun)} + files-to-download:${listToString(filesToDownload)} # iOS flank - test-targets: -${listToString(testTargets)} + test-targets:${listToString(testTargets)} disable-sharding: $disableSharding project: $project local-result-dir: $localResultDir + run-timeout: $runTimeout """.trimIndent() } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 3103c83ad0..a0cbd1e2bb 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -3,6 +3,7 @@ package ftl.args.yml import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.ArgsHelper +import ftl.config.FtlConstants /** Flank specific parameters for both iOS and Android */ @JsonIgnoreProperties(ignoreUnknown = true) @@ -34,12 +35,15 @@ class FlankYmlParams( val project: String = ArgsHelper.getDefaultProjectId() ?: "", @field:JsonProperty("local-result-dir") - val localResultDir: String = defaultLocalResultDir + val localResultDir: String = defaultLocalResultDir, + + @field:JsonProperty("run-timeout") + val runTimeout: String = FtlConstants.runTimeout ) { companion object : IYmlKeys { override val keys = listOf( "max-test-shards", "shard-time", "num-test-runs", "smart-flank-gcs-path", "smart-flank-disable-upload", - "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir" + "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir", "run-timeout" ) const val defaultLocalResultDir = "results" diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 317d032f87..ff48d9007c 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -374,4 +374,10 @@ class AndroidRunCommand : Runnable { "Required when file names are not unique."] ) var keepFilePath: Boolean? = null + + @Option( + names = ["--run-timeout"], + description = ["The max time this test run can execute before it is cancelled (default: unlimited)."] + ) + var runTimeout: String? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index e6af8b7443..52dde97429 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -270,4 +270,10 @@ class IosRunCommand : Runnable { "names to run (default: run all test targets)."] ) var testTargets: List? = null + + @Option( + names = ["--run-timeout"], + description = ["The max time this test run can execute before it is cancelled (default: unlimited)."] + ) + var runTimeout: String? = null } diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index 4e81d9b6e4..059e896c3e 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -51,6 +51,7 @@ object FtlConstants { const val matrixIdsFile = "matrix_ids.json" const val applicationName = "Flank" const val GCS_PREFIX = "gs://" + const val runTimeout = "-1" val JSON_FACTORY: JsonFactory by lazy { Utils.getDefaultJsonFactory() } val bugsnag = Bugsnag(if (useMock) null else "3d5f8ba4ee847d6bb51cb9c347eda74f") diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index c0b99200b1..dcd4ec8f7e 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -22,7 +22,7 @@ import ftl.args.AndroidArgs import ftl.args.ShardChunks import ftl.util.fatalError import ftl.util.join -import ftl.util.testTimeoutToSeconds +import ftl.util.timeoutToSeconds object GcAndroidTestMatrix { @@ -84,7 +84,7 @@ object GcAndroidTestMatrix { args.environmentVariables.map { it.toEnvironmentVariable() } } - val testTimeoutSeconds = testTimeoutToSeconds(args.testTimeout) + val testTimeoutSeconds = timeoutToSeconds(args.testTimeout) val testSpecification = TestSpecification() .setAndroidInstrumentationTest(androidInstrumentation) diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index 69037b3031..09187cbaa6 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -19,7 +19,7 @@ import ftl.ios.Xctestrun.toByteArray import ftl.util.ShardCounter import ftl.util.fatalError import ftl.util.join -import ftl.util.testTimeoutToSeconds +import ftl.util.timeoutToSeconds object GcIosTestMatrix { @@ -57,7 +57,7 @@ object GcIosTestMatrix { val iOSTestSetup = IosTestSetup() .setNetworkProfile(null) - val testTimeoutSeconds = testTimeoutToSeconds(args.testTimeout) + val testTimeoutSeconds = timeoutToSeconds(args.testTimeout) val testSpecification = TestSpecification() .setDisableVideoRecording(!args.recordVideo) diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt index 97bd6d62d2..da26d67f20 100644 --- a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt @@ -3,25 +3,29 @@ package ftl.run import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs +import ftl.json.SavedMatrix import ftl.reports.util.ReportManager import ftl.run.model.TestResult import ftl.run.common.fetchArtifacts import ftl.run.common.pollMatrices import ftl.run.platform.runAndroidTests import ftl.run.platform.runIosTests -import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeoutOrNull import kotlin.system.exitProcess -suspend fun newTestRun(args: IArgs) = coroutineScope { - println(args) - val (matrixMap, testShardChunks) = runTests(args) +suspend fun newTestRun(args: IArgs) { + withTimeoutOrNull(args.parsedTimeout) { + println(args) + val (matrixMap, testShardChunks) = cancelTestsOnTimeout(args.project) { runTests(args) } - if (!args.async) { - pollMatrices(matrixMap, args) - fetchArtifacts(matrixMap, args) + if (!args.async) { + cancelTestsOnTimeout(args.project, matrixMap.map) { pollMatrices(matrixMap, args) } + cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) } - val exitCode = ReportManager.generate(matrixMap, args, testShardChunks) - exitProcess(exitCode) + val exitCode = ReportManager.generate(matrixMap, args, testShardChunks) + exitProcess(exitCode) + } } } @@ -32,3 +36,17 @@ private suspend fun runTests(args: IArgs): TestResult { else -> throw RuntimeException("Unknown config type") } } + +private suspend fun cancelTestsOnTimeout( + projectId: String, + savedMatrix: Map? = null, + block: suspend () -> T +) = try { + block() + } catch (_: TimeoutCancellationException) { + println("\nCanceling flank due to timeout") + savedMatrix?.run { + cancelMatrices(savedMatrix, projectId) + } + exitProcess(1) + } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt index ef9e7b74f7..2006f2e95c 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt @@ -10,7 +10,7 @@ import ftl.util.uniqueObjectName import java.io.File internal fun beforeRunTests(args: IArgs): Pair { - println("RunTests") + println("\nRunTests") assertMockUrl() val stopwatch = StopWatch().start() diff --git a/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt b/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt index 5f5869848c..3c2fb15563 100644 --- a/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt +++ b/test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt @@ -15,7 +15,7 @@ fun resolveLocalRunPath(matrices: MatrixMap, args: IArgs): String { return matrices.runPath } -fun testTimeoutToSeconds(timeout: String): Long { +fun timeoutToSeconds(timeout: String): Long { return when { timeout.contains("h") -> TimeUnit.HOURS.toSeconds(timeout.removeSuffix("h").toLong()) // Hours timeout.contains("m") -> TimeUnit.MINUTES.toSeconds(timeout.removeSuffix("m").toLong()) // Minutes @@ -23,3 +23,5 @@ fun testTimeoutToSeconds(timeout: String): Long { else -> timeout.removeSuffix("s").toLong() // Seconds } } + +fun timeoutToMils(timeout: String): Long = timeoutToSeconds(timeout) * 1_000L diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index c2ab26d1a5..dbd283c370 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -13,6 +13,7 @@ import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath import ftl.test.util.TestHelper.getString +import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemErrRule @@ -184,4 +185,10 @@ class AndroidArgsFileTest { assert(config.resultsBucket, "tmp_bucket_2") } + + @Test + fun `verify run timeout value from yml file`() { + val args = AndroidArgs.load(localYamlFile) + assertEquals(60 * 60 * 1000L, args.parsedTimeout) + } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index f572e22449..cd77a2bb11 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -8,6 +8,7 @@ import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion import ftl.run.platform.runAndroidTests import ftl.test.util.FlankTestRunner +import ftl.test.util.TestHelper.getPath import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import kotlinx.coroutines.runBlocking @@ -28,6 +29,7 @@ class AndroidArgsTest { private val testErrorApk = "../test_app/apks/error-androidTest.apk" private val appApkAbsolutePath = appApk.absolutePath() private val testApkAbsolutePath = testApk.absolutePath() + private val simpleFlankPath = getPath("src/test/kotlin/ftl/fixtures/simple-android-flank.yml") private val androidNonDefault = """ gcloud: @@ -79,6 +81,7 @@ class AndroidArgsTest { additional-app-test-apks: - app: foo test: bar + run-timeout: 20m """ @Rule @@ -197,6 +200,7 @@ class AndroidArgsTest { ) ) assert(disableSharding, true) + assert(runTimeout, "20m") } } @@ -257,10 +261,57 @@ AndroidArgs additional-app-test-apks: - app: foo test: bar + run-timeout: 20m """.trimIndent() ) } + @Test + fun `verify default yml toString`() { + val args = AndroidArgs.load(simpleFlankPath) + assertEquals(""" +AndroidArgs + gcloud: + results-bucket: mockBucket + results-dir: null + record-video: false + timeout: 15m + async: false + results-history-name: null + # Android gcloud + app: $appApkAbsolutePath + test: $testApkAbsolutePath + auto-google-login: false + use-orchestrator: true + directories-to-pull: + performance-metrics: false + test-runner-class: null + test-targets: + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + files-to-download: + test-targets-always-run: + disable-sharding: false + project: mockProjectId + local-result-dir: results + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + """.trimIndent(), args.toString()) + } + @Test fun androidArgsDefault() { val androidArgs = AndroidArgs.load( @@ -298,6 +349,7 @@ AndroidArgs assert(filesToDownload, empty) assert(testTargetsAlwaysRun, empty) assert(disableSharding, false) + assert(runTimeout, "-1") } } @@ -960,4 +1012,10 @@ AndroidArgs assertEquals(4, matrixMap.map.size) assertEquals(4, chunks.size) } + + @Test + fun `verify run timeout default value - android`() { + val args = AndroidArgs.load(simpleFlankPath) + assertEquals(Long.MAX_VALUE, args.parsedTimeout) + } } diff --git a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt b/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt index 612900706e..d92b0080ae 100644 --- a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt @@ -30,6 +30,7 @@ class FlankYmlTest { assertThat(yml.flank.maxTestShards).isEqualTo(1) assertThat(yml.flank.shardTime).isEqualTo(58) assertThat(yml.flank.testTargetsAlwaysRun).isEqualTo(emptyList()) + assertThat(yml.flank.runTimeout).isEqualTo("-1") assertThat(FlankYml.map).isNotEmpty() } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt index c678d1795c..a0b2d3b6ad 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt @@ -6,6 +6,7 @@ import ftl.config.FtlConstants import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath +import org.junit.Assert.assertEquals import org.junit.Assume import org.junit.Rule import org.junit.Test @@ -91,4 +92,10 @@ class IosArgsFileTest { assertThat(testShardChunks[0]).isEqualTo(chunk0) assertThat(testShardChunks[1]).isEqualTo(chunk1) } + + @Test + fun `verify run timeout value from yml file`() { + val args = IosArgs.load(yamlFile) + assertEquals(60 * 60 * 1000L, args.parsedTimeout) + } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 9897d0e9a5..9e52cd9cb2 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -15,6 +15,7 @@ import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assume import org.junit.Rule @@ -71,6 +72,7 @@ class IosArgsTest { - b/testBasicSelection - b/testBasicSelection2 disable-sharding: true + run-timeout: 15m """ @Rule @@ -153,6 +155,7 @@ flank: assert(flakyTestAttempts, 4) assert(disableSharding, true) + assert(runTimeout, "15m") } } @@ -202,10 +205,51 @@ IosArgs disable-sharding: true project: projectFoo local-result-dir: results + run-timeout: 15m """.trimIndent() ) } + @Test + fun `verify default yml toString`() { + val args = IosArgs.load(simpleFlankPath) + assertEquals(""" +IosArgs + gcloud: + results-bucket: mockBucket + results-dir: null + record-video: false + timeout: 15m + async: false + results-history-name: null + # iOS gcloud + test: $testAbsolutePath + xctestrun-file: $xctestrunFileAbsolutePath + xcode-version: null + device: + - model: iphone8 + version: 12.0 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + test-targets-always-run: + files-to-download: + # iOS flank + test-targets: + disable-sharding: false + project: mockProjectId + local-result-dir: results + run-timeout: -1 + """.trimIndent(), args.toString()) + } + @Test fun argsDefault() { val args = IosArgs.load( @@ -240,6 +284,7 @@ IosArgs // IosFlankYml assert(testTargets, empty) + assert(runTimeout, "-1") } } @@ -758,4 +803,10 @@ IosArgs val args = IosArgs.load(simpleFlankPath) assertFalse(args.recordVideo) } + + @Test + fun `verify run timeout default value - ios`() { + val iosArgs = IosArgs.load(simpleFlankPath) + assertEquals(Long.MAX_VALUE, iosArgs.parsedTimeout) + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index 4cdeeccbe9..8797ff16ea 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -96,6 +96,7 @@ class AndroidRunCommandTest { assertThat(cmd.smartFlankGcsPath).isNull() assertThat(cmd.additionalAppTestApks).isNull() assertThat(cmd.keepFilePath).isNull() + assertThat(cmd.runTimeout).isNull() } @Test @@ -385,4 +386,12 @@ class AndroidRunCommandTest { assertThat(cmd.dryRun).isEqualTo(true) } + + @Test + fun `run-timeout parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--run-timeout=20s") + + assertThat(cmd.runTimeout).isEqualTo("20s") + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index fed4913d0e..96ae26ff78 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -1,6 +1,5 @@ package ftl.cli.firebase.test.ios -import com.google.common.truth.Truth import com.google.common.truth.Truth.assertThat import ftl.config.Device import ftl.config.FtlConstants @@ -30,8 +29,8 @@ class IosRunCommandTest { CommandLine(iosRun).execute("-h") val output = systemOutRule.log - Truth.assertThat(output).startsWith("Run tests on Firebase Test Lab") - Truth.assertThat(output).contains("run [-h]") + assertThat(output).startsWith("Run tests on Firebase Test Lab") + assertThat(output).contains("run [-h]") assertThat(iosRun.usageHelpRequested).isTrue() } @@ -46,7 +45,7 @@ class IosRunCommandTest { runCmd.run() val output = systemOutRule.log - Truth.assertThat(output).contains("1 / 1 (100.00%)") + assertThat(output).contains("1 / 1 (100.00%)") } @Test @@ -89,6 +88,7 @@ class IosRunCommandTest { assertThat(cmd.localResultsDir).isNull() assertThat(cmd.smartFlankDisableUpload).isNull() assertThat(cmd.smartFlankGcsPath).isNull() + assertThat(cmd.runTimeout).isNull() } @Test @@ -286,4 +286,12 @@ class IosRunCommandTest { assertThat(cmd.dumpShards).isEqualTo(true) } + + @Test + fun `run-test parse`() { + val cmd = IosRunCommand() + CommandLine(cmd).parseArgs("--run-timeout=20s") + + assertThat(cmd.runTimeout).isEqualTo("20s") + } } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml index ee76cc5e98..b5e5688ad9 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml @@ -16,3 +16,4 @@ flank: - EarlGreyExampleSwiftTests/testLayout max-test-shards: 1 num-test-runs: 1 + run-timeout: 60m diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml index 976aed03be..942e147ad0 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.local.yml @@ -28,3 +28,4 @@ gcloud: flank: max-test-shards: 1 num-test-runs: 1 + run-timeout: 60m diff --git a/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml b/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml index 1a1bcf50e1..46ce598dbe 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml @@ -1,3 +1,3 @@ gcloud: app: "../test_app/apks/app-debug.apk" - test: "../test_app/apks/error-androidTest.apk" + test: "../test_app/apks/app-debug-androidTest.apk" diff --git a/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt b/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt index 4ddea5f65a..374b60be3a 100644 --- a/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/MatrixUtilTest.kt @@ -14,19 +14,19 @@ class MatrixUtilTest { @Test fun `testTimeoutToSeconds validInput`() { - assertThat(testTimeoutToSeconds("0")).isEqualTo(0) - assertThat(testTimeoutToSeconds("0s")).isEqualTo(0) - assertThat(testTimeoutToSeconds("0m")).isEqualTo(0) - assertThat(testTimeoutToSeconds("0h")).isEqualTo(0) - assertThat(testTimeoutToSeconds("1")).isEqualTo(1) - assertThat(testTimeoutToSeconds("1s")).isEqualTo(1) - assertThat(testTimeoutToSeconds("1m")).isEqualTo(60) - assertThat(testTimeoutToSeconds("1h")).isEqualTo(60 * 60) + assertThat(timeoutToSeconds("0")).isEqualTo(0) + assertThat(timeoutToSeconds("0s")).isEqualTo(0) + assertThat(timeoutToSeconds("0m")).isEqualTo(0) + assertThat(timeoutToSeconds("0h")).isEqualTo(0) + assertThat(timeoutToSeconds("1")).isEqualTo(1) + assertThat(timeoutToSeconds("1s")).isEqualTo(1) + assertThat(timeoutToSeconds("1m")).isEqualTo(60) + assertThat(timeoutToSeconds("1h")).isEqualTo(60 * 60) } @Test(expected = NumberFormatException::class) fun `testTimeoutToSeconds rejectsInvalid`() { - testTimeoutToSeconds("1d") + timeoutToSeconds("1d") } @Test