Flank timeout feature

README.md

@@ -166,6 +166,9 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
+
+  ## The max time this test run can execute before it is cancelled (default: unlimited).
+  # run-timeout: 60m
 ```
 
 ### Android example
@@ -322,6 +325,9 @@ flank:
   #  - app: ../test_app/apks/app-debug.apk
   #    test: ../test_app/apks/app1-debug-androidTest.apk
   #  - test: ../test_app/apks/app2-debug-androidTest.apk
+
+  ## The max time this test run can execute before it is cancelled (default: unlimited).
+  # run-timeout: 60m
 ```
 
 ### Android code coverage

release_notes.md

@@ -1,6 +1,7 @@
 ## next (unreleased)
 
-- [#657](https://github.com/Flank/flank/pull/657) Fix execution hangs. ([pawelpasterz](https://github.com/pawelpasterz)) 
+- [#672](https://github.com/Flank/flank/pull/672) Flank timeout feature. ([pawelpasterz](https://github.com/pawelpasterz)) 
+- [#657](https://github.com/Flank/flank/pull/657) Fix execution hangs. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#654](https://github.com/Flank/flank/pull/654) Fix test filters when using both notPackage and notClass. ([jan-gogo](https://github.com/jan-gogo))
 - [#648](https://github.com/Flank/flank/pull/648) Include @Ignore JUnit tests in JUnit XML. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#646](https://github.com/Flank/flank/pull/646) Adopt kotlin-logging as a logging framework. ([jan-gogo](https://github.com/jan-gogo))

test_runner/build.gradle.kts

@@ -85,6 +85,7 @@ tasks.jacocoTestReport {
     classDirectories.setFrom(
         fileTree("build/classes/kotlin/main").apply {
             exclude("**/*\$run$1.class")
+            exclude("**/ftl/mock/*")
         })
 
     reports {

test_runner/flank.ios.yml

@@ -109,3 +109,6 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
+
+  ## The max time this test run can execute before it is cancelled (default: unlimited).
+  # run-timeout: 60m

test_runner/flank.yml

@@ -142,3 +142,6 @@ flank:
   #  - app: ../test_app/apks/app-debug.apk
   #    test: ../test_app/apks/app-debug-androidTest.apk
   #  - test: ../test_app/apks/app-debug-androidTest.apk
+
+  ## The max time this test run can execute before it is cancelled (default: unlimited).
+  # run-timeout: 60m

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -72,6 +72,7 @@ class AndroidArgs(
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
     override val project = cli?.project ?: flank.project
     override val localResultDir = cli?.localResultDir ?: flank.localResultDir
+    override val runTimeout = cli?.runTimeout ?: flank.runTimeout
 
     private val androidFlank = androidFlankYml.flank
     val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks
@@ -126,14 +127,11 @@ AndroidArgs
       test: $testApk
       auto-google-login: $autoGoogleLogin
       use-orchestrator: $useOrchestrator
-      directories-to-pull:
-${listToString(directoriesToPull)}
+      directories-to-pull:${listToString(directoriesToPull)}
       performance-metrics: $performanceMetrics
       test-runner-class: $testRunnerClass
-      test-targets:
-${listToString(testTargets)}
-      device:
-${devicesToString(devices)}
+      test-targets:${listToString(testTargets)}
+      device:${devicesToString(devices)}
       num-flaky-test-attempts: $flakyTestAttempts
 
     flank:
@@ -142,17 +140,15 @@ ${devicesToString(devices)}
       num-test-runs: $repeatTests
       smart-flank-gcs-path: $smartFlankGcsPath
       smart-flank-disable-upload: $smartFlankDisableUpload
-      files-to-download:
-${listToString(filesToDownload)}
-      test-targets-always-run:
-${listToString(testTargetsAlwaysRun)}
+      files-to-download:${listToString(filesToDownload)}
+      test-targets-always-run:${listToString(testTargetsAlwaysRun)}
       disable-sharding: $disableSharding
       project: $project
       local-result-dir: $localResultDir
       # Android Flank Yml
       keep-file-path: $keepFilePath
-      additional-app-test-apks:
-${apksToString(additionalAppTestApks)}
+      additional-app-test-apks:${apksToString(additionalAppTestApks)}
+      run-timeout: $runTimeout
    """.trimIndent()
     }
 

test_runner/src/main/kotlin/ftl/args/ArgsToString.kt

@@ -3,27 +3,30 @@ package ftl.args
 import ftl.args.yml.AppTestPair
 import ftl.config.Device
 
+private const val NEW_LINE = '\n'
+
 object ArgsToString {
 
     fun mapToString(map: Map<String, String>?): String {
         if (map.isNullOrEmpty()) return ""
-        return map.map { (key, value) -> "        $key: $value" }
+        return NEW_LINE + map.map { (key, value) -> "        $key: $value" }
             .joinToString("\n")
     }
 
     fun listToString(list: List<String?>?): String {
         if (list.isNullOrEmpty()) return ""
-        return list.filterNotNull()
+        return NEW_LINE + list.filterNotNull()
             .joinToString("\n") { dir -> "        - $dir" }
     }
 
     fun devicesToString(devices: List<Device?>?): String {
         if (devices.isNullOrEmpty()) return ""
-        return devices.filterNotNull()
+        return NEW_LINE + devices.filterNotNull()
             .joinToString("\n") { "$it" }
     }
 
     fun apksToString(devices: List<AppTestPair>): String {
-        return devices.joinToString("\n") { (app, test) -> "        - app: $app\n          test: $test" }
+        if (devices.isNullOrEmpty()) return ""
+        return NEW_LINE + devices.joinToString("\n") { (app, test) -> "        - app: $app\n          test: $test" }
     }
 }

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -1,6 +1,7 @@
 package ftl.args
 
 import ftl.args.yml.FlankYmlParams
+import ftl.util.timeoutToMils
 
 // Properties common to both Android and iOS
 interface IArgs {
@@ -27,6 +28,12 @@ interface IArgs {
     val filesToDownload: List<String>
     val disableSharding: Boolean
     val localResultDir: String
+    val runTimeout: String
+    val parsedTimeout: Long
+        get() = timeoutToMils(runTimeout).let {
+            if (it < 0) Long.MAX_VALUE
+            else it
+        }
 
     fun useLocalResultDir() = localResultDir != FlankYmlParams.defaultLocalResultDir
 

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -61,6 +61,7 @@ class IosArgs(
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
     override val project = cli?.project ?: flank.project
     override val localResultDir = cli?.localResultsDir ?: flank.localResultDir
+    override val runTimeout = cli?.runTimeout ?: flank.runTimeout
 
     private val iosFlank = iosFlankYml.flank
     val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull()
@@ -121,8 +122,7 @@ IosArgs
       test: $xctestrunZip
       xctestrun-file: $xctestrunFile
       xcode-version: $xcodeVersion
-      device:
-${devicesToString(devices)}
+      device:${devicesToString(devices)}
       num-flaky-test-attempts: $flakyTestAttempts
 
     flank:
@@ -131,16 +131,14 @@ ${devicesToString(devices)}
       num-test-runs: $repeatTests
       smart-flank-gcs-path: $smartFlankGcsPath
       smart-flank-disable-upload: $smartFlankDisableUpload
-      test-targets-always-run:
-${listToString(testTargetsAlwaysRun)}
-      files-to-download:
-${listToString(filesToDownload)}
+      test-targets-always-run:${listToString(testTargetsAlwaysRun)}
+      files-to-download:${listToString(filesToDownload)}
       # iOS flank
-      test-targets:
-${listToString(testTargets)}
+      test-targets:${listToString(testTargets)}
       disable-sharding: $disableSharding
       project: $project
       local-result-dir: $localResultDir
+      run-timeout: $runTimeout
     """.trimIndent()
     }
 

test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt

@@ -3,6 +3,7 @@ package ftl.args.yml
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.ArgsHelper
+import ftl.config.FtlConstants
 
 /** Flank specific parameters for both iOS and Android */
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -34,12 +35,15 @@ class FlankYmlParams(
     val project: String = ArgsHelper.getDefaultProjectId() ?: "",
 
     @field:JsonProperty("local-result-dir")
-    val localResultDir: String = defaultLocalResultDir
+    val localResultDir: String = defaultLocalResultDir,
+
+    @field:JsonProperty("run-timeout")
+    val runTimeout: String = FtlConstants.runTimeout
 ) {
     companion object : IYmlKeys {
         override val keys = listOf(
             "max-test-shards", "shard-time", "num-test-runs", "smart-flank-gcs-path", "smart-flank-disable-upload",
-            "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir"
+            "disable-sharding", "test-targets-always-run", "files-to-download", "project", "local-result-dir", "run-timeout"
         )
 
         const val defaultLocalResultDir = "results"

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -374,4 +374,10 @@ class AndroidRunCommand : Runnable {
                     "Required when file names are not unique."]
     )
     var keepFilePath: Boolean? = null
+
+    @Option(
+        names = ["--run-timeout"],
+        description = ["The max time this test run can execute before it is cancelled (default: unlimited)."]
+    )
+    var runTimeout: String? = null
 }

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -270,4 +270,10 @@ class IosRunCommand : Runnable {
                 "names to run (default: run all test targets)."]
     )
     var testTargets: List<String>? = null
+
+    @Option(
+        names = ["--run-timeout"],
+        description = ["The max time this test run can execute before it is cancelled (default: unlimited)."]
+    )
+    var runTimeout: String? = null
 }

test_runner/src/main/kotlin/ftl/config/FtlConstants.kt

@@ -51,6 +51,7 @@ object FtlConstants {
     const val matrixIdsFile = "matrix_ids.json"
     const val applicationName = "Flank"
     const val GCS_PREFIX = "gs://"
+    const val runTimeout = "-1"
     val JSON_FACTORY: JsonFactory by lazy { Utils.getDefaultJsonFactory() }
 
     val bugsnag = Bugsnag(if (useMock) null else "3d5f8ba4ee847d6bb51cb9c347eda74f")

test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt

@@ -22,7 +22,7 @@ import ftl.args.AndroidArgs
 import ftl.args.ShardChunks
 import ftl.util.fatalError
 import ftl.util.join
-import ftl.util.testTimeoutToSeconds
+import ftl.util.timeoutToSeconds
 
 object GcAndroidTestMatrix {
 
@@ -84,7 +84,7 @@ object GcAndroidTestMatrix {
                 args.environmentVariables.map { it.toEnvironmentVariable() }
         }
 
-        val testTimeoutSeconds = testTimeoutToSeconds(args.testTimeout)
+        val testTimeoutSeconds = timeoutToSeconds(args.testTimeout)
 
         val testSpecification = TestSpecification()
             .setAndroidInstrumentationTest(androidInstrumentation)

test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt

@@ -19,7 +19,7 @@ import ftl.ios.Xctestrun.toByteArray
 import ftl.util.ShardCounter
 import ftl.util.fatalError
 import ftl.util.join
-import ftl.util.testTimeoutToSeconds
+import ftl.util.timeoutToSeconds
 
 object GcIosTestMatrix {
 
@@ -57,7 +57,7 @@ object GcIosTestMatrix {
         val iOSTestSetup = IosTestSetup()
             .setNetworkProfile(null)
 
-        val testTimeoutSeconds = testTimeoutToSeconds(args.testTimeout)
+        val testTimeoutSeconds = timeoutToSeconds(args.testTimeout)
 
         val testSpecification = TestSpecification()
             .setDisableVideoRecording(!args.recordVideo)

test_runner/src/main/kotlin/ftl/run/NewTestRun.kt

@@ -3,25 +3,29 @@ package ftl.run
 import ftl.args.AndroidArgs
 import ftl.args.IArgs
 import ftl.args.IosArgs
+import ftl.json.SavedMatrix
 import ftl.reports.util.ReportManager
 import ftl.run.model.TestResult
 import ftl.run.common.fetchArtifacts
 import ftl.run.common.pollMatrices
 import ftl.run.platform.runAndroidTests
 import ftl.run.platform.runIosTests
-import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.TimeoutCancellationException
+import kotlinx.coroutines.withTimeoutOrNull
 import kotlin.system.exitProcess
 
-suspend fun newTestRun(args: IArgs) = coroutineScope {
-    println(args)
-    val (matrixMap, testShardChunks) = runTests(args)
+suspend fun newTestRun(args: IArgs) {
+    withTimeoutOrNull(args.parsedTimeout) {
+        println(args)
+        val (matrixMap, testShardChunks) = cancelTestsOnTimeout(args.project) { runTests(args) }
 
-    if (!args.async) {
-        pollMatrices(matrixMap, args)
-        fetchArtifacts(matrixMap, args)
+        if (!args.async) {
+            cancelTestsOnTimeout(args.project, matrixMap.map) { pollMatrices(matrixMap, args) }
+            cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) }
 
-        val exitCode = ReportManager.generate(matrixMap, args, testShardChunks)
-        exitProcess(exitCode)
+            val exitCode = ReportManager.generate(matrixMap, args, testShardChunks)
+            exitProcess(exitCode)
+        }
     }
 }
 
@@ -32,3 +36,17 @@ private suspend fun runTests(args: IArgs): TestResult {
         else -> throw RuntimeException("Unknown config type")
     }
 }
+
+private suspend fun <T> cancelTestsOnTimeout(
+    projectId: String,
+    savedMatrix: Map<String, SavedMatrix>? = null,
+    block: suspend () -> T
+) = try {
+        block()
+    } catch (_: TimeoutCancellationException) {
+        println("\nCanceling flank due to timeout")
+        savedMatrix?.run {
+            cancelMatrices(savedMatrix, projectId)
+        }
+        exitProcess(1)
+    }

test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunTests.kt

@@ -10,7 +10,7 @@ import ftl.util.uniqueObjectName
 import java.io.File
 
 internal fun beforeRunTests(args: IArgs): Pair<StopWatch, String> {
-    println("RunTests")
+    println("\nRunTests")
     assertMockUrl()
 
     val stopwatch = StopWatch().start()

test_runner/src/main/kotlin/ftl/util/MatrixUtil.kt

@@ -15,11 +15,13 @@ fun resolveLocalRunPath(matrices: MatrixMap, args: IArgs): String {
     return matrices.runPath
 }
 
-fun testTimeoutToSeconds(timeout: String): Long {
+fun timeoutToSeconds(timeout: String): Long {
     return when {
         timeout.contains("h") -> TimeUnit.HOURS.toSeconds(timeout.removeSuffix("h").toLong()) // Hours
         timeout.contains("m") -> TimeUnit.MINUTES.toSeconds(timeout.removeSuffix("m").toLong()) // Minutes
         timeout.contains("s") -> timeout.removeSuffix("s").toLong() // Seconds
         else -> timeout.removeSuffix("s").toLong() // Seconds
     }
 }
+
+fun timeoutToMils(timeout: String): Long = timeoutToSeconds(timeout) * 1_000L

test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt

@@ -13,6 +13,7 @@ import ftl.test.util.TestHelper.absolutePath
 import ftl.test.util.TestHelper.assert
 import ftl.test.util.TestHelper.getPath
 import ftl.test.util.TestHelper.getString
+import org.junit.Assert.assertEquals
 import org.junit.Rule
 import org.junit.Test
 import org.junit.contrib.java.lang.system.SystemErrRule
@@ -184,4 +185,10 @@ class AndroidArgsFileTest {
 
         assert(config.resultsBucket, "tmp_bucket_2")
     }
+
+    @Test
+    fun `verify run timeout value from yml file`() {
+        val args = AndroidArgs.load(localYamlFile)
+        assertEquals(60 * 60 * 1000L, args.parsedTimeout)
+    }
 }