Add --local-result-dir

Use exit code 3 on fatal error instead of -1

README.md

@@ -151,6 +151,9 @@ flank:
 
   ## The billing enabled Google Cloud Platform project name to use
   # project: delta-essence-114723
+
+  ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+  # local-result-dir: flank
 ```
 
 ### Android example
@@ -285,6 +288,9 @@ flank:
 
   ## The billing enabled Google Cloud Platform project name to use
   # project: delta-essence-114723
+
+  ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+  # local-result-dir: flank
 ```
 
 ### Android code coverage

release_notes.md

@@ -5,6 +5,7 @@
 - [#514](https://github.com/TestArmada/flank/pull/514) Rename `testShards` to `maxTestShards` ([miguelslemos](https://github.com/miguelslemos))
 - [#518](https://github.com/TestArmada/flank/pull/518) Add deprecation warnings when old key names are used. `flank android doctor --fix` will auto fix the YAML file. ([bootstraponline](https://github.com/bootstraponline))
 - [#519](https://github.com/TestArmada/flank/pull/519)  Rename `maxTestShards` to `max-test-shards`, `shardTime` to `shard-time`, `repeatTests` to `repeat-tests`, `smartFlankGcsPath` to `smart-flank-gcs-path`, `disableSharding` to `disable-sharding`. Moved `project` from `gcloud` to `flank` ([bootstraponline](https://github.com/bootstraponline))
+- [#523](https://github.com/TestArmada/flank/pull/523) Add `--local-result-dir` to make it easy to find the test result at a fixed path. ([bootstraponline](https://github.com/bootstraponline))
 
 ## v4.4.0
 

test_runner/flank.ios.yml

@@ -102,3 +102,6 @@ flank:
 
   ## The billing enabled Google Cloud Platform project name to use
   # project: delta-essence-114723
+
+  ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+  # local-result-dir: flank

test_runner/flank.yml

@@ -121,3 +121,7 @@ flank:
 
   ## The billing enabled Google Cloud Platform project name to use
   # project: delta-essence-114723
+
+  ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+  # local-result-dir: flank
+

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -19,6 +19,7 @@ import ftl.args.ArgsToString.devicesToString
 import ftl.args.ArgsToString.listToString
 import ftl.args.ArgsToString.mapToString
 import ftl.args.yml.AndroidGcloudYml
+import ftl.args.yml.AndroidGcloudYmlParams
 import ftl.args.yml.FlankYml
 import ftl.args.yml.GcloudYml
 import ftl.args.yml.YamlDeprecated
@@ -72,6 +73,7 @@ class AndroidArgs(
     override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
     override val project = cli?.project ?: flank.project
+    override val localResultDir = cli?.localResultDir ?: flank.localResultDir
 
     // computed properties not specified in yaml
     override val testShardChunks: List<List<String>> by lazy {
@@ -173,6 +175,7 @@ ${listToString(filesToDownload)}
 ${listToString(testTargetsAlwaysRun)}
       disable-sharding: $disableSharding
       project: $project
+      local-result-dir: $localResultDir
    """.trimIndent()
     }
 
@@ -199,5 +202,9 @@ ${listToString(testTargetsAlwaysRun)}
                 cli
             )
         }
+
+        fun default(): AndroidArgs {
+            return AndroidArgs(GcloudYml(), AndroidGcloudYml(AndroidGcloudYmlParams(app = ".", test = ".")), FlankYml(), "", AndroidRunCommand())
+        }
     }
 }

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -22,6 +22,7 @@ interface IArgs {
     val testTargetsAlwaysRun: List<String>
     val filesToDownload: List<String>
     val disableSharding: Boolean
+    val localResultDir: String?
 
     // computed property
     val testShardChunks: List<List<String>>

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -56,6 +56,7 @@ class IosArgs(
     override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
     override val disableSharding = cli?.disableSharding ?: flank.disableSharding
     override val project = cli?.project ?: flank.project
+    override val localResultDir = cli?.localResultsDir ?: flank.localResultDir
 
     private val iosFlank = iosFlankYml.flank
     val testTargets = cli?.testTargets ?: iosFlank.testTargets
@@ -132,6 +133,7 @@ ${listToString(filesToDownload)}
 ${listToString(testTargets)}
       disable-sharding: $disableSharding
       project: $project
+      local-result-dir: $localResultDir
     """.trimIndent()
     }
 

test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt

@@ -52,7 +52,7 @@ class AndroidGcloudYmlParams(
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 class AndroidGcloudYml(
-    val gcloud: AndroidGcloudYmlParams
+    val gcloud: AndroidGcloudYmlParams = AndroidGcloudYmlParams()
 
 ) {
     companion object : IYmlMap {

test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt

@@ -32,13 +32,18 @@ class FlankYmlParams(
     @field:JsonProperty("files-to-download")
     val filesToDownload: List<String> = emptyList(),
 
-    val project: String = ArgsHelper.getDefaultProjectId() ?: ""
+    val project: String = ArgsHelper.getDefaultProjectId() ?: "",
+
+    @field:JsonProperty("local-result-dir")
+    val localResultDir: String = defaultLocalResultDir
 ) {
     companion object : IYmlKeys {
         override val keys = listOf(
             "max-test-shards", "shard-time", "repeat-tests", "smart-flank-gcs-path", "disable-sharding",
-            "test-targets-always-run", "files-to-download", "project"
+            "test-targets-always-run", "files-to-download", "project", "local-result-dir"
         )
+
+        const val defaultLocalResultDir = "results"
     }
 
     init {

test_runner/src/main/kotlin/ftl/cli/firebase/CancelCommand.kt

@@ -1,5 +1,6 @@
 package ftl.cli.firebase
 
+import ftl.args.AndroidArgs
 import ftl.run.TestRunner
 import picocli.CommandLine
 
@@ -18,7 +19,7 @@ Reads in the matrix_ids.json file. Cancels any incomplete matrices.
 )
 class CancelCommand : Runnable {
     override fun run() {
-        TestRunner.cancelLastRun()
+        TestRunner.cancelLastRun(AndroidArgs.default())
     }
 
     @CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["Prints this help message"])

test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt

@@ -1,5 +1,6 @@
 package ftl.cli.firebase
 
+import ftl.args.AndroidArgs
 import ftl.run.TestRunner
 import kotlinx.coroutines.runBlocking
 import picocli.CommandLine.Command
@@ -21,7 +22,7 @@ Reads in the matrix_ids.json file. Refreshes any incomplete matrices.
 class RefreshCommand : Runnable {
     override fun run() {
         runBlocking {
-            TestRunner.refreshLastRun()
+            TestRunner.refreshLastRun(AndroidArgs.default())
         }
     }
 

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -273,4 +273,10 @@ class AndroidRunCommand : Runnable {
                 "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."]
     )
     var flakyTestAttempts: Int? = null
+
+    @Option(
+        names = ["--local-result-dir"],
+        description = ["Saves test result to this local folder. Deleted before each run."]
+    )
+    var localResultDir: String? = null
 }

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -214,4 +214,10 @@ class IosRunCommand : Runnable {
                 "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."]
     )
     var flakyTestAttempts: Int? = null
+
+    @Option(
+        names = ["--local-result-dir"],
+        description = ["Saves test result to this local folder. Deleted before each run."]
+    )
+    var localResultsDir: String? = null
 }

test_runner/src/main/kotlin/ftl/config/FtlConstants.kt

@@ -90,8 +90,6 @@ object FtlConstants {
         }
     }
 
-    const val localResultsDir = "results"
-
     fun configFileName(args: IArgs): String {
         return when (args) {
             is IosArgs -> defaultIosConfig

test_runner/src/main/kotlin/ftl/reports/CostReport.kt

@@ -1,5 +1,6 @@
 package ftl.reports
 
+import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
@@ -36,14 +37,14 @@ object CostReport : IReport {
         }
     }
 
-    private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices)
+    private fun write(matrices: MatrixMap, output: String, args: IArgs) {
+        val reportPath = reportPath(matrices, args)
         reportPath.write(output)
     }
 
-    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) {
+    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) {
         val output = generate(matrices)
         if (printToStdout) print(output)
-        write(matrices, output)
+        write(matrices, output, args)
     }
 }

test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt

@@ -1,6 +1,7 @@
 package ftl.reports
 
 import com.google.gson.Gson
+import ftl.args.IArgs
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestCase
@@ -77,7 +78,7 @@ object HtmlErrorReport : IReport {
         return groupJson to itemJson
     }
 
-    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) {
+    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) {
         if (testSuite == null) return
         val reactJson = reactJson(testSuite) ?: return
         val newGroupJson = reactJson.first
@@ -88,7 +89,7 @@ object HtmlErrorReport : IReport {
         templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson)
         templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson)
 
-        val writePath = Paths.get(reportPath(matrices))
+        val writePath = Paths.get(reportPath(matrices, args))
         Files.write(writePath, templateData.toByteArray())
     }
 

test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt

@@ -1,5 +1,6 @@
 package ftl.reports
 
+import ftl.args.IArgs
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
@@ -10,18 +11,18 @@ import ftl.util.Utils.write
 object JUnitReport : IReport {
     override val extension = ".xml"
 
-    private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices)
+    private fun write(matrices: MatrixMap, output: String, args: IArgs) {
+        val reportPath = reportPath(matrices, args)
         reportPath.write(output)
     }
 
-    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) {
+    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) {
         val output = testSuite.xmlToString()
 
         if (printToStdout) {
             print(output)
         } else {
-            write(matrices, output)
+            write(matrices, output, args)
         }
     }
 }

test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

@@ -1,5 +1,6 @@
 package ftl.reports
 
+import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
 import ftl.json.MatrixMap
 import ftl.json.SavedMatrix
@@ -62,14 +63,14 @@ object MatrixResultsReport : IReport {
         }
     }
 
-    private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices)
+    private fun write(matrices: MatrixMap, output: String, args: IArgs) {
+        val reportPath = reportPath(matrices, args)
         reportPath.write(output)
     }
 
-    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean) {
+    override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) {
         val output = generate(matrices)
         if (printToStdout) print(output)
-        write(matrices, output)
+        write(matrices, output, args)
     }
 }

test_runner/src/main/kotlin/ftl/reports/util/IReport.kt

@@ -1,21 +1,22 @@
 package ftl.reports.util
 
+import ftl.args.IArgs
 import ftl.json.MatrixMap
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.util.resolveLocalRunPath
 import java.nio.file.Paths
 
 interface IReport {
-    fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean = false)
+    fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean = false, args: IArgs)
 
     fun reportName(): String {
         return this::class.java.simpleName
     }
 
     val extension: String
 
-    fun reportPath(matrices: MatrixMap): String {
-        val path = resolveLocalRunPath(matrices)
+    fun reportPath(matrices: MatrixMap, args: IArgs): String {
+        val path = resolveLocalRunPath(matrices, args)
         return Paths.get(path, reportName() + extension).toString()
     }
 }

test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt

@@ -20,9 +20,9 @@ import kotlin.math.roundToInt
 
 object ReportManager {
 
-    private fun findXmlFiles(matrices: MatrixMap): List<File> {
+    private fun findXmlFiles(matrices: MatrixMap, args: IArgs): List<File> {
         val xmlFiles = mutableListOf<File>()
-        val rootFolder = File(resolveLocalRunPath(matrices))
+        val rootFolder = File(resolveLocalRunPath(matrices, args))
 
         rootFolder.walk().forEach {
             if (it.name.matches(Artifacts.testResultRgx)) {
@@ -53,10 +53,10 @@ object ReportManager {
         return matchResult?.groupValues?.last().orEmpty()
     }
 
-    private fun processXml(matrices: MatrixMap, process: (file: File) -> JUnitTestResult): JUnitTestResult? {
+    private fun processXml(matrices: MatrixMap, args: IArgs, process: (file: File) -> JUnitTestResult): JUnitTestResult? {
         var mergedXml: JUnitTestResult? = null
 
-        findXmlFiles(matrices).forEach { xmlFile ->
+        findXmlFiles(matrices, args).forEach { xmlFile ->
             val parsedXml = process(xmlFile)
             val webLink = getWebLink(matrices, xmlFile)
             val suiteName = getDeviceString(xmlFile.parentFile.name)
@@ -77,9 +77,9 @@ object ReportManager {
     private fun parseTestSuite(matrices: MatrixMap, args: IArgs): JUnitTestResult? {
         val iosXml = args is IosArgs
         return if (iosXml) {
-            processXml(matrices, ::parseAllSuitesXml)
+            processXml(matrices, args, ::parseAllSuitesXml)
         } else {
-            processXml(matrices, ::parseOneSuiteXml)
+            processXml(matrices, args, ::parseOneSuiteXml)
         }
     }
 
@@ -96,16 +96,16 @@ object ReportManager {
             CostReport,
             MatrixResultsReport
         ).map {
-            it.run(matrices, testSuite, printToStdout = true)
+            it.run(matrices, testSuite, printToStdout = true, args = args)
         }
 
         if (!testSuccessful) {
             listOf(
                 HtmlErrorReport
-            ).map { it.run(matrices, testSuite) }
+            ).map { it.run(matrices, testSuite, printToStdout = false, args = args) }
         }
 
-        JUnitReport.run(matrices, testSuite)
+        JUnitReport.run(matrices, testSuite, printToStdout = false, args = args)
         processJunitXml(testSuite, args)
 
         // FTL has a bug with matrix roll-up when using flakyTestAttempts