Add files-to-download

README.md

@@ -77,6 +77,9 @@ flank:
   # test targets - a list of tests to run. omit to run all tests.
   test-targets:
     - b/testBasicSelection
+  # regex is matched against bucket paths, for example: 2019-01-09_00:18:07.314000_hCMY/shard_0/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun
+  files-to-download:
+    - .*\.png$
 ```
 
 ### Android example
@@ -127,6 +130,9 @@ flank:
   # useful if you need to grant permissions or login before other tests run
   test-targets-always-run:
     - class com.example.app.ExampleUiTest#testPasses
+ # regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt
+  files-to-download:
+    - .*\.mp4$
 ```
 
 ### CI integration

contributing.md

@@ -1,4 +1,5 @@
 ## Setup
+
 - Install [Oracle JDK 8](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
   - JDK 9 or later will not work
 - [IntelliJ Toolbox](https://www.jetbrains.com/toolbox/app/)
@@ -21,6 +22,11 @@ See the main readme for instructions on how to run the iOS and Android samples.
 - Update `IArgs` with new property
 - Update `AndroidArgs` to reference the propery and update `toString`
 - Update `IosArgs` to reference the propery and `toString`
+- Add test to `IosArgsTest`
+- Add test to `IosRunCommandTest` and update `empty_params_parse_null` test
+- Add test to `AndroidArgsTest`
+- Add test to `AndroidRunCommandTest` and update `empty_params_parse_null` test
+- Update Android/iOS example config in README.md, `flank.yml` and `flank.ios.yml`
 
 ## CLA
 

release_notes.md

@@ -1,6 +1,7 @@
 ## v4.2.0 (unreleased)
 
 - Fix create Gcs bucket [#444](https://github.com/TestArmada/flank/pull/444)
+- Add `directories-to-download` to Android and iOS. Specify a list of regular expressions to download files from the Google Cloud Storage bucket. [#441](https://github.com/TestArmada/flank/pull/441)
 
 ## v4.1.1
 

test_runner/build.gradle.kts

@@ -135,7 +135,7 @@ dependencies {
 // https://stackoverflow.com/a/45286710
 configurations.all {
     resolutionStrategy {
-        force("com.google.guava:guava:23.6-jre")
+        force("com.google.guava:guava:25.1-jre")
         force(Libs.KOTLIN_REFLECT)
         exclude(group = "com.google.guava", module = "guava-jdk5")
     }

test_runner/flank.ios.yml

@@ -41,3 +41,7 @@ flank:
   #
   # test-targets:
   #   - b/testBasicSelection
+
+  # # regex is matched against bucket paths, for example: 2019-01-09_00:18:07.314000_hCMY/shard_0/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun
+  # files-to-download:
+  #   - .*\.png$

test_runner/flank.yml

@@ -46,3 +46,7 @@ flank:
   #
   # test-targets-always-run:
   #   - class com.example.app.ExampleUiTest#testPasses
+
+  # # regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt
+  # files-to-download:
+  #   - .*\.mp4$

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -67,6 +67,7 @@ class AndroidArgs(
     override val repeatTests = cli?.repeatTests ?: flank.repeatTests
     override val smartFlankGcsPath = flank.smartFlankGcsPath
     override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
+    override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
 
     // computed properties not specified in yaml
     override val testShardChunks: List<List<String>> by lazy {
@@ -159,6 +160,8 @@ ${devicesToString(devices)}
       testShards: $testShards
       repeatTests: $repeatTests
       smartFlankGcsPath: $smartFlankGcsPath
+      files-to-download:
+${listToString(filesToDownload)}
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
    """.trimIndent()

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -18,6 +18,7 @@ interface IArgs {
     val repeatTests: Int
     val smartFlankGcsPath: String
     val testTargetsAlwaysRun: List<String>
+    val filesToDownload: List<String>
 
     // computed property
     val testShardChunks: List<List<String>>

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -52,6 +52,7 @@ class IosArgs(
     override val repeatTests = cli?.repeatTests ?: flank.repeatTests
     override val smartFlankGcsPath = flank.smartFlankGcsPath
     override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
+    override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
 
     private val iosFlank = iosFlankYml.flank
     val testTargets = cli?.testTargets ?: iosFlank.testTargets
@@ -123,6 +124,8 @@ ${devicesToString(devices)}
       smartFlankGcsPath: $smartFlankGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
+      files-to-download:
+${listToString(filesToDownload)}
       # iOS flank
       test-targets:
 ${listToString(testTargets)}

test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt

@@ -13,10 +13,13 @@ class FlankYmlParams(
     val smartFlankGcsPath: String = "",
 
     @field:JsonProperty("test-targets-always-run")
-    val testTargetsAlwaysRun: List<String> = emptyList()
+    val testTargetsAlwaysRun: List<String> = emptyList(),
+
+    @field:JsonProperty("files-to-download")
+    val filesToDownload: List<String> = emptyList()
 ) {
     companion object : IYmlKeys {
-        override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run")
+        override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run", "files-to-download")
     }
 
     init {

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -235,6 +235,16 @@ class AndroidRunCommand : Runnable {
     )
     var testTargetsAlwaysRun: List<String>? = null
 
+    @Option(
+        names = ["--files-to-download"],
+        split = ",",
+        description = ["A list of paths that will be downloaded from the resulting bucket " +
+                "to the local results folder after the test is complete. These must be absolute paths " +
+                "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " +
+                "Path names are restricted to the characters a-zA-Z0-9_-./+."]
+    )
+    var filesToDownload: List<String>? = null
+
     @Option(
         names = ["--results-dir"],
         description = [

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -126,6 +126,16 @@ class IosRunCommand : Runnable {
     )
     var testTargets: List<String>? = null
 
+    @Option(
+        names = ["--files-to-download"],
+        split = ",",
+        description = ["A list of paths that will be downloaded from the resulting bucket " +
+                "to the local results folder after the test is complete. These must be absolute paths " +
+                "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " +
+                "Path names are restricted to the characters a-zA-Z0-9_-./+."]
+    )
+    var filesToDownload: List<String>? = null
+
     @Option(
         names = ["--test"],
         description = ["The path to the test package (a zip file containing the iOS app " +

test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt

@@ -11,7 +11,7 @@ import ftl.reports.MatrixResultsReport
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.reports.xml.parseOneSuiteXml
 import ftl.reports.xml.parseAllSuitesXml
-import ftl.util.ArtifactRegex
+import ftl.util.Artifacts
 import ftl.util.resolveLocalRunPath
 import java.io.File
 import java.nio.file.Paths
@@ -23,7 +23,7 @@ object ReportManager {
         val rootFolder = File(resolveLocalRunPath(matrices))
 
         rootFolder.walk().forEach {
-            if (it.name.matches(ArtifactRegex.testResultRgx)) {
+            if (it.name.matches(Artifacts.testResultRgx)) {
                 xmlFiles.add(it)
             }
         }

test_runner/src/main/kotlin/ftl/run/TestRunner.kt

@@ -22,7 +22,7 @@ import ftl.gc.GcToolResults
 import ftl.json.MatrixMap
 import ftl.json.SavedMatrix
 import ftl.reports.util.ReportManager
-import ftl.util.ArtifactRegex
+import ftl.util.Artifacts
 import ftl.util.MatrixState
 import ftl.util.StopWatch
 import ftl.util.Utils
@@ -188,8 +188,7 @@ object TestRunner {
         return MatrixMap(map, path)
     }
 
-    /** fetch test_result_0.xml & *.png **/
-    private fun fetchArtifacts(matrixMap: MatrixMap) {
+    private fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) {
         println("FetchArtifacts")
         val fields = Storage.BlobListOption.fields(Storage.BlobField.NAME)
 
@@ -206,10 +205,11 @@ object TestRunner {
                 launch {
                     val prefix = Storage.BlobListOption.prefix(matrix.gcsPathWithoutRootBucket)
                     val result = GcStorage.storage.list(matrix.gcsRootBucket, prefix, fields)
+                    val artifactsList = Artifacts.regexList(args)
 
                     result.iterateAll().forEach { blob ->
                         val blobPath = blob.blobId.name
-                        if (blobPath.matches(ArtifactRegex.testResultRgx)) {
+                        if (artifactsList.any { blobPath.matches(it) }) {
                             val downloadFile = Paths.get(FtlConstants.localResultsDir, blobPath)
                             print(".")
                             if (!downloadFile.toFile().exists()) {
@@ -325,7 +325,7 @@ object TestRunner {
 
         refreshMatrices(matrixMap, args)
         pollMatrices(matrixMap, args)
-        fetchArtifacts(matrixMap)
+        fetchArtifacts(matrixMap, args)
 
         // Must generate reports *after* fetching xml artifacts since reports require xml
         val exitCode = ReportManager.generate(matrixMap, args)
@@ -346,7 +346,7 @@ object TestRunner {
 
         if (!args.async) {
             pollMatrices(matrixMap, args)
-            fetchArtifacts(matrixMap)
+            fetchArtifacts(matrixMap, args)
 
             val exitCode = ReportManager.generate(matrixMap, args)
             System.exit(exitCode)

test_runner/src/main/kotlin/ftl/util/Artifacts.kt

@@ -0,0 +1,12 @@
+package ftl.util
+
+import ftl.args.IArgs
+
+object Artifacts {
+
+    val testResultRgx = Regex(".*test_result_\\d+\\.xml$")
+
+    fun regexList(args: IArgs): List<Regex> {
+        return listOf(testResultRgx) + args.filesToDownload.map { Regex(it) }
+    }
+}

test_runner/src/main/resources/version.txt

@@ -1 +1 @@
-v4.1-SNAPSHOT
+v4.2-SNAPSHOT

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -59,6 +59,9 @@ class AndroidArgsTest {
         flank:
           testShards: 7
           repeatTests: 8
+          files-to-download:
+            - /sdcard/screenshots
+            - /sdcard/screenshots2
           test-targets-always-run:
             - class example.Test#grantPermission
             - class example.Test#grantPermission2
@@ -154,6 +157,7 @@ class AndroidArgsTest {
             // FlankYml
             assert(testShards, 7)
             assert(repeatTests, 8)
+            assert(filesToDownload, listOf("/sdcard/screenshots", "/sdcard/screenshots2"))
             assert(
                 testTargetsAlwaysRun, listOf(
                     "class example.Test#grantPermission",
@@ -206,6 +210,9 @@ AndroidArgs
       testShards: 7
       repeatTests: 8
       smartFlankGcsPath:${' '}
+      files-to-download:
+        - /sdcard/screenshots
+        - /sdcard/screenshots2
       test-targets-always-run:
         - class example.Test#grantPermission
         - class example.Test#grantPermission2
@@ -245,6 +252,7 @@ AndroidArgs
             // FlankYml
             assert(testShards, 1)
             assert(repeatTests, 1)
+            assert(filesToDownload, empty)
             assert(testTargetsAlwaysRun, empty)
         }
     }
@@ -476,6 +484,22 @@ AndroidArgs
         assertThat(androidArgs.directoriesToPull).isEqualTo(listOf("a", "b"))
     }
 
+    @Test
+    fun cli_filesToDownload() {
+        val cli = AndroidRunCommand()
+        CommandLine(cli).parse("--files-to-download=a,b")
+
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+      """
+        assertThat(AndroidArgs.load(yaml).filesToDownload).isEmpty()
+
+        val androidArgs = AndroidArgs.load(yaml, cli)
+        assertThat(androidArgs.filesToDownload).isEqualTo(listOf("a", "b"))
+    }
+
     @Test
     fun cli_device() {
         val cli = AndroidRunCommand()

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -53,6 +53,8 @@ class IosArgsTest {
         flank:
           testShards: 7
           repeatTests: 8
+          files-to-download:
+            - /sdcard/screenshots
           test-targets-always-run:
             - a/testGrantPermissions
             - a/testGrantPermissions2
@@ -159,6 +161,8 @@ IosArgs
       test-targets-always-run:
         - a/testGrantPermissions
         - a/testGrantPermissions2
+      files-to-download:
+        - /sdcard/screenshots
       # iOS flank
       test-targets:
         - b/testBasicSelection
@@ -194,6 +198,7 @@ IosArgs
             assert(testShards, 1)
             assert(repeatTests, 1)
             assert(testTargetsAlwaysRun, emptyList<String>())
+            assert(filesToDownload, emptyList<String>())
 
             // IosFlankYml
             assert(testTargets, empty)
@@ -507,4 +512,20 @@ IosArgs
         assertThat(IosArgs.load(yaml).resultsDir).isEqualTo("a")
         assertThat(IosArgs.load(yaml, cli).resultsDir).isEqualTo("b")
     }
+
+    @Test
+    fun cli_filesToDownload() {
+        val cli = IosRunCommand()
+        CommandLine(cli).parse("--files-to-download=a,b")
+
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+      """
+        assertThat(IosArgs.load(yaml).filesToDownload).isEmpty()
+
+        val androidArgs = IosArgs.load(yaml, cli)
+        assertThat(androidArgs.filesToDownload).isEqualTo(listOf("a", "b"))
+    }
 }

test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt

@@ -80,6 +80,7 @@ class AndroidRunCommandTest {
         assertThat(cmd.testShards).isNull()
         assertThat(cmd.repeatTests).isNull()
         assertThat(cmd.testTargetsAlwaysRun).isNull()
+        assertThat(cmd.filesToDownload).isNull()
         assertThat(cmd.resultsDir).isNull()
     }
 
@@ -273,4 +274,12 @@ class AndroidRunCommandTest {
 
         assertThat(cmd.resultsDir).isEqualTo("a")
     }
+
+    @Test
+    fun filesToDownload_parse() {
+        val cmd = AndroidRunCommand()
+        CommandLine(cmd).parse("--files-to-download=a,b")
+
+        assertThat(cmd.filesToDownload).isEqualTo(arrayListOf("a", "b"))
+    }
 }