Add additional-app-test-apks support (#542)

* Parse additionalAppTestApks * Add additional-app-test-apks support * Scope test shards to individual apks * Set default dispatcher when using async * Use default dispatcher when refreshing matrix * Update TestRunner.kt
Flank · May 1, 2019 · f9af5aa · f9af5aa
1 parent 42bfc7b
commit f9af5aa
Show file tree

Hide file tree

Showing 29 changed files with 367 additions and 209 deletions.
diff --git a/README.md b/README.md
@@ -307,6 +307,12 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
+
+  ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair.
+  # additional-app-test-apks:
+  #  - app: ../test_app/apks/app-debug.apk
+  #    test: ../test_app/apks/app-debug-androidTest.apk
+  #  - test: ../test_app/apks/app-debug-androidTest.apk
 ```
 
 ### Android code coverage

diff --git a/release_notes.md b/release_notes.md
@@ -1,6 +1,6 @@
 ## next (unreleased)
 
-- 
+- [#542](https://github.com/TestArmada/flank/pull/542) Add `additional-app-test-apks` to include multiple app/test apk pairs in a single run. ([bootstraponline](https://github.com/bootstraponline))
 
 ## v6.0.1
 

diff --git a/test_runner/flank.yml b/test_runner/flank.yml
@@ -128,4 +128,9 @@ flank:
 
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
-
+
+  ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair.
+  # additional-app-test-apks:
+  #  - app: ../test_app/apks/app-debug.apk
+  #    test: ../test_app/apks/app-debug-androidTest.apk
+  #  - test: ../test_app/apks/app-debug-androidTest.apk
diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
@@ -1,23 +1,22 @@
 package ftl.args
 
-import com.linkedin.dex.parser.DexParser
-import com.linkedin.dex.parser.TestMethod
 import ftl.android.AndroidCatalog
 import ftl.android.IncompatibleModelVersion
 import ftl.android.SupportedDeviceConfig
 import ftl.android.UnsupportedModelId
 import ftl.android.UnsupportedVersionId
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
-import ftl.args.ArgsHelper.calculateShards
 import ftl.args.ArgsHelper.createGcsBucket
 import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
+import ftl.args.ArgsToString.apksToString
 import ftl.args.ArgsToString.devicesToString
 import ftl.args.ArgsToString.listToString
 import ftl.args.ArgsToString.mapToString
+import ftl.args.yml.AndroidFlankYml
 import ftl.args.yml.AndroidGcloudYml
 import ftl.args.yml.AndroidGcloudYmlParams
 import ftl.args.yml.FlankYml
@@ -26,19 +25,15 @@ import ftl.args.yml.YamlDeprecated
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FtlConstants
-import ftl.config.FtlConstants.useMock
-import ftl.filter.TestFilters
-import ftl.gc.GcStorage
-import ftl.util.Utils
 import java.nio.file.Files
 import java.nio.file.Path
-import kotlinx.coroutines.runBlocking
 
 // set default values, init properties, etc.
 class AndroidArgs(
     gcloudYml: GcloudYml,
     androidGcloudYml: AndroidGcloudYml,
     flankYml: FlankYml,
+    androidFlankYml: AndroidFlankYml,
     override val data: String,
     val cli: AndroidRunCommand? = null
 ) : IArgs {
@@ -76,21 +71,8 @@ class AndroidArgs(
     override val project = cli?.project ?: flank.project
     override val localResultDir = cli?.localResultDir ?: flank.localResultDir
 
-    // computed properties not specified in yaml
-    override val testShardChunks: List<List<String>> by lazy {
-        if (disableSharding) return@lazy listOf(emptyList<String>())
-
-        // Download test APK if necessary so it can be used to validate test methods
-        var testLocalApk = testApk
-        if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
-            runBlocking {
-                testLocalApk = GcStorage.downloadTestApk(this@AndroidArgs)
-            }
-        }
-
-        val filteredTests = getTestMethods(testLocalApk)
-        calculateShards(filteredTests, this)
-    }
+    private val androidFlank = androidFlankYml.flank
+    val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks
 
     init {
         resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket)
@@ -113,21 +95,6 @@ class AndroidArgs(
         devices.forEach { device -> assertDeviceSupported(device) }
     }
 
-    private fun getTestMethods(testLocalApk: String): List<String> {
-        val allTestMethods = DexParser.findTestMethods(testLocalApk)
-        require(allTestMethods.isNotEmpty()) { Utils.fatalError("Test APK has no tests") }
-        val testFilter = TestFilters.fromTestTargets(testTargets)
-        val filteredTests = allTestMethods
-            .asSequence()
-            .distinct()
-            .filter(testFilter.shouldRun)
-            .map(TestMethod::testName)
-            .map { "class $it" }
-            .toList()
-        require(useMock || filteredTests.isNotEmpty()) { Utils.fatalError("All tests filtered out") }
-        return filteredTests
-    }
-
     private fun assertDeviceSupported(device: Device) {
         val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version)
         when (deviceConfigTest) {
@@ -178,12 +145,15 @@ ${listToString(testTargetsAlwaysRun)}
       disable-sharding: $disableSharding
       project: $project
       local-result-dir: $localResultDir
+      # Android Flank Yml
+      additional-app-test-apks:
+${apksToString(additionalAppTestApks)}
    """.trimIndent()
     }
 
     companion object : IArgsCompanion {
         override val validArgs by lazy {
-            mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml)
+            mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml, AndroidFlankYml)
         }
 
         fun load(data: Path, cli: AndroidRunCommand? = null): AndroidArgs =
@@ -195,18 +165,26 @@ ${listToString(testTargetsAlwaysRun)}
             val flankYml = yamlMapper.readValue(data, FlankYml::class.java)
             val gcloudYml = yamlMapper.readValue(data, GcloudYml::class.java)
             val androidGcloudYml = yamlMapper.readValue(data, AndroidGcloudYml::class.java)
+            val androidFlankYml = yamlMapper.readValue(data, AndroidFlankYml::class.java)
 
             return AndroidArgs(
                 gcloudYml,
                 androidGcloudYml,
                 flankYml,
+                androidFlankYml,
                 data,
                 cli
             )
         }
 
         fun default(): AndroidArgs {
-            return AndroidArgs(GcloudYml(), AndroidGcloudYml(AndroidGcloudYmlParams(app = ".", test = ".")), FlankYml(), "", AndroidRunCommand())
+            return AndroidArgs(
+                GcloudYml(),
+                AndroidGcloudYml(AndroidGcloudYmlParams(app = ".", test = ".")),
+                FlankYml(),
+                AndroidFlankYml(),
+                "",
+                AndroidRunCommand())
         }
     }
 }
diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt
@@ -0,0 +1,43 @@
+package ftl.args
+
+import com.linkedin.dex.parser.DexParser
+import com.linkedin.dex.parser.TestMethod
+import ftl.config.FtlConstants
+import ftl.filter.TestFilters
+import ftl.gc.GcStorage
+import ftl.util.Utils
+import kotlinx.coroutines.runBlocking
+
+object AndroidTestShard {
+
+    // computed properties not specified in yaml
+    fun getTestShardChunks(args: AndroidArgs, testApk: String): List<List<String>> {
+        if (args.disableSharding) return listOf(emptyList())
+
+        // Download test APK if necessary so it can be used to validate test methods
+        var testLocalApk = testApk
+        if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
+            runBlocking {
+                testLocalApk = GcStorage.download(testApk)
+            }
+        }
+
+        val filteredTests = getTestMethods(args, testLocalApk)
+        return ArgsHelper.calculateShards(filteredTests, args)
+    }
+
+    private fun getTestMethods(args: AndroidArgs, testLocalApk: String): List<String> {
+        val allTestMethods = DexParser.findTestMethods(testLocalApk)
+        require(allTestMethods.isNotEmpty()) { Utils.fatalError("Test APK has no tests") }
+        val testFilter = TestFilters.fromTestTargets(args.testTargets)
+        val filteredTests = allTestMethods
+            .asSequence()
+            .distinct()
+            .filter(testFilter.shouldRun)
+            .map(TestMethod::testName)
+            .map { "class $it" }
+            .toList()
+        require(FtlConstants.useMock || filteredTests.isNotEmpty()) { Utils.fatalError("All tests filtered out") }
+        return filteredTests
+    }
+}
diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt b/test_runner/src/main/kotlin/ftl/args/ArgsToString.kt
@@ -1,5 +1,6 @@
 package ftl.args
 
+import ftl.args.yml.AppTestPair
 import ftl.config.Device
 
 object ArgsToString {
@@ -21,4 +22,8 @@ object ArgsToString {
         return devices.filterNotNull()
             .joinToString("\n") { "$it" }
     }
+
+    fun apksToString(devices: List<AppTestPair>): String {
+        return devices.joinToString("\n") { (app, test) -> "        - app: $app\n          test: $test" }
+    }
 }
diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt
@@ -2,6 +2,7 @@ package ftl.args
 
 import ftl.args.yml.FlankYmlParams
 
+// Properties common to both Android and iOS
 interface IArgs {
     // original YAML data
     val data: String
@@ -27,8 +28,5 @@ interface IArgs {
     val disableSharding: Boolean
     val localResultDir: String
 
-    // computed property
-    val testShardChunks: List<List<String>>
-
     fun useLocalResultDir() = localResultDir != FlankYmlParams.defaultLocalResultDir
 }
diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt
@@ -63,7 +63,7 @@ class IosArgs(
     val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull()
 
     // computed properties not specified in yaml
-    override val testShardChunks: List<List<String>> by lazy {
+    val testShardChunks: List<List<String>> by lazy {
         if (disableSharding) return@lazy listOf(emptyList<String>())
 
         val validTestMethods = Xctestrun.findTestNames(xctestrunFile)

diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt
@@ -0,0 +1,32 @@
+package ftl.args.yml
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+
+data class AppTestPair(
+    val app: String?,
+    val test: String
+)
+
+/** Flank specific parameters for Android */
+@JsonIgnoreProperties(ignoreUnknown = true)
+class AndroidFlankYmlParams(
+    @field:JsonProperty("additional-app-test-apks")
+    val additionalAppTestApks: List<AppTestPair> = emptyList()
+) {
+    companion object : IYmlKeys {
+        override val keys = listOf("additional-app-test-apks")
+    }
+}
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+class AndroidFlankYml(
+    @field:JsonProperty("flank")
+    private val parsedFlank: AndroidFlankYmlParams? = AndroidFlankYmlParams()
+) {
+    val flank = parsedFlank ?: AndroidFlankYmlParams()
+
+    companion object : IYmlMap {
+        override val map = mapOf("flank" to AndroidFlankYmlParams.keys)
+    }
+}
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/RefreshCommand.kt
@@ -22,7 +22,7 @@ Reads in the matrix_ids.json file. Refreshes any incomplete matrices.
 class RefreshCommand : Runnable {
     override fun run() {
         runBlocking {
-            TestRunner.refreshLastRun(AndroidArgs.default())
+            TestRunner.refreshLastRun(AndroidArgs.default(), emptyList())
         }
     }