Dump shards

release_notes.md

@@ -12,6 +12,7 @@
 - [#781](https://github.com/Flank/flank/pull/781) Remove local exists check on cloud results-dir. Fixes crash when results-dir is set by the user. ([adamfilipow92](https://github.com/adamfilipow92))
 - [#656](https://github.com/Flank/flank/issues/656) Improve error message reporting. ([adamfilipow92](https://github.com/adamfilipow92))
 - [#783](https://github.com/Flank/flank/pull/783) Use legacy results for iOS by default. ([pawelpasterz](https://github.com/pawelpasterz))
+- [#794](https://github.com/Flank/flank/pull/794) Enhance `--dump-shards` to dump shards from all test apks ([bootstraponline](https://github.com/bootstraponline), [jan-gogo](https://github.com/jan-gogo))
 
 ## v20.05.1
 

test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt

@@ -224,7 +224,11 @@ object ArgsHelper {
         return ArgsFileVisitor("glob:$filePath").walk(searchDir)
     }
 
-    fun calculateShards(filteredTests: List<FlankTestMethod>, args: IArgs): ShardChunks {
+    fun calculateShards(
+        filteredTests: List<FlankTestMethod>,
+        args: IArgs,
+        forcedShardCount: Int? = null
+    ): ShardChunks {
         if (filteredTests.isEmpty()) {
             // Avoid unnecessary computing if we already know there aren't tests to run.
             return listOf(emptyList())
@@ -234,7 +238,7 @@ object ArgsHelper {
             listOf(filteredTests.map { it.testName }.toMutableList())
         } else {
             val oldTestResult = GcStorage.downloadJunitXml(args) ?: JUnitTestResult(mutableListOf())
-            val shardCount = Shard.shardCountByTime(filteredTests, oldTestResult, args)
+            val shardCount = forcedShardCount ?: Shard.shardCountByTime(filteredTests, oldTestResult, args)
             Shard.createShardsByShardCount(filteredTests, oldTestResult, args, shardCount).stringShards()
         }
 

test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorParser.kt

@@ -5,7 +5,7 @@ internal object ConfigurationErrorParser {
     //region regex patterns
     private val propertyNameRegex = "(?<=property\\s)[a-z]*".toRegex()
     private val referenceChainRegex = "(?<=chain:\\s).*(?=[)])".toRegex()
-    private val referenceChainCleanUpRegex = "(?<=[\\[])\"?[\\w]*\"?(?=])".toRegex()
+    private val referenceChainCleanUpRegex = "(?<=[\\[])\"?(\\w|-)*\"?(?=])".toRegex()
     private val lineAndColumnRegex = "((?<=line:\\s)\\d*), column:\\s(\\d*)".toRegex()
     //endregion
 

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -1,8 +1,6 @@
 package ftl.cli.firebase.test.android
 
 import ftl.args.AndroidArgs
-import ftl.args.AndroidTestShard
-import ftl.args.ShardChunks
 import ftl.args.yml.AppTestPair
 import ftl.cli.firebase.test.CommonRunCommand
 import ftl.config.Device
@@ -12,12 +10,12 @@ import ftl.config.FtlConstants.defaultAndroidVersion
 import ftl.config.FtlConstants.defaultLocale
 import ftl.config.FtlConstants.defaultOrientation
 import ftl.mock.MockServer
-import ftl.run.common.prettyPrint
+import ftl.run.ANDROID_SHARD_FILE
+import ftl.run.dumpShards
 import ftl.run.newTestRun
 import kotlinx.coroutines.runBlocking
 import picocli.CommandLine.Command
 import picocli.CommandLine.Option
-import java.nio.file.Files
 import java.nio.file.Paths
 
 @Command(
@@ -45,25 +43,18 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {
         val config = AndroidArgs.load(Paths.get(configPath), cli = this)
 
         if (dumpShards) {
-            val testShardChunks: ShardChunks = AndroidTestShard.getTestShardChunks(config, config.testApk!!)
-            val testShardChunksJson: String = prettyPrint.toJson(testShardChunks)
-
-            Files.write(Paths.get(shardFile), testShardChunksJson.toByteArray())
-            println("Saved shards to $shardFile")
-        } else {
-            runBlocking {
-                newTestRun(config)
-            }
+            dumpShards(config)
+        } else runBlocking {
+            newTestRun(config)
         }
     }
 
-    companion object {
-        private const val shardFile = "android_shards.json"
-    }
-
     // Flank debug
 
-    @Option(names = ["--dump-shards"], description = ["Dumps the shards to $shardFile for debugging"])
+    @Option(
+        names = ["--dump-shards"],
+        description = ["Measures test shards from given test apks and writes them into $ANDROID_SHARD_FILE file instead of executing."]
+    )
     var dumpShards: Boolean = false
 
     // Flank specific

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -7,12 +7,12 @@ import ftl.config.FtlConstants
 import ftl.config.FtlConstants.defaultIosModel
 import ftl.config.FtlConstants.defaultIosVersion
 import ftl.mock.MockServer
-import ftl.run.common.prettyPrint
+import ftl.run.IOS_SHARD_FILE
+import ftl.run.dumpShards
 import ftl.run.newTestRun
 import kotlinx.coroutines.runBlocking
 import picocli.CommandLine.Command
 import picocli.CommandLine.Option
-import java.nio.file.Files
 import java.nio.file.Paths
 
 @Command(
@@ -39,23 +39,18 @@ class IosRunCommand : CommonRunCommand(), Runnable {
         val config = IosArgs.load(Paths.get(configPath), cli = this)
 
         if (dumpShards) {
-            val testShardChunksJson: String = prettyPrint.toJson(config.testShardChunks)
-            Files.write(Paths.get(shardFile), testShardChunksJson.toByteArray())
-            println("Saved shards to $shardFile")
-        } else {
-            runBlocking {
-                newTestRun(config)
-            }
+            dumpShards(config)
+        } else runBlocking {
+            newTestRun(config)
         }
     }
 
-    companion object {
-        private const val shardFile = "ios_shards.json"
-    }
-
     // Flank debug
 
-    @Option(names = ["--dump-shards"], description = ["Dumps the shards to $shardFile for debugging"])
+    @Option(
+        names = ["--dump-shards"],
+        description = ["Measures test shards from given test apks and writes them into $IOS_SHARD_FILE file instead of executing."]
+    )
     var dumpShards: Boolean = false
 
     // Flank specific

test_runner/src/main/kotlin/ftl/run/DumpShards.kt

@@ -0,0 +1,45 @@
+package ftl.run
+
+import ftl.args.AndroidArgs
+import ftl.args.IosArgs
+import ftl.run.common.prettyPrint
+import ftl.run.model.AndroidMatrixTestShards
+import ftl.run.platform.android.getAndroidMatrixShards
+import ftl.util.FlankFatalError
+import java.nio.file.Files
+import java.nio.file.Paths
+
+fun dumpShards(args: AndroidArgs) {
+    if (!args.isInstrumentationTest) throw FlankFatalError(
+        "Cannot dump shards for non instrumentation test, ensure test apk has been set."
+    )
+    val shards: AndroidMatrixTestShards = getAndroidMatrixShards(args)
+    saveShardChunks(
+        shardFilePath = ANDROID_SHARD_FILE,
+        shards = shards,
+        size = shards.size
+    )
+}
+
+fun dumpShards(args: IosArgs) {
+    saveShardChunks(
+        shardFilePath = IOS_SHARD_FILE,
+        shards = args.testShardChunks,
+        size = args.testShardChunks.size
+    )
+}
+
+private fun saveShardChunks(
+    shardFilePath: String,
+    shards: Any,
+    size: Int
+) {
+    Files.write(
+        Paths.get(shardFilePath),
+        prettyPrint.toJson(shards).toByteArray()
+    )
+    println("Saved $size shards to $shardFilePath")
+}
+
+const val ANDROID_SHARD_FILE = "android_shards.json"
+const val IOS_SHARD_FILE = "ios_shards.json"

test_runner/src/main/kotlin/ftl/run/model/AndroidMatrixTestShards.kt

@@ -0,0 +1,3 @@
+package ftl.run.model
+
+typealias AndroidMatrixTestShards = Map<String, AndroidTestShards>

test_runner/src/main/kotlin/ftl/run/model/AndroidTestShards.kt

@@ -0,0 +1,7 @@
+package ftl.run.model
+
+data class AndroidTestShards(
+    val app: String,
+    val test: String,
+    val shards: Map<String, List<String>> = emptyMap()
+)

test_runner/src/main/kotlin/ftl/run/model/InstrumentationTestApk.kt

@@ -0,0 +1,8 @@
+package ftl.run.model
+
+import ftl.util.FileReference
+
+data class InstrumentationTestApk(
+    val app: FileReference = FileReference(),
+    val test: FileReference = FileReference()
+)

test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt

@@ -3,8 +3,8 @@ package ftl.run.platform
 import com.google.api.services.testing.Testing
 import com.google.api.services.testing.model.TestMatrix
 import ftl.args.AndroidArgs
-import ftl.args.AndroidTestShard
 import ftl.args.ShardChunks
+import ftl.run.platform.android.getAndroidShardChunks
 import ftl.args.yml.ResolvedApks
 import ftl.gc.GcAndroidDevice
 import ftl.gc.GcAndroidTestMatrix
@@ -40,7 +40,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
 
     args.resolveApks().forEachIndexed { index: Int, apks: ResolvedApks ->
         val testShards = apks.test?.let { test ->
-            AndroidTestShard.getTestShardChunks(args, test)
+            getAndroidShardChunks(args, test)
         }
         // We can't return if testShards is null since it can be a robo test.
         testShards?.let {

test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt

@@ -0,0 +1,44 @@
+package ftl.run.platform.android
+
+import ftl.args.AndroidArgs
+import ftl.args.ShardChunks
+import ftl.run.model.AndroidMatrixTestShards
+import ftl.run.model.AndroidTestShards
+import ftl.run.model.InstrumentationTestApk
+import ftl.util.asFileReference
+
+fun getAndroidMatrixShards(
+    args: AndroidArgs
+): AndroidMatrixTestShards =
+    getInstrumentationShardChunks(
+        args = args,
+        testApks = args.createInstrumentationTestApks()
+    ).asMatrixTestShards()
+
+private fun AndroidArgs.createInstrumentationTestApks(): List<InstrumentationTestApk> =
+    listOfNotNull(
+        testApk?.let { testApk ->
+            InstrumentationTestApk(
+                app = appApk.asFileReference(),
+                test = testApk.asFileReference()
+            )
+        }
+    ) + additionalAppTestApks.map {
+        InstrumentationTestApk(
+            app = (it.app ?: appApk).asFileReference(),
+            test = it.test.asFileReference()
+        )
+    }
+
+private fun Map<InstrumentationTestApk, ShardChunks>.asMatrixTestShards(): AndroidMatrixTestShards =
+    map { (testApks, shards: List<List<String>>) ->
+        AndroidTestShards(
+            app = testApks.app.local,
+            test = testApks.test.local,
+            shards = shards.mapIndexed { index, testCases ->
+                "shard-$index" to testCases
+            }.toMap()
+        )
+    }.mapIndexed { index, androidTestShards ->
+        "matrix-$index" to androidTestShards
+    }.toMap()

test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidShardChunks.kt

@@ -0,0 +1,15 @@
+package ftl.run.platform.android
+
+import ftl.args.AndroidArgs
+import ftl.args.ShardChunks
+import ftl.run.model.InstrumentationTestApk
+import ftl.util.asFileReference
+
+fun getAndroidShardChunks(
+    args: AndroidArgs,
+    testApk: String
+): ShardChunks =
+    getInstrumentationShardChunks(
+        args = args,
+        testApks = listOf(InstrumentationTestApk(test = testApk.asFileReference()))
+    ).flatMap { (_, shardChunks) -> shardChunks }

test_runner/src/main/kotlin/ftl/run/platform/android/GetInstrumentationShardChunks.kt

@@ -0,0 +1,55 @@
+package ftl.run.platform.android
+
+import com.linkedin.dex.parser.DexParser
+import ftl.args.AndroidArgs
+import ftl.args.ArgsHelper
+import ftl.args.ShardChunks
+import ftl.config.FtlConstants
+import ftl.filter.TestFilter
+import ftl.filter.TestFilters
+import ftl.run.model.InstrumentationTestApk
+import ftl.util.FlankTestMethod
+import ftl.util.downloadIfNeeded
+import java.io.File
+
+fun getInstrumentationShardChunks(
+    args: AndroidArgs,
+    testApks: List<InstrumentationTestApk>
+): Map<InstrumentationTestApk, ShardChunks> =
+    getFlankTestMethods(
+        testApks = testApks.download(),
+        testFilter = TestFilters.fromTestTargets(args.testTargets)
+    ).mapValues { (_, testMethods: List<FlankTestMethod>) ->
+        if (testMethods.isNotEmpty()) {
+            ArgsHelper.calculateShards(testMethods, args, args.numUniformShards)
+        } else {
+            printNoTests(testApks)
+            emptyList()
+        }
+    }
+
+private fun getFlankTestMethods(
+    testApks: List<InstrumentationTestApk>,
+    testFilter: TestFilter
+): Map<InstrumentationTestApk, List<FlankTestMethod>> =
+    testApks.associateWith { testApk ->
+        DexParser.findTestMethods(testApk.test.local).asSequence().distinct().filter(testFilter.shouldRun).map { testMethod ->
+            FlankTestMethod(
+                testName = "class ${testMethod.testName}",
+                ignored = testMethod.annotations.any { it.name == "org.junit.Ignore" }
+            )
+        }.toList()
+    }
+
+private fun List<InstrumentationTestApk>.download(): List<InstrumentationTestApk> =
+    map { reference ->
+        reference.copy(
+            app = reference.app.downloadIfNeeded(),
+            test = reference.test.downloadIfNeeded()
+        )
+    }
+
+private fun printNoTests(testApks: List<InstrumentationTestApk>) {
+    val testApkNames = testApks.joinToString(", ") { pathname -> File(pathname.test.local).name }
+    println("${FtlConstants.indent}No tests for $testApkNames")
+}