Smart Flank

test_runner/build.gradle.kts

@@ -38,7 +38,7 @@ tasks.withType<JacocoReport> {
 }
 
 tasks.withType<KotlinCompile>().configureEach {
-    kotlinOptions.allWarningsAsErrors = true
+    kotlinOptions.allWarningsAsErrors = false
 }
 
 apply {

test_runner/flank.yml

@@ -27,13 +27,22 @@ gcloud:
       version: 28
 
 flank:
+  # Google cloud storage path to store the JUnit XML results from the last run.
+  #
+  # junitGcsPath: gs://tmp_flank/flank/test_app_android.xml
+
   # test shards - the amount of groups to split the test suite into
   # set to -1 to use one shard per test.
+  #
   testShards: 1
+
   # repeat tests - the amount of times to run the tests.
   # 1 runs the tests once. 10 runs all the tests 10x
+  #
   repeatTests: 1
+
   # always run - these tests are inserted at the beginning of every shard
   # useful if you need to grant permissions or login before other tests run
+  #
   # test-targets-always-run:
     # - class com.example.app.ExampleUiTest#testPasses

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -11,7 +11,9 @@ import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
 import ftl.args.ArgsHelper.calculateShards
 import ftl.args.ArgsHelper.evaluateFilePath
-import ftl.args.ArgsHelper.getGcsBucket
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
+import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.devicesToString
@@ -63,6 +65,7 @@ class AndroidArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val junitGcsPath = flank.junitGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     // computed properties not specified in yaml
@@ -85,7 +88,8 @@ class AndroidArgs(
     }
 
     init {
-        resultsBucket = getGcsBucket(projectId, gcloud.resultsBucket)
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.junitGcsPath)
 
         if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(appApk)
@@ -157,6 +161,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      junitGcsPath: $junitGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
    """.trimIndent()

test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt

@@ -124,25 +124,31 @@ object ArgsHelper {
         return testShardChunks
     }
 
-    fun getGcsBucket(projectId: String, resultsBucket: String): String {
+    fun createJunitBucket(projectId: String, junitGcsPath: String) {
+        if (FtlConstants.useMock || junitGcsPath.isEmpty()) return
+        val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/')
+        createGcsBucket(projectId, bucket)
+    }
+
+    fun createGcsBucket(projectId: String, bucket: String): String {
         // com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list
         // when testing, use a hard coded results bucket instead.
-        if (FtlConstants.useMock) return resultsBucket
+        if (FtlConstants.useMock) return bucket
         // test lab supports using a special free storage bucket
         // because we don't have access to the root account, it won't show up in the storage list.
-        if (resultsBucket.startsWith("test-lab-")) return resultsBucket
+        if (bucket.startsWith("test-lab-")) return bucket
 
         val storage = StorageOptions.newBuilder().setProjectId(projectId).build().service
         val bucketLabel = mapOf(Pair("flank", ""))
         val storageLocation = "us-central1"
 
-        val bucketListOption = Storage.BucketListOption.prefix(resultsBucket)
+        val bucketListOption = Storage.BucketListOption.prefix(bucket)
         val storageList = storage.list(bucketListOption).values?.map { it.name } ?: emptyList()
-        val bucket = storageList.find { it == resultsBucket }
-        if (bucket != null) return bucket
+        val targetBucket = storageList.find { it == bucket }
+        if (targetBucket != null) return targetBucket
 
         return storage.create(
-            BucketInfo.newBuilder(resultsBucket)
+            BucketInfo.newBuilder(targetBucket)
                 .setStorageClass(StorageClass.REGIONAL)
                 .setLocation(storageLocation)
                 .setLabels(bucketLabel)

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -15,6 +15,7 @@ interface IArgs {
     // FlankYml
     val testShards: Int
     val repeatTests: Int
+    val junitGcsPath: String
     val testTargetsAlwaysRun: List<String>
 
     // computed property

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -2,6 +2,8 @@ package ftl.args
 
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.validateTestMethods
@@ -29,7 +31,7 @@ class IosArgs(
 ) : IArgs {
 
     private val gcloud = gcloudYml.gcloud
-    override val resultsBucket = gcloud.resultsBucket
+    override val resultsBucket: String
     override val recordVideo = gcloud.recordVideo
     override val testTimeout = gcloud.timeout
     override val async = gcloud.async
@@ -45,6 +47,7 @@ class IosArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val junitGcsPath = flank.junitGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     private val iosFlank = iosFlankYml.flank
@@ -68,6 +71,9 @@ class IosArgs(
     }
 
     init {
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.junitGcsPath)
+
         if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(xctestrunZip)
         } else {
@@ -114,6 +120,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      junitGcsPath: $junitGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
       # iOS flank

test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt

@@ -2,24 +2,35 @@ package ftl.args.yml
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
+import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.util.Utils.fatalError
 
 /** Flank specific parameters for both iOS and Android */
 @JsonIgnoreProperties(ignoreUnknown = true)
 class FlankYmlParams(
     val testShards: Int = 1,
     val repeatTests: Int = 1,
+    val junitGcsPath: String = "",
 
     @field:JsonProperty("test-targets-always-run")
     val testTargetsAlwaysRun: List<String> = emptyList()
 ) {
     companion object : IYmlKeys {
-        override val keys = listOf("testShards", "repeatTests", "test-targets-always-run")
+        override val keys = listOf("testShards", "repeatTests", "junitGcsPath", "test-targets-always-run")
     }
 
     init {
         if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1")
         if (repeatTests < 1) fatalError("repeatTests must be >= 1")
+
+        if (junitGcsPath.isNotEmpty()) {
+            if (!junitGcsPath.startsWith(GCS_PREFIX)) {
+                fatalError("junitGcsPath must start with gs://")
+            }
+            if (junitGcsPath.count { it == '/' } <= 2 || !junitGcsPath.endsWith(".xml")) {
+                fatalError("junitGcsPath must be in the format gs://bucket/foo.xml")
+            }
+        }
     }
 }
 

test_runner/src/main/kotlin/ftl/gc/GcStorage.kt

@@ -5,6 +5,7 @@ import com.google.cloud.storage.Storage
 import com.google.cloud.storage.StorageOptions
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper
 import ftl.args.AndroidArgs
+import ftl.args.IArgs
 import ftl.args.IosArgs
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
@@ -40,6 +41,24 @@ object GcStorage {
             runGcsPath = runGcsPath
         )
 
+    fun uploadJunitXml(localJunitXml: String, args: IArgs) {
+        if (args.junitGcsPath.isEmpty()) return
+        if (File(localJunitXml).exists().not()) return
+
+        // bucket/path/to/object
+        val rawPath = args.junitGcsPath.drop(GCS_PREFIX.length)
+        val bucket = rawPath.substringBefore('/')
+        val name = rawPath.substringAfter('/')
+
+        val fileBlob = BlobInfo.newBuilder(bucket, name).build()
+
+        try {
+            storage.create(fileBlob, Files.readAllBytes(Paths.get(localJunitXml)))
+        } catch (e: Exception) {
+            fatalError(e)
+        }
+    }
+
     fun uploadAppApk(args: AndroidArgs, gcsBucket: String, runGcsPath: String): String =
         upload(args.appApk, gcsBucket, runGcsPath)
 
@@ -60,6 +79,10 @@ object GcStorage {
     fun downloadTestApk(args: AndroidArgs): String =
         download(args.testApk)
 
+    // junit xml may not exist. ignore error if it doesn't exist
+    fun downloadJunitXml(args: IArgs): String =
+        download(args.junitGcsPath, ignoreError = true)
+
     private fun upload(file: String, fileBytes: ByteArray, rootGcsBucket: String, runGcsPath: String): String {
         val fileName = Paths.get(file).fileName.toString()
         val gcsFilePath = GCS_PREFIX + join(rootGcsBucket, runGcsPath, fileName)
@@ -76,12 +99,12 @@ object GcStorage {
         return gcsFilePath
     }
 
-    private fun download(gcsUriString: String): String {
+    private fun download(gcsUriString: String, ignoreError: Boolean = false): String {
         val gcsURI = URI.create(gcsUriString)
         val bucket = gcsURI.authority
         val path = gcsURI.path.drop(1) // Drop leading slash
 
-        val outputFile = File.createTempFile("apk", null)
+        val outputFile = File.createTempFile("tmp", null)
         outputFile.deleteOnExit()
 
         try {
@@ -91,6 +114,7 @@ object GcStorage {
             output.channel.transferFrom(readChannel, 0, Long.MAX_VALUE)
             output.close()
         } catch (e: Exception) {
+            if (ignoreError) return ""
             fatalError(e)
         }
 

test_runner/src/main/kotlin/ftl/reports/CostReport.kt

@@ -12,6 +12,8 @@ import java.io.StringWriter
 /** Calculates cost based on the matrix map. Always run. */
 object CostReport : IReport {
 
+    override val extension = ".txt"
+
     private fun estimate(matrices: MatrixMap): String {
         var totalBillableVirtualMinutes = 0L
         var totalBillablePhysicalMinutes = 0L
@@ -35,7 +37,7 @@ object CostReport : IReport {
     }
 
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".txt"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt

@@ -13,6 +13,8 @@ import java.nio.file.Paths
  * Outputs HTML report for Bitrise based on JUnit XML. Only run on failures.
  * */
 object HtmlErrorReport : IReport {
+    override val extension = ".html"
+
     data class Group(val key: String, val name: String, val startIndex: Int, val count: Int)
     data class Item(val key: String, val name: String, val link: String)
 
@@ -86,7 +88,7 @@ object HtmlErrorReport : IReport {
         templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson)
         templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson)
 
-        val writePath = Paths.get(reportPath(matrices) + ".html")
+        val writePath = Paths.get(reportPath(matrices))
         Files.write(writePath, templateData.toByteArray())
     }
 

test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt

@@ -8,8 +8,10 @@ import ftl.util.Utils.write
 
 /** Calculates cost based on the matrix map. Always run. */
 object JUnitReport : IReport {
+    override val extension = ".xml"
+
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".xml"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

@@ -21,6 +21,7 @@ Example:
 
  **/
 object MatrixResultsReport : IReport {
+    override val extension = ".txt"
 
     private val percentFormat by lazy { DecimalFormat("#0.00") }
 
@@ -62,7 +63,7 @@ object MatrixResultsReport : IReport {
     }
 
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".txt"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

test_runner/src/main/kotlin/ftl/reports/util/IReport.kt

@@ -12,8 +12,10 @@ interface IReport {
         return this::class.java.simpleName
     }
 
+    val extension: String
+
     fun reportPath(matrices: MatrixMap): String {
         val path = resolveLocalRunPath(matrices)
-        return Paths.get(path, reportName()).toString()
+        return Paths.get(path, reportName() + extension).toString()
     }
 }