Add JUnit XML uploading for smart flank

After each test run, the JUnit XML file is uploaded to Google storage
Flank · Nov 9, 2018 · 74e366a · 74e366a
1 parent c11234b
commit 74e366a
Show file tree

Hide file tree

Showing 17 changed files with 97 additions and 20 deletions.
diff --git a/test_runner/flank.yml b/test_runner/flank.yml
@@ -27,13 +27,22 @@ gcloud:
       version: 28
 
 flank:
+  # Google cloud storage path to store the JUnit XML results from the last run.
+  #
+  # junitGcsPath: gs://tmp_flank/flank/test_app_android.xml
+
   # test shards - the amount of groups to split the test suite into
   # set to -1 to use one shard per test.
+  #
   testShards: 1
+
   # repeat tests - the amount of times to run the tests.
   # 1 runs the tests once. 10 runs all the tests 10x
+  #
   repeatTests: 1
+
   # always run - these tests are inserted at the beginning of every shard
   # useful if you need to grant permissions or login before other tests run
+  #
   # test-targets-always-run:
     # - class com.example.app.ExampleUiTest#testPasses
diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
@@ -10,7 +10,8 @@ import ftl.android.UnsupportedVersionId
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
 import ftl.args.ArgsHelper.calculateShards
-import ftl.args.ArgsHelper.getGcsBucket
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.devicesToString
@@ -58,6 +59,7 @@ class AndroidArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val junitGcsPath = flank.junitGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     // computed properties not specified in yaml
@@ -80,7 +82,8 @@ class AndroidArgs(
     }
 
     init {
-        resultsBucket = getGcsBucket(projectId, gcloud.resultsBucket)
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.junitGcsPath)
 
         if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(appApk)
@@ -150,6 +153,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      junitGcsPath: $junitGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
    """.trimIndent()

diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt
@@ -102,25 +102,31 @@ object ArgsHelper {
         return testShardChunks
     }
 
-    fun getGcsBucket(projectId: String, resultsBucket: String): String {
+    fun createJunitBucket(projectId: String, junitGcsPath: String) {
+        if (FtlConstants.useMock || junitGcsPath.isEmpty()) return
+        val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/')
+        createGcsBucket(projectId, bucket)
+    }
+
+    fun createGcsBucket(projectId: String, bucket: String): String {
         // com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list
         // when testing, use a hard coded results bucket instead.
-        if (FtlConstants.useMock) return resultsBucket
+        if (FtlConstants.useMock) return bucket
         // test lab supports using a special free storage bucket
         // because we don't have access to the root account, it won't show up in the storage list.
-        if (resultsBucket.startsWith("test-lab-")) return resultsBucket
+        if (bucket.startsWith("test-lab-")) return bucket
 
         val storage = StorageOptions.newBuilder().setProjectId(projectId).build().service
         val bucketLabel = mapOf(Pair("flank", ""))
         val storageLocation = "us-central1"
 
-        val bucketListOption = Storage.BucketListOption.prefix(resultsBucket)
+        val bucketListOption = Storage.BucketListOption.prefix(bucket)
         val storageList = storage.list(bucketListOption).values?.map { it.name } ?: emptyList()
-        val bucket = storageList.find { it == resultsBucket }
-        if (bucket != null) return bucket
+        val targetBucket = storageList.find { it == bucket }
+        if (targetBucket != null) return targetBucket
 
         return storage.create(
-            BucketInfo.newBuilder(resultsBucket)
+            BucketInfo.newBuilder(targetBucket)
                 .setStorageClass(StorageClass.REGIONAL)
                 .setLocation(storageLocation)
                 .setLabels(bucketLabel)

diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt
@@ -15,6 +15,7 @@ interface IArgs {
     // FlankYml
     val testShards: Int
     val repeatTests: Int
+    val junitGcsPath: String
     val testTargetsAlwaysRun: List<String>
 
     // computed property

diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt
@@ -2,6 +2,8 @@ package ftl.args
 
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
+import ftl.args.ArgsHelper.createGcsBucket
+import ftl.args.ArgsHelper.createJunitBucket
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.validateTestMethods
 import ftl.args.ArgsHelper.yamlMapper
@@ -28,7 +30,7 @@ class IosArgs(
 ) : IArgs {
 
     private val gcloud = gcloudYml.gcloud
-    override val resultsBucket = gcloud.resultsBucket
+    override val resultsBucket: String
     override val recordVideo = gcloud.recordVideo
     override val testTimeout = gcloud.timeout
     override val async = gcloud.async
@@ -44,6 +46,7 @@ class IosArgs(
     private val flank = flankYml.flank
     override val testShards = flank.testShards
     override val repeatTests = flank.repeatTests
+    override val junitGcsPath = flank.junitGcsPath
     override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun
 
     private val iosFlank = iosFlankYml.flank
@@ -67,6 +70,9 @@ class IosArgs(
     }
 
     init {
+        resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
+        createJunitBucket(projectId, flank.junitGcsPath)
+
         if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) {
             assertGcsFileExists(xctestrunZip)
         } else {
@@ -111,6 +117,7 @@ ${devicesToString(devices)}
     flank:
       testShards: $testShards
       repeatTests: $repeatTests
+      junitGcsPath: $junitGcsPath
       test-targets-always-run:
 ${listToString(testTargetsAlwaysRun)}
       # iOS flank

diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt
@@ -2,24 +2,35 @@ package ftl.args.yml
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
+import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.util.Utils.fatalError
 
 /** Flank specific parameters for both iOS and Android */
 @JsonIgnoreProperties(ignoreUnknown = true)
 class FlankYmlParams(
     val testShards: Int = 1,
     val repeatTests: Int = 1,
+    val junitGcsPath: String = "",
 
     @field:JsonProperty("test-targets-always-run")
     val testTargetsAlwaysRun: List<String> = emptyList()
 ) {
     companion object : IYmlKeys {
-        override val keys = listOf("testShards", "repeatTests", "test-targets-always-run")
+        override val keys = listOf("testShards", "repeatTests", "junitGcsPath", "test-targets-always-run")
     }
 
     init {
         if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1")
         if (repeatTests < 1) fatalError("repeatTests must be >= 1")
+
+        if (junitGcsPath.isNotEmpty()) {
+            if (!junitGcsPath.startsWith(GCS_PREFIX)) {
+                fatalError("junitGcsPath must start with gs://")
+            }
+            if (junitGcsPath.count { it == '/' } <= 2 || !junitGcsPath.endsWith(".xml"))  {
+                fatalError("junitGcsPath must be in the format gs://bucket/foo.xml")
+            }
+        }
     }
 }
 

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt
@@ -5,6 +5,7 @@ import com.google.cloud.storage.Storage
 import com.google.cloud.storage.StorageOptions
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper
 import ftl.args.AndroidArgs
+import ftl.args.IArgs
 import ftl.args.IosArgs
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
@@ -40,6 +41,24 @@ object GcStorage {
             runGcsPath = runGcsPath
         )
 
+    fun uploadJunitXml(localJunitXml: String, args: IArgs) {
+        if (args.junitGcsPath.isEmpty()) return
+        if (File(localJunitXml).exists().not()) return
+
+        // bucket/path/to/object
+        val rawPath = args.junitGcsPath.drop(GCS_PREFIX.length)
+        val bucket = rawPath.substringBefore('/')
+        val name = rawPath.substringAfter('/')
+
+        val fileBlob = BlobInfo.newBuilder(bucket, name).build()
+
+        try {
+            storage.create(fileBlob, Files.readAllBytes(Paths.get(localJunitXml)))
+        } catch (e: Exception) {
+            fatalError(e)
+        }
+    }
+
     fun uploadAppApk(args: AndroidArgs, gcsBucket: String, runGcsPath: String): String =
         upload(args.appApk, gcsBucket, runGcsPath)
 

diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt
@@ -12,6 +12,8 @@ import java.io.StringWriter
 /** Calculates cost based on the matrix map. Always run. */
 object CostReport : IReport {
 
+    override val extension = ".txt"
+
     private fun estimate(matrices: MatrixMap): String {
         var totalBillableVirtualMinutes = 0L
         var totalBillablePhysicalMinutes = 0L
@@ -35,7 +37,7 @@ object CostReport : IReport {
     }
 
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".txt"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt
@@ -13,6 +13,8 @@ import java.nio.file.Paths
  * Outputs HTML report for Bitrise based on JUnit XML. Only run on failures.
  * */
 object HtmlErrorReport : IReport {
+    override val extension = ".html"
+
     data class Group(val key: String, val name: String, val startIndex: Int, val count: Int)
     data class Item(val key: String, val name: String, val link: String)
 
@@ -86,7 +88,7 @@ object HtmlErrorReport : IReport {
         templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson)
         templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson)
 
-        val writePath = Paths.get(reportPath(matrices) + ".html")
+        val writePath = Paths.get(reportPath(matrices))
         Files.write(writePath, templateData.toByteArray())
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt
@@ -8,8 +8,10 @@ import ftl.util.Utils.write
 
 /** Calculates cost based on the matrix map. Always run. */
 object JUnitReport : IReport {
+    override val extension = ".xml"
+
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".xml"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt
@@ -21,6 +21,7 @@ Example:
 
  **/
 object MatrixResultsReport : IReport {
+    override val extension = ".txt"
 
     private val percentFormat by lazy { DecimalFormat("#0.00") }
 
@@ -62,7 +63,7 @@ object MatrixResultsReport : IReport {
     }
 
     private fun write(matrices: MatrixMap, output: String) {
-        val reportPath = reportPath(matrices) + ".txt"
+        val reportPath = reportPath(matrices)
         reportPath.write(output)
     }
 

diff --git a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt
@@ -12,8 +12,10 @@ interface IReport {
         return this::class.java.simpleName
     }
 
+    val extension: String
+
     fun reportPath(matrices: MatrixMap): String {
         val path = resolveLocalRunPath(matrices)
-        return Paths.get(path, reportName()).toString()
+        return Paths.get(path, reportName() + extension).toString()
     }
 }
diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -2,6 +2,7 @@ package ftl.reports.util
 
 import ftl.args.IArgs
 import ftl.args.IosArgs
+import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.CostReport
 import ftl.reports.HtmlErrorReport
@@ -81,6 +82,9 @@ object ReportManager {
 
         JUnitReport.run(matrices, testSuite)
 
+        val localJunitXmlPath = JUnitReport.reportPath(matrices)
+        GcStorage.uploadJunitXml(localJunitXmlPath, args)
+
         if (!testSuccessful) {
             listOf(
                 HtmlErrorReport

diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
@@ -197,6 +197,7 @@ AndroidArgs
     flank:
       testShards: 7
       repeatTests: 8
+      junitGcsPath:${' '}
       test-targets-always-run:
         - class example.Test#grantPermission
         - class example.Test#grantPermission2

diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt
@@ -4,7 +4,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.args.ArgsHelper.assertFileExists
 import ftl.args.ArgsHelper.assertGcsFileExists
 import ftl.args.ArgsHelper.calculateShards
-import ftl.args.ArgsHelper.getGcsBucket
+import ftl.args.ArgsHelper.createGcsBucket
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.validateTestMethods
 import ftl.args.yml.GcloudYml
@@ -153,7 +153,7 @@ class ArgsHelperTest {
 
     @Test
     fun getGcsBucket_succeeds() {
-        getGcsBucket("123", "results_bucket")
+        createGcsBucket("123", "results_bucket")
     }
 
     @Test

diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
@@ -147,6 +147,7 @@ IosArgs
     flank:
       testShards: 7
       repeatTests: 8
+      junitGcsPath:${' '}
       test-targets-always-run:
         - a/testGrantPermissions
         - a/testGrantPermissions2

diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt
@@ -9,6 +9,7 @@ import org.junit.Test
 import org.junit.contrib.java.lang.system.SystemErrRule
 import org.junit.contrib.java.lang.system.SystemOutRule
 import org.junit.runner.RunWith
+import org.mockito.Mockito.`when`
 import org.mockito.Mockito.mock
 
 @RunWith(FlankTestRunner::class)
@@ -25,12 +26,16 @@ class ReportManagerTest {
     @Test
     fun generate_fromErrorResult() {
         val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result")
-        ReportManager.generate(matrix, mock(AndroidArgs::class.java))
+        val mockArgs = mock(AndroidArgs::class.java)
+        `when`(mockArgs.junitGcsPath).thenReturn("")
+        ReportManager.generate(matrix, mockArgs)
     }
 
     @Test
     fun generate_fromSuccessResult() {
         val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result")
-        ReportManager.generate(matrix, mock(AndroidArgs::class.java))
+        val mockArgs = mock(AndroidArgs::class.java)
+        `when`(mockArgs.junitGcsPath).thenReturn("")
+        ReportManager.generate(matrix, mockArgs)
     }
 }