Merge branch 'master' into rainnapper/weblink

README.md

@@ -201,6 +201,10 @@ flank:
 
   ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
   # full-junit-result: false
+
+  ## Disables flank results upload on gcloud storage.
+  ## Default: false
+  # disable-results-upload: false
 ```
 
 ### Android example
@@ -420,6 +424,10 @@ flank:
 
   ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
   # full-junit-result: false
+
+  ## Disables flank results upload on gcloud storage.
+  ## Default: false
+  # disable-results-upload: false
 ```
 
 ### Android code coverage

release_notes.md

@@ -2,6 +2,9 @@
 - [#857](https://github.com/Flank/flank/pull/857) Added multimodule setup for test app. ([piotradamczyk5](https://github.com/piotradamczyk5))
 - [#837](https://github.com/Flank/flank/pull/837) Added obfuscate option to dump shards. ([piotradamczyk5](https://github.com/piotradamczyk5))
 - [#868](https://github.com/Flank/flank/pull/868) Restored weblinks to all test results, not just failures. ([rainnapper](https://github.com/rainnapper))
+- [#828](https://github.com/Flank/flank/pull/828) Store test results in gcloud bucket. ([adamfilipow92](https://github.com/adamfilipow92))
+-
+-
 
 ## v20.06.2
 - [#853](https://github.com/Flank/flank/pull/853) Store @Ignore tests in the JUnit XML without sending ignored tests to FTL. ([piotradamczyk5](https://github.com/piotradamczyk5), [adamfilipow92](https://github.com/adamfilipow92))

test_runner/docs/ascii/flank.jar_-android-run.adoc

@@ -162,6 +162,9 @@ Configuration is read from flank.yml
 *--app*=_<app>_::
   The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
 
+*--disable-results-upload*::
+  Disables flank results upload on gcloud storage.
+
 *--test*=_<test>_::
   The path to the binary file containing instrumentation tests. The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://.
 
@@ -174,6 +177,9 @@ Configuration is read from flank.yml
 *--no-auto-google-login*::
   Google account not logged in (default behavior). Use --auto-google-login to enable
 
+*--app*=_<app>_::
+  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+
 *--use-orchestrator*::
   Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more information about Android Test Orchestrator.
 

test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc

@@ -13,6 +13,7 @@ flank.jar
 
 *flank.jar
  firebase test android run* [*-h*] [*--async*] [*--auto-google-login*]
+                                     [*--disable-results-upload*]
                                      [*--disable-sharding*] [*--dry*]
                                      [*--dump-shards*] [*--full-junit-result*]
                                      [*--ignore-failed-tests*] [*--keep-file-path*]
@@ -186,6 +187,9 @@ Configuration is read from flank.yml
 *--no-auto-google-login*::
   Google account not logged in (default behavior). Use --auto-google-login to enable
 
+*--app*=_<app>_::
+  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+
 *--use-orchestrator*::
   Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more information about Android Test Orchestrator.
 

test_runner/flank.ios.yml

@@ -140,3 +140,6 @@ flank:
 
   ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
   # full-junit-result: false
+
+  ## Disables flank results upload on gcloud storage.
+  # disable-results-upload: false

test_runner/flank.yml

@@ -212,3 +212,6 @@ flank:
 
   ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests.
   # full-junit-result: false
+
+  ## Disables flank results upload on gcloud storage.
+  # disable-results-upload: false

test_runner/src/main/kotlin/ftl/args/CommonArgs.kt

@@ -33,5 +33,6 @@ data class CommonArgs(
     override val fullJUnitResult: Boolean,
     override val ignoreFailedTests: Boolean,
     override val keepFilePath: Boolean,
-    override val outputStyle: OutputStyle
+    override val outputStyle: OutputStyle,
+    override val disableResultsUpload: Boolean
 ) : IArgs

test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt

@@ -40,7 +40,8 @@ fun CommonConfig.createCommonArgs(
     ignoreFailedTests = flank.ignoreFailedTests!!,
     filesToDownload = flank.filesToDownload!!,
     disableSharding = flank.disableSharding!!,
-    localResultDir = flank.localResultsDir!!
+    localResultDir = flank.localResultsDir!!,
+    disableResultsUpload = flank.disableResultsUpload!!
 ).apply {
     ArgsHelper.createJunitBucket(project, smartFlankGcsPath)
 }

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -51,6 +51,8 @@ interface IArgs {
     val hasMultipleExecutions
         get() = flakyTestAttempts > 0 || (!disableSharding && maxTestShards > 0)
 
+    val disableResultsUpload: Boolean get() = false
+
     fun useLocalResultDir() = localResultDir != defaultLocalResultsDir
 
     companion object {

test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt

@@ -133,6 +133,13 @@ data class CommonFlankConfig @JsonIgnore constructor(
     @set:JsonProperty("output-style")
     var outputStyle: String? by data
 
+    @set:CommandLine.Option(
+        names = ["--disable-results-upload"],
+        description = ["Disables flank results upload on gcloud storage."]
+    )
+    @set:JsonProperty("disable-results-upload")
+    var disableResultsUpload: Boolean? by data
+
     constructor() : this(mutableMapOf<String, Any?>().withDefault { null })
 
     companion object : IYmlKeys {
@@ -153,7 +160,8 @@ data class CommonFlankConfig @JsonIgnore constructor(
             "legacy-junit-result",
             "ignore-failed-tests",
             "keep-file-path",
-            "output-style"
+            "output-style",
+            "disable-results-upload"
         )
 
         const val defaultLocalResultsDir = "results"
@@ -174,6 +182,7 @@ data class CommonFlankConfig @JsonIgnore constructor(
             ignoreFailedTests = false
             keepFilePath = false
             outputStyle = null
+            disableResultsUpload = false
         }
     }
 }

test_runner/src/main/kotlin/ftl/gc/GcStorage.kt

@@ -80,11 +80,11 @@ object GcStorage {
         }
     }
 
-    fun uploadCiJUnitXml(testResult: JUnitTestResult, args: IArgs, fileName: String) {
-        if (args.resultsBucket.isBlank() || args.resultsDir.isBlank()) return
+    fun uploadReportResult(testResult: String, args: IArgs, fileName: String) {
+        if (args.resultsBucket.isBlank() || args.resultsDir.isBlank() || args.disableResultsUpload) return
         upload(
             filePath = fileName,
-            fileBytes = testResult.xmlToString().toByteArray(),
+            fileBytes = testResult.toByteArray(),
             rootGcsBucket = args.resultsBucket,
             runGcsPath = args.resultsDir
         )

test_runner/src/main/kotlin/ftl/reports/CostReport.kt

@@ -2,6 +2,7 @@ package ftl.reports
 
 import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
+import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
@@ -41,5 +42,6 @@ object CostReport : IReport {
         val output = generate(matrices)
         if (printToStdout) print(output)
         write(matrices, output, args)
+        GcStorage.uploadReportResult(output, args, fileName())
     }
 }

test_runner/src/main/kotlin/ftl/reports/FullJUnitReport.kt

@@ -1,12 +1,12 @@
 package ftl.reports
 
 import ftl.args.IArgs
+import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.reports.xml.xmlToString
 
-/** Calculates cost based on the matrix map. Always run. */
 object FullJUnitReport : IReport {
     override val extension = ".xml"
 
@@ -17,5 +17,6 @@ object FullJUnitReport : IReport {
         } else {
             write(matrices, output, args)
         }
+        GcStorage.uploadReportResult(output, args, fileName())
     }
 }

test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt

@@ -2,6 +2,7 @@ package ftl.reports
 
 import com.google.gson.Gson
 import ftl.args.IArgs
+import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestCase
@@ -28,10 +29,11 @@ object HtmlErrorReport : IReport {
         args: IArgs
     ) {
         val suites = result?.testsuites?.process()?.takeIf { it.isNotEmpty() } ?: return
-        Files.write(
-            Paths.get(reportPath(matrices, args)),
-            suites.createHtmlReport().toByteArray()
-        )
+        Paths.get(reportPath(matrices, args)).let {
+            val htmlReport = suites.createHtmlReport()
+            Files.write(it, htmlReport.toByteArray())
+            GcStorage.uploadReportResult(htmlReport, args, it.fileName.toString())
+        }
     }
 }
 

test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt

@@ -6,7 +6,6 @@ import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.reports.xml.xmlToString
-import java.io.File
 
 object JUnitReport : IReport {
     override val extension = ".xml"
@@ -21,6 +20,6 @@ object JUnitReport : IReport {
         } else {
             write(matrices, output, args)
         }
-        GcStorage.uploadCiJUnitXml(result, args, reportPath(matrices, args).let(::File).name)
+        GcStorage.uploadReportResult(result.xmlToString(), args, fileName())
     }
 }

test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt

@@ -2,6 +2,7 @@ package ftl.reports
 
 import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
+import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
@@ -45,6 +46,7 @@ object MatrixResultsReport : IReport {
                 writer.println("$indent$failed matrices failed")
                 writer.println()
             }
+
             if (matrices.map.isNotEmpty()) {
                 writer.println("More details are available at [${matrices.map.values.first().webLinkWithoutExecutionDetails}]")
                 writer.println(matrices.map.values.toList().asPrintableTable())
@@ -58,5 +60,6 @@ object MatrixResultsReport : IReport {
         val output = generate(matrices)
         if (printToStdout) print(output)
         write(matrices, output, args)
+        GcStorage.uploadReportResult(output, args, fileName())
     }
 }

test_runner/src/main/kotlin/ftl/reports/util/IReport.kt

@@ -2,7 +2,6 @@ package ftl.reports.util
 
 import ftl.args.IArgs
 import ftl.json.MatrixMap
-import ftl.reports.JUnitReport
 import ftl.reports.xml.model.JUnitTestResult
 import ftl.util.resolveLocalRunPath
 import ftl.util.write
@@ -14,15 +13,17 @@ interface IReport {
     fun reportName(): String {
         return this::class.java.simpleName
     }
+
     fun fileName() = reportName() + extension
+
     val extension: String
 
     fun reportPath(matrices: MatrixMap, args: IArgs): String {
         val path = resolveLocalRunPath(matrices, args)
         return Paths.get(path, fileName()).toString()
     }
     fun write(matrices: MatrixMap, output: String, args: IArgs) {
-        val reportPath = JUnitReport.reportPath(matrices, args)
+        val reportPath = reportPath(matrices, args)
         reportPath.write(output)
     }
 }

test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt

@@ -1,5 +1,6 @@
 package ftl.reports.util
 
+import com.google.common.annotations.VisibleForTesting
 import ftl.args.IArgs
 import ftl.args.IgnoredTestCases
 import ftl.args.IosArgs
@@ -67,7 +68,8 @@ object ReportManager {
         return matchResult?.groupValues?.last().orEmpty()
     }
 
-    private fun processXmlFromFile(
+    @VisibleForTesting
+    internal fun processXmlFromFile(
         matrices: MatrixMap,
         args: IArgs,
         process: (file: File) -> JUnitTestResult

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -1426,6 +1426,33 @@ AndroidArgs
         val args = AndroidArgs.load(yaml)
         assertEquals(AVAILABLE_SHARD_COUNT_RANGE.last, args.maxTestShards)
     }
+
+    @Test
+    fun `should set disableResultsUpload to true`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+        flank:
+          max-test-shards: -1
+          disable-results-upload: true
+        """.trimIndent()
+        val args = AndroidArgs.load(yaml)
+        assertTrue(args.disableResultsUpload)
+    }
+
+    @Test
+    fun `should disableResultsUpload set to false`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+        flank:
+          max-test-shards: -1
+        """.trimIndent()
+        val args = AndroidArgs.load(yaml)
+        assertFalse(args.disableResultsUpload)
+    }
 }
 
 private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = load(StringReader(yamlData), cli)

test_runner/src/test/kotlin/ftl/fixtures/success_result/shard_0/NexusLowRes-28-en-portrait/test_result_1.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8' ?>
+<testsuite name="" tests="2" failures="0" errors="0" skipped="0" time="6.548" timestamp="2018-09-09T00:16:49"
+           hostname="localhost">
+    <properties/>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="1.082" />
+    <testcase name="testPasses2" classname="com.example.app.ExampleUiTest" time="0.327"/>
+</testsuite>