feat: Change uploading file output

Flank · Dec 28, 2020 · a9bf787 · a9bf787
1 parent 02b7cd1
commit a9bf787
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 54 deletions.
diff --git a/common/src/main/kotlin/flank.common/Strings.kt b/common/src/main/kotlin/flank.common/Strings.kt
@@ -6,6 +6,8 @@ import java.nio.file.Paths
 
 fun String.withNewLineAtTheEnd() = plus(System.lineSeparator())
 
+fun String.startWithNewLine() = System.lineSeparator() + this
+
 fun String.normalizeLineEnding(): String {
     // required for tests to pass on Windows
     return this.replace("\r\n", "\n")

diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt
@@ -29,6 +29,7 @@ object FtlConstants {
     const val matrixIdsFile = "matrix_ids.json"
     const val applicationName = "Flank"
     const val GCS_PREFIX = "gs://"
+    const val GCS_STORAGE_LINK = "https://console.developers.google.com/storage/browser/"
     const val runTimeout = "-1"
 
     val JSON_FACTORY: JsonFactory by lazy { Utils.getDefaultJsonFactory() }

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt
@@ -12,9 +12,9 @@ import com.google.common.annotations.VisibleForTesting
 import flank.common.join
 import flank.common.logLn
 import ftl.args.IArgs
-import ftl.args.IosArgs
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
+import ftl.config.FtlConstants.GCS_STORAGE_LINK
 import ftl.config.credential
 import ftl.json.MatrixMap
 import ftl.reports.xml.model.JUnitTestResult
@@ -72,14 +72,11 @@ object GcStorage {
 
         // bucket/path/to/object
         val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length)
-        val bucket = rawPath.substringBefore('/')
-        val name = rawPath.substringAfter('/')
 
-        val fileBlob = BlobInfo.newBuilder(bucket, name).build()
-        runWithProgress(
-            startMessage = "Uploading smart flank XML",
-            action = { storage.create(fileBlob, testResult.xmlToString().toByteArray()) },
-            onError = { throw FlankGeneralError(it) }
+        testResult.xmlToString().toByteArray().uploadWithProgress(
+            bucket = rawPath.substringBefore('/'),
+            path = rawPath.substringAfter('/'),
+            name = "smart flank XML"
         )
     }
 
@@ -114,9 +111,6 @@ object GcStorage {
         )
     }
 
-    fun uploadXCTestZip(args: IosArgs, runGcsPath: String): String =
-        upload(args.xctestrunZip, args.resultsBucket, runGcsPath)
-
     fun uploadXCTestFile(fileName: String, gcsBucket: String, runGcsPath: String, fileBytes: ByteArray): String =
         upload(
             filePath = fileName,
@@ -126,14 +120,11 @@ object GcStorage {
         )
 
     // junit xml may not exist. ignore error if it doesn't exist
-    fun downloadJunitXml(args: IArgs): JUnitTestResult? {
-        val oldXmlPath = download(args.smartFlankGcsPath, ignoreError = true)
-        if (oldXmlPath.isNotEmpty()) {
-            return parseAllSuitesXml(Paths.get(oldXmlPath))
-        }
-
-        return null
-    }
+    fun downloadJunitXml(
+        args: IArgs
+    ): JUnitTestResult? = download(args.smartFlankGcsPath, ignoreError = true)
+        .takeIf { it.isNotEmpty() }
+        ?.let { parseAllSuitesXml(Paths.get(it)) }
 
     private val duplicatedGcsPathCounter = ConcurrentHashMap<String, Int>()
 
@@ -142,8 +133,7 @@ object GcStorage {
         filePath: String,
         fileBytes: ByteArray,
         rootGcsBucket: String,
-        runGcsPath: String,
-        storage: Storage = GcStorage.storage
+        runGcsPath: String
     ): String {
         val file = File(filePath)
         return uploadCache.computeIfAbsent("$runGcsPath-${file.absolutePath}") {
@@ -156,16 +146,27 @@ object GcStorage {
             }
 
             // 404 Not Found error when rootGcsBucket does not exist
-            val fileBlob = BlobInfo.newBuilder(rootGcsBucket, validGcsPath).build()
-            runWithProgress(
-                startMessage = "Uploading ${file.name}",
-                action = { storage.create(fileBlob, fileBytes) },
-                onError = { throw FlankGeneralError("Error on uploading ${file.name}\nCause: $it") }
+            fileBytes.uploadWithProgress(
+                bucket = rootGcsBucket,
+                path = validGcsPath,
+                name = file.name
             )
             GCS_PREFIX + join(rootGcsBucket, validGcsPath)
         }
     }
 
+    private fun ByteArray.uploadWithProgress(
+        bucket: String,
+        path: String,
+        name: String
+    ) {
+        runWithProgress(
+            startMessage = "Uploading [$name] to ${GCS_STORAGE_LINK + join(bucket, path).replace(name, "")}..",
+            action = { storage.create(BlobInfo.newBuilder(bucket, path).build(), this) },
+            onError = { throw FlankGeneralError("Error on uploading $name\nCause: $it") }
+        )
+    }
+
     fun download(gcsUriString: String, ignoreError: Boolean = false): String {
         val gcsURI = URI.create(gcsUriString)
         val bucket = gcsURI.authority

diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt
@@ -2,6 +2,7 @@ package ftl.reports
 
 import flank.common.log
 import flank.common.println
+import flank.common.startWithNewLine
 import ftl.args.IArgs
 import ftl.config.FtlConstants.indent
 import ftl.gc.GcStorage
@@ -14,7 +15,7 @@ import ftl.reports.xml.model.JUnitTestResult
 import java.io.StringWriter
 import java.text.DecimalFormat
 import java.text.DecimalFormatSymbols
-import java.util.Locale
+import java.util.*
 
 /**
 
@@ -43,7 +44,7 @@ object MatrixResultsReport : IReport {
         if (failed > 0) outputData += "$failed matrices failed"
 
         StringWriter().use { writer ->
-            writer.println(reportName())
+            writer.println(reportName().startWithNewLine())
             writer.println("$indent$success / $total ($successPercent%)")
             if (failed > 0) {
                 writer.println("$indent$failed matrices failed")

diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt
@@ -5,6 +5,7 @@ import flank.common.logLn
 import ftl.args.AndroidArgs
 import ftl.args.IosArgs
 import ftl.args.isInstrumentationTest
+import ftl.config.FtlConstants
 import ftl.ios.xctest.common.XcTestRunVersion.V1
 import ftl.ios.xctest.common.XcTestRunVersion.V2
 import ftl.run.common.prettyPrint
@@ -65,7 +66,7 @@ fun saveShardChunks(
         Paths.get(shardFilePath),
         getGson(obfuscatedOutput).toJson(shards).toByteArray()
     )
-    logLn("Saved $size shards to $shardFilePath", OutputLogLevel.DETAILED)
+    logLn("${FtlConstants.indent}Saved $size shards to $shardFilePath", OutputLogLevel.DETAILED)
 }
 
 private fun getGson(obfuscatedOutput: Boolean) =

diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt
@@ -76,5 +76,4 @@ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = corouti
         logLn(FtlConstants.indent + "Updating matrix file")
         args.updateMatrixFile(matrixMap)
     }
-    logLn()
 }
diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt
@@ -4,6 +4,7 @@ import com.google.cloud.storage.Storage
 import flank.common.OutputLogLevel
 import flank.common.log
 import flank.common.logLn
+import flank.common.startWithNewLine
 import ftl.args.IArgs
 import ftl.config.FtlConstants
 import ftl.gc.GcStorage
@@ -20,7 +21,7 @@ import java.nio.file.Paths
 
 // TODO needs refactor
 internal suspend fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) = coroutineScope {
-    logLn("FetchArtifacts", OutputLogLevel.DETAILED)
+    logLn("FetchArtifacts".startWithNewLine(), OutputLogLevel.DETAILED)
     val fields = Storage.BlobListOption.fields(Storage.BlobField.NAME)
 
     var dirty = false
@@ -42,8 +43,6 @@ internal suspend fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) = corouti
             val matched = artifactsList.any { blobPath.matches(it) }
             if (matched) {
                 val downloadFile = getDownloadPath(args, blobPath)
-
-                log(".", OutputLogLevel.DETAILED)
                 if (!downloadFile.toFile().exists()) {
                     val parentFile = downloadFile.parent.toFile()
                     parentFile.mkdirs()
@@ -56,12 +55,10 @@ internal suspend fun fetchArtifacts(matrixMap: MatrixMap, args: IArgs) = corouti
         filtered[index] = matrix.copy(downloaded = true)
         jobs
     }.joinAll()
-    logLn(level = OutputLogLevel.DETAILED)
 
     if (dirty) {
         logLn(FtlConstants.indent + "Updating matrix file", level = OutputLogLevel.DETAILED)
         args.updateMatrixFile(matrixMap)
-        logLn(level = OutputLogLevel.DETAILED)
     }
 }
 

diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt
@@ -2,8 +2,10 @@ package ftl.run.platform.common
 
 import com.google.testing.model.TestMatrix
 import flank.common.logLn
+import flank.common.startWithNewLine
 import ftl.args.IArgs
 import ftl.config.FtlConstants
+import ftl.config.FtlConstants.GCS_STORAGE_LINK
 import ftl.gc.GcTestMatrix
 import ftl.json.MatrixMap
 import ftl.json.createSavedMatrix
@@ -29,10 +31,8 @@ internal suspend fun IArgs.afterRunTests(
     saveConfigFile(matrixMap)
 
     logLn(FtlConstants.indent + "${matrixMap.map.size} matrix ids created in ${stopwatch.check()}")
-    val gcsBucket = "https://console.developers.google.com/storage/browser/" +
-        resultsBucket + "/" + matrixMap.runPath
-    logLn(FtlConstants.indent + gcsBucket)
-    logLn()
+    val gcsBucket = GCS_STORAGE_LINK + resultsBucket + "/" + matrixMap.runPath
+    logLn("${FtlConstants.indent}Raw results will be stored in your GCS bucket at [$gcsBucket]")
     matrixMap.printMatricesWebLinks(project)
 }
 
@@ -49,7 +49,7 @@ private fun IArgs.saveConfigFile(matrixMap: MatrixMap) {
 }
 
 internal suspend inline fun MatrixMap.printMatricesWebLinks(project: String) = coroutineScope {
-    logLn("Matrices webLink")
+    logLn("Matrices webLink".startWithNewLine())
     map.values.map {
         launch(Dispatchers.IO) {
             logLn("${FtlConstants.indent}${it.matrixId} ${getOrUpdateWebLink(it.webLink, project, it.matrixId)}")

diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt
@@ -10,7 +10,7 @@ internal fun IArgs.beforeRunMessage(testShardChunks: List<Chunk>): String {
     val shardCount = testShardChunks.size
     val (classesCount, testsCount) = testShardChunks.partitionedTestCases.testAndClassesCount
 
-    val result = StringBuilder()
+    val result = StringBuilder("\n")
     val testString = if (testsCount == 0 && classesCount != 0) "" else "$testsCount test${s(testsCount)}"
     val classString = if (classesCount > 0) "$classesCount parameterized class${es(classesCount)}" else ""
 

diff --git a/test_runner/src/main/kotlin/ftl/util/ProgressBar.kt b/test_runner/src/main/kotlin/ftl/util/ProgressBar.kt
@@ -11,7 +11,7 @@ class ProgressBar {
     private val timer = Timer(true)
 
     fun start(msg: String) {
-        log("  $msg ", OutputLogLevel.DETAILED)
+        log("  $msg", OutputLogLevel.DETAILED)
         timer.scheduleAtFixedRate(task, 0, 10_000)
     }
 
@@ -21,8 +21,7 @@ class ProgressBar {
         task.cancel()
     }
 }
-
-private class ProgressBarTask : TimerTask() {
+private class ProgressBarTask() : TimerTask() {
     override fun run() {
         log(".", OutputLogLevel.DETAILED)
     }
@@ -31,12 +30,14 @@ private class ProgressBarTask : TimerTask() {
 fun runWithProgress(
     startMessage: String,
     action: () -> Unit,
-    onError: (Exception) -> Unit
+    onSuccess: () -> Unit = {},
+    onError: (Exception) -> Unit = {}
 ) {
     val progress = ProgressBar()
     try {
         progress.start(startMessage)
         action()
+        onSuccess()
     } catch (e: Exception) {
         onError(e)
     } finally {

diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt
@@ -22,14 +22,14 @@ fun main() {
 //            "--debug",
             "firebase",
             "test",
-            "ios",
+            "android",
             "run",
 //            "--dry",
 //            "--dump-shards",
             "--output-style=single",
 //            "--full-junit-result",
 //            "--legacy-junit-result",
-            "-c=./src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml",
+            "-c=/Users/piotr/Projekty/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml",
             "--project=$projectId"
 //            "--client-details=key1=value1,key2=value2"
         )

diff --git a/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt
@@ -68,8 +68,7 @@ class GcStorageTest {
                     filePath = filePath,
                     fileBytes = ByteArray(0),
                     rootGcsBucket = "bucket",
-                    runGcsPath = "gcsPath",
-                    storage = mockk(relaxed = true)
+                    runGcsPath = "gcsPath"
                 )
             }
         )

diff --git a/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/GenericTestRunnerTest.kt
@@ -1,7 +1,9 @@
 package ftl.run
 
 import flank.common.normalizeLineEnding
+import flank.common.startWithNewLine
 import flank.common.trimStartLine
+import flank.common.withNewLineAtTheEnd
 import ftl.args.IArgs
 import ftl.run.platform.common.beforeRunMessage
 import ftl.shard.Chunk
@@ -31,7 +33,7 @@ class GenericTestRunnerTest {
         val result = createMock(1).beforeRunMessage(
             listOf(Chunk(listOf(TestMethod(name = "", time = 0.0))))
         ).normalizeLineEnding()
-        assert(result, "  1 test / 1 shard\n")
+        assert(result, "  1 test / 1 shard".withNewLineAtTheEnd().startWithNewLine())
     }
 
     @Test
@@ -42,6 +44,7 @@ class GenericTestRunnerTest {
         assert(
             result,
             """
+
   1 test / 1 shard
   Running 2x
     2 total shards
@@ -58,6 +61,7 @@ class GenericTestRunnerTest {
         assert(
             result,
             """
+
   6 tests / 6 shards
   Running 2x
     12 total shards
@@ -74,6 +78,7 @@ class GenericTestRunnerTest {
         assert(
             result,
             """
+
   10 tests / 2 shards
   Running 100x
     200 total shards

diff --git a/test_runner/src/test/kotlin/ftl/util/ProgressBarTest.kt b/test_runner/src/test/kotlin/ftl/util/ProgressBarTest.kt
@@ -29,10 +29,10 @@ class ProgressBarTest {
 
         progress.start("hi")
         Thread.sleep(300)
-        assertThat(systemOutRule.log.normalizeLineEnding()).isEqualTo("  hi .")
+        assertThat(systemOutRule.log.normalizeLineEnding()).isEqualTo("  hi.")
 
         progress.stop()
-        assertThat(systemOutRule.log.normalizeLineEnding()).isEqualTo("  hi .\n")
+        assertThat(systemOutRule.log.normalizeLineEnding()).isEqualTo("  hi.\n")
     }
 
     @Test
@@ -49,11 +49,12 @@ class ProgressBarTest {
         runWithProgress(
             startMessage = "TEST",
             action = { action.run() },
+            onSuccess = {},
             onError = { onError.run() }
         )
 
         // then
-        assertThat(systemOutRule.log).contains("  TEST ")
+        assertThat(systemOutRule.log).contains("  TEST")
         verify { action.run() }
         verify(exactly = 0) { onError.run() }
     }
@@ -75,11 +76,12 @@ class ProgressBarTest {
                 action.run()
                 throw Exception()
             },
+            onSuccess = {},
             onError = { onError.run() }
         )
 
         // then
-        assertThat(systemOutRule.log).contains("  TEST ")
+        assertThat(systemOutRule.log).contains("  TEST")
         verify { action.run() }
         verify { onError.run() }
     }
-Original file line number
+Diff line change
@@ Expand Up @@
             logLn(FtlConstants.indent + "Updating matrix file")
             args.updateMatrixFile(matrixMap)
         }
-        logLn()
     }