fix: Downloading performance Metrics (#1326)

* fix: Downloading performance Metrics * fix: Downloading performance Metrics * formatting change * update test * Fix simple integration test android * Update TestsConsts.kt * Update TestsConsts.kt Co-authored-by: Adam <[email protected]>
Flank · Nov 18, 2020 · 415330a · 415330a
1 parent 733e444
commit 415330a
Show file tree

Hide file tree

Showing 8 changed files with 48 additions and 29 deletions.
diff --git a/integration_tests/src/test/kotlin/utils/TestsConsts.kt b/integration_tests/src/test/kotlin/utils/TestsConsts.kt
@@ -1,16 +1,17 @@
 package utils
 
-const val defaultAndroidOutputPattern = "AndroidArgs.*?" +
-    "gcloud:.*?" +
-    "flank:.*?" +
-    "RunTests.*?" +
-    "Matrices webLink.*?" +
-    "matrix-.*?" +
-    "FetchArtifacts.*?" +
-    "Updating matrix file.*?" +
-    "CostReport.*?MatrixResultsReport.*?" +
-    "1 test cases passed, 1 skipped.*?" +
-    "Uploading JUnitReport.xml ."
+const val defaultAndroidOutputPattern = "AndroidArgs\\s*" +
+    "gcloud:[\\s\\S]*" +
+    "flank:[\\s\\S]*" +
+    "RunTests[\\s\\S]*" +
+    "Matrices webLink[\\s\\S]*" +
+    "matrix-[\\s\\S]*" +
+    "CostReport[\\s\\S]*" +
+    "MatrixResultsReport[\\s\\S]*" +
+    "1 test cases passed, 1 skipped[\\s\\S]*" +
+    "Uploading JUnitReport.xml[\\s\\S]*" +
+    "FetchArtifacts[\\s\\S]*" +
+    "Updating matrix file"
 
 const val defaultIosOutputPattern = "IosArgs.*?" +
     "gcloud:.*?" +

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt
@@ -85,11 +85,10 @@ object GcStorage {
         }
     }
 
-    fun uploadPerformanceMetrics(perfMetricsSummary: PerfMetricsSummary, resultsBucket: String, resultDir: String) {
-        val performanceMetricsFileName = "performanceMetrics.json"
+    fun uploadPerformanceMetrics(perfMetricsSummary: PerfMetricsSummary, resultsBucket: String, resultDir: String) =
         runCatching {
             upload(
-                performanceMetricsFileName,
+                "performanceMetrics.json",
                 perfMetricsSummary.toPrettyString().toByteArray(),
                 resultsBucket,
                 resultDir
@@ -98,8 +97,7 @@ object GcStorage {
             println("Cannot upload performance metrics ${it.message}")
         }.onSuccess {
             println("Performance metrics uploaded to https://console.developers.google.com/storage/browser/$resultsBucket/$resultDir")
-        }
-    }
+        }.getOrNull()
 
     fun uploadReportResult(testResult: String, args: IArgs, fileName: String) {
         if (args.resultsBucket.isBlank() || args.resultsDir.isBlank() || args.disableResultsUpload) return

diff --git a/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt b/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt
@@ -3,25 +3,39 @@ package ftl.reports.api
 import com.google.api.services.testing.model.TestExecution
 import com.google.api.services.toolresults.model.PerfMetricsSummary
 import ftl.android.AndroidCatalog
+import ftl.args.IArgs
 import ftl.gc.GcStorage
 import ftl.gc.GcToolResults
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.runBlocking
+import java.nio.file.Files
+import java.nio.file.Paths
 
 internal fun List<Pair<TestExecution, String>>.getAndUploadPerformanceMetrics(
-    resultBucket: String
+    args: IArgs
 ) = runBlocking {
     filterAndroidPhysicalDevicesRuns()
         .map { (testExecution, gcsStoragePath) ->
             async(Dispatchers.IO) {
-                testExecution.getPerformanceMetric().upload(resultBucket = resultBucket, resultDir = gcsStoragePath)
+                val performanceMetrics = testExecution.getPerformanceMetric()
+                performanceMetrics.save(gcsStoragePath, args)
+                performanceMetrics.upload(resultBucket = args.resultsBucket, resultDir = gcsStoragePath)
             }
         }
         .awaitAll()
 }
 
+private fun PerfMetricsSummary.save(resultsDir: String, args: IArgs) {
+    val configFilePath =
+        if (args.useLocalResultDir()) Paths.get(args.localResultDir, "performanceMetrics.json")
+        else Paths.get(args.localResultDir, resultsDir, "performanceMetrics.json")
+
+    configFilePath.parent.toFile().mkdirs()
+    Files.write(configFilePath, toPrettyString().toByteArray())
+}
+
 private fun List<Pair<TestExecution, String>>.filterAndroidPhysicalDevicesRuns() = filterNot { (testExecution, _) ->
     AndroidCatalog.isVirtualDevice(testExecution.environment.androidDevice, testExecution.projectId)
 }

diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt
@@ -143,9 +143,7 @@ object ReportManager {
     ) {
         testExecutions
             .takeIf { args is AndroidArgs }
-            ?.run {
-                withGcsStoragePath(matrices, args.resultsDir).getAndUploadPerformanceMetrics(args.resultsBucket)
-            }
+            ?.run { withGcsStoragePath(matrices, args.resultsDir).getAndUploadPerformanceMetrics(args) }
     }
 
     private fun List<TestExecution>.withGcsStoragePath(

diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt
@@ -24,9 +24,8 @@ suspend fun newTestRun(args: IArgs) = withTimeoutOrNull(args.parsedTimeout) {
 
     if (!args.async) {
         cancelTestsOnTimeout(args.project, matrixMap.map) { pollMatrices(matrixMap.map.keys, args).updateMatrixMap(matrixMap) }
-        cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) }
-
         ReportManager.generate(matrixMap, args, testShardChunks, ignoredTests)
+        cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) }
 
         println()
         matrixMap.printMatricesWebLinks(args.project)

diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt
@@ -68,9 +68,9 @@ internal fun getDownloadPath(args: IArgs, blobPath: String): Path {
     val objName = if (args.useLocalResultDir()) "" else parsed.getName(0).toString()
     // for iOS it is shardName, remove this comment after FTL introduce server side sharding for iOS
     val matrixName = parsed.getName(1).toString()
-    val deviceName = parsed.getName(2).toString()
-    val filePathName = if (args.keepFilePath) parsed.parent.drop(3).joinToString("/") else ""
     val fileName = parsed.fileName.toString()
+    val deviceName = parsed.getName(2).toString().takeUnless { it == fileName }.orEmpty()
+    val filePathName = if (args.keepFilePath) parsed.parent.drop(3).joinToString("/") else ""
 
     return Paths.get("$localDir/$objName/$matrixName/$deviceName/$filePathName/$fileName")
 }
diff --git a/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt
@@ -37,10 +37,10 @@ class GcStorageTest {
             .setAppStartTime(AppStartTime().setInitialDisplayTime(Duration().setSeconds(5)))
 
         // when
-        GcStorage.uploadPerformanceMetrics(expectedPerformanceMetrics, "bucket", "path/test")
+        val filePath = GcStorage.uploadPerformanceMetrics(expectedPerformanceMetrics, "bucket", "path/test")
 
         // then
-        assertTrue(GcStorage.exist("gs://bucket/path/test/performanceMetrics.json"))
+        assertTrue(GcStorage.exist(filePath.orEmpty()))
     }
 
     @Test

diff --git a/test_runner/src/test/kotlin/ftl/reports/api/PerformanceMetricsTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/PerformanceMetricsTest.kt
@@ -5,6 +5,7 @@ import com.google.api.services.testing.model.TestExecution
 import com.google.api.services.testing.model.ToolResultsStep
 import com.google.common.truth.Truth.assertThat
 import ftl.android.AndroidCatalog
+import ftl.args.IArgs
 import ftl.gc.GcStorage
 import ftl.gc.GcToolResults
 import ftl.test.util.FlankTestRunner
@@ -27,8 +28,11 @@ class PerformanceMetricsTest {
     fun `should not get and upload performance metrics for virtual devices`() {
         mockkObject(AndroidCatalog) {
             every { AndroidCatalog.isVirtualDevice(any<AndroidDevice>(), any()) } returns true
+            val args = mockk<IArgs> {
+                every { resultsBucket } returns "b8ce"
+            }
 
-            assertThat(testExecutions.map { it to "path" }.getAndUploadPerformanceMetrics("b8ce")).isEmpty()
+           assertThat(testExecutions.map { it to "path" }.getAndUploadPerformanceMetrics(args)).isEmpty()
         }
     }
 
@@ -44,7 +48,12 @@ class PerformanceMetricsTest {
             }
 
             mockkObject(GcStorage) {
-                testExecutions.map { it to expectedPath }.getAndUploadPerformanceMetrics(expectedBucket)
+                val args = mockk<IArgs> {
+                    every { resultsBucket } returns expectedBucket
+                    every { useLocalResultDir() } returns false
+                    every { localResultDir } returns "local"
+                }
+                testExecutions.map { it to expectedPath }.getAndUploadPerformanceMetrics(args)
                 performanceMetrics.forEach {
                     verify { GcStorage.uploadPerformanceMetrics(it, expectedBucket, expectedPath) }
                 }