#829 Added printing outcome details

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -1,20 +1,29 @@
 package ftl.json
 
-import com.google.api.client.json.GenericJson
 import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.toolresults.model.Outcome
 import ftl.android.AndroidCatalog
 import ftl.gc.GcToolResults
+import ftl.reports.api.createTestExecutionDataListAsync
+import ftl.reports.api.createTestSuitOverviewData
+import ftl.reports.api.data.TestSuiteOverviewData
 import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
-import ftl.util.StepOutcome.unset
+import ftl.util.getDetails
 import ftl.util.webLink
 
+private data class FinishedTestMatrixData(
+    val stepOutcome: Outcome,
+    val isVirtualDevice: Boolean,
+    val testSuiteOverviewData: TestSuiteOverviewData?,
+    val billableMinutes: Long?
+)
+
 // execution gcs paths aren't API accessible.
 class SavedMatrix(matrix: TestMatrix) {
     val matrixId: String = matrix.testMatrixId
@@ -81,23 +90,25 @@ class SavedMatrix(matrix: TestMatrix) {
         outcome = success
         if (matrix.testExecutions == null) return
 
-        matrix.testExecutions.forEach {
-            val executionResult = GcToolResults.getExecutionResult(it)
-            updateOutcome(executionResult.outcome)
-
-            // testExecutionStep, testTiming, etc. can all be null.
-            // sometimes testExecutionStep is present and testTiming is null
-            val stepResult = GcToolResults.getStepResult(it.toolResultsStep)
-            val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
-
-            val billableMinutes = Billing.billableMinutes(testTimeSeconds)
+        updateFinishedMatrixData(matrix)
+    }
 
-            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) {
-                billableVirtualMinutes += billableMinutes
-            } else {
-                billablePhysicalMinutes += billableMinutes
+    private fun updateFinishedMatrixData(matrix: TestMatrix) {
+        matrix.testExecutions.createTestExecutionDataListAsync()
+            .map {
+                FinishedTestMatrixData(
+                    stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
+                    isVirtualDevice = AndroidCatalog.isVirtualDevice(it.testExecution.environment.androidDevice, matrix.projectId.orEmpty()),
+                    testSuiteOverviewData = it.createTestSuitOverviewData(),
+                    billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
+                        ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
+                )
+            }
+            .forEach { (stepOutcome, isVirtualDevice, testSuiteOverviewData, billableMinutes) ->
+                updateOutcome(stepOutcome)
+                updateOutcomeDetails(stepOutcome, testSuiteOverviewData)
+                billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
             }
-        }
     }
 
     private fun updateOutcome(stepOutcome: Outcome) {
@@ -110,19 +121,18 @@ class SavedMatrix(matrix: TestMatrix) {
 
         // Treat flaky outcome as a success
         if (outcome == flaky) outcome = success
+    }
 
-        outcomeDetails = when (outcome) {
-            failure -> stepOutcome.failureDetail.keysToString()
-            success -> stepOutcome.successDetail.keysToString()
-            inconclusive -> stepOutcome.inconclusiveDetail.keysToString()
-            skipped -> stepOutcome.skippedDetail.keysToString()
-            unset -> "unset"
-            else -> "unknown"
-        }
+    private fun updateOutcomeDetails(stepOutcome: Outcome, testSuiteOverviewData: TestSuiteOverviewData?) {
+        outcomeDetails = stepOutcome.getDetails(testSuiteOverviewData) ?: "---"
     }
 
-    private fun GenericJson?.keysToString(): String {
-        return this?.keys?.joinToString(",") ?: ""
+    private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) {
+        if (isVirtualDevice) {
+            billableVirtualMinutes += billableMinutes
+        } else {
+            billablePhysicalMinutes += billableMinutes
+        }
     }
 
     val gcsPathWithoutRootBucket get() = gcsPath.substringAfter('/')

test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt

@@ -5,7 +5,7 @@ import com.google.common.annotations.VisibleForTesting
 data class TableColumn(
     val header: String,
     val data: List<String>,
-    val columnSize: Int = header.length + DEFAULT_COLUMN_PADDING,
+    val columnSize: Int = (data + header).maxBy { it.length }!!.length + DEFAULT_COLUMN_PADDING,
     val dataColor: List<SystemOutColor> = listOf()
 )
 

test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt

@@ -0,0 +1,74 @@
+package ftl.util
+
+import com.google.api.services.toolresults.model.FailureDetail
+import com.google.api.services.toolresults.model.InconclusiveDetail
+import com.google.api.services.toolresults.model.Outcome
+import com.google.api.services.toolresults.model.SkippedDetail
+import com.google.api.services.toolresults.model.SuccessDetail
+import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
+import ftl.util.StepOutcome.inconclusive
+import ftl.util.StepOutcome.skipped
+import ftl.util.StepOutcome.success
+import ftl.util.StepOutcome.unset
+import java.lang.StringBuilder
+
+fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (summary) {
+    success, flaky -> testSuiteOverviewData?.let { getSuccessOutcomeDetails(it, successDetail) }
+    failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData)
+    inconclusive -> inconclusiveDetail.formatOutcomeDetails()
+    skipped -> skippedDetail.formatOutcomeDetails()
+    unset -> "unset"
+    else -> "unknown"
+}
+
+private fun getSuccessOutcomeDetails(
+    testSuiteOverviewData: TestSuiteOverviewData,
+    successDetail: SuccessDetail?
+) = StringBuilder("${testSuiteOverviewData.successCount} test cases passed").apply {
+    if (testSuiteOverviewData.skipped > 0) append(skippedMessage(testSuiteOverviewData.skipped))
+    if (testSuiteOverviewData.flakes > 0) append(flakesMessage(testSuiteOverviewData.flakes))
+    if (successDetail?.otherNativeCrash == true) append(NATIVE_CRASH_MESSAGE)
+}.toString()
+
+private val TestSuiteOverviewData.successCount
+    get() = total - errors - failures - skipped - flakes
+
+private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when {
+    this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+    crashed -> "Application crashed"
+    timedOut -> "Test timed out"
+    notInstalled -> "App failed to install"
+    else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+} + this?.takeIf { it.otherNativeCrash }?.let { NATIVE_CRASH_MESSAGE }.orEmpty()
+
+private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
+    return StringBuilder("$failures test casess failed").apply {
+        if (errors > 0) append(errorMessage(errors))
+        successCount.takeIf { it > 0 }?.let(successMessage)
+        if (skipped > 0) append(skippedMessage(skipped))
+        if (flakes > 0) append(flakesMessage(flakes))
+    }.toString()
+}
+
+private fun InconclusiveDetail?.formatOutcomeDetails() = when {
+    this == null -> "Unknown reason"
+    infrastructureFailure -> "Infrastructure failure"
+    abortedByUser -> "Test run aborted by user"
+    else -> "Unknown reason"
+}
+
+private fun SkippedDetail?.formatOutcomeDetails(): String = when {
+    this == null  -> "Unknown reason"
+    incompatibleAppVersion == true -> "Incompatible device/OS combination"
+    incompatibleArchitecture == true -> "App does not support the device architecture"
+    incompatibleAppVersion == true -> "App does not support the OS version"
+    else -> "Unknown reason"
+}
+
+private const val NATIVE_CRASH_MESSAGE = " (Native crash)"
+private val flakesMessage: (Int) -> String = { ", $it flaky" }
+private val skippedMessage: (Int) -> String = { ", $it skipped" }
+private val successMessage: (Int) -> String = { ", $it passed" }
+private val errorMessage: (Int) -> String = { ", $it errors" }

test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt

@@ -7,9 +7,9 @@ import ftl.util.StepOutcome.success
 fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
 
 fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
-    TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, OUTCOME_COLUMN_SIZE, map { getOutcomeColor(it.outcome) }),
-    TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }, MATRIX_ID_COLUMN_SIZE),
-    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails }, OUTCOME_DETAILS_COLUMN_SIZE)
+    TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }),
+    TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }),
+    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails })
 )
 
 private fun getOutcomeColor(outcome: String): SystemOutColor {
@@ -22,8 +22,5 @@ private fun getOutcomeColor(outcome: String): SystemOutColor {
 }
 
 private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
-private const val OUTCOME_COLUMN_SIZE = 9
-private const val MATRIX_ID_COLUMN_HEADER = "TEST_AXIS_VALUE"
-private const val MATRIX_ID_COLUMN_SIZE = 24
-private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST_DETAILS"
-private const val OUTCOME_DETAILS_COLUMN_SIZE = 20
+private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
+private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"