diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt index c529f5fb35..7f9d528966 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt @@ -1,10 +1,12 @@ package ftl.json -import com.google.api.client.json.GenericJson import com.google.api.services.testing.model.TestMatrix import com.google.api.services.toolresults.model.Outcome import ftl.android.AndroidCatalog import ftl.gc.GcToolResults +import ftl.reports.api.createTestExecutionDataListAsync +import ftl.reports.api.createTestSuitOverviewData +import ftl.reports.api.data.TestSuiteOverviewData import ftl.util.Billing import ftl.util.MatrixState.FINISHED import ftl.util.StepOutcome.failure @@ -12,9 +14,16 @@ import ftl.util.StepOutcome.flaky import ftl.util.StepOutcome.inconclusive import ftl.util.StepOutcome.skipped import ftl.util.StepOutcome.success -import ftl.util.StepOutcome.unset +import ftl.util.getDetails import ftl.util.webLink +private data class FinishedTestMatrixData( + val stepOutcome: Outcome, + val isVirtualDevice: Boolean, + val testSuiteOverviewData: TestSuiteOverviewData?, + val billableMinutes: Long? +) + // execution gcs paths aren't API accessible. class SavedMatrix(matrix: TestMatrix) { val matrixId: String = matrix.testMatrixId @@ -81,23 +90,25 @@ class SavedMatrix(matrix: TestMatrix) { outcome = success if (matrix.testExecutions == null) return - matrix.testExecutions.forEach { - val executionResult = GcToolResults.getExecutionResult(it) - updateOutcome(executionResult.outcome) - - // testExecutionStep, testTiming, etc. can all be null. - // sometimes testExecutionStep is present and testTiming is null - val stepResult = GcToolResults.getStepResult(it.toolResultsStep) - val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return - - val billableMinutes = Billing.billableMinutes(testTimeSeconds) + updateFinishedMatrixData(matrix) + } - if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) { - billableVirtualMinutes += billableMinutes - } else { - billablePhysicalMinutes += billableMinutes + private fun updateFinishedMatrixData(matrix: TestMatrix) { + matrix.testExecutions.createTestExecutionDataListAsync() + .map { + FinishedTestMatrixData( + stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome, + isVirtualDevice = AndroidCatalog.isVirtualDevice(it.testExecution.environment.androidDevice, matrix.projectId.orEmpty()), + testSuiteOverviewData = it.createTestSuitOverviewData(), + billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds + ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) } + ) + } + .forEach { (stepOutcome, isVirtualDevice, testSuiteOverviewData, billableMinutes) -> + updateOutcome(stepOutcome) + updateOutcomeDetails(stepOutcome, testSuiteOverviewData) + billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) } } - } } private fun updateOutcome(stepOutcome: Outcome) { @@ -110,19 +121,18 @@ class SavedMatrix(matrix: TestMatrix) { // Treat flaky outcome as a success if (outcome == flaky) outcome = success + } - outcomeDetails = when (outcome) { - failure -> stepOutcome.failureDetail.keysToString() - success -> stepOutcome.successDetail.keysToString() - inconclusive -> stepOutcome.inconclusiveDetail.keysToString() - skipped -> stepOutcome.skippedDetail.keysToString() - unset -> "unset" - else -> "unknown" - } + private fun updateOutcomeDetails(stepOutcome: Outcome, testSuiteOverviewData: TestSuiteOverviewData?) { + outcomeDetails = stepOutcome.getDetails(testSuiteOverviewData) ?: "---" } - private fun GenericJson?.keysToString(): String { - return this?.keys?.joinToString(",") ?: "" + private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) { + if (isVirtualDevice) { + billableVirtualMinutes += billableMinutes + } else { + billablePhysicalMinutes += billableMinutes + } } val gcsPathWithoutRootBucket get() = gcsPath.substringAfter('/') diff --git a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt index 2f79e7be29..1ef93384b7 100644 --- a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt +++ b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt @@ -5,7 +5,7 @@ import com.google.common.annotations.VisibleForTesting data class TableColumn( val header: String, val data: List, - val columnSize: Int = header.length + DEFAULT_COLUMN_PADDING, + val columnSize: Int = (data + header).maxBy { it.length }!!.length + DEFAULT_COLUMN_PADDING, val dataColor: List = listOf() ) diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt new file mode 100644 index 0000000000..a7bc9ccf34 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt @@ -0,0 +1,74 @@ +package ftl.util + +import com.google.api.services.toolresults.model.FailureDetail +import com.google.api.services.toolresults.model.InconclusiveDetail +import com.google.api.services.toolresults.model.Outcome +import com.google.api.services.toolresults.model.SkippedDetail +import com.google.api.services.toolresults.model.SuccessDetail +import ftl.reports.api.data.TestSuiteOverviewData +import ftl.util.StepOutcome.failure +import ftl.util.StepOutcome.flaky +import ftl.util.StepOutcome.inconclusive +import ftl.util.StepOutcome.skipped +import ftl.util.StepOutcome.success +import ftl.util.StepOutcome.unset +import java.lang.StringBuilder + +fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (summary) { + success, flaky -> testSuiteOverviewData?.let { getSuccessOutcomeDetails(it, successDetail) } + failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData) + inconclusive -> inconclusiveDetail.formatOutcomeDetails() + skipped -> skippedDetail.formatOutcomeDetails() + unset -> "unset" + else -> "unknown" +} + +private fun getSuccessOutcomeDetails( + testSuiteOverviewData: TestSuiteOverviewData, + successDetail: SuccessDetail? +) = StringBuilder("${testSuiteOverviewData.successCount} test cases passed").apply { + if (testSuiteOverviewData.skipped > 0) append(skippedMessage(testSuiteOverviewData.skipped)) + if (testSuiteOverviewData.flakes > 0) append(flakesMessage(testSuiteOverviewData.flakes)) + if (successDetail?.otherNativeCrash == true) append(NATIVE_CRASH_MESSAGE) +}.toString() + +private val TestSuiteOverviewData.successCount + get() = total - errors - failures - skipped - flakes + +private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when { + this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() + crashed -> "Application crashed" + timedOut -> "Test timed out" + notInstalled -> "App failed to install" + else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() +} + this?.takeIf { it.otherNativeCrash }?.let { NATIVE_CRASH_MESSAGE }.orEmpty() + +private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String { + return StringBuilder("$failures test casess failed").apply { + if (errors > 0) append(errorMessage(errors)) + successCount.takeIf { it > 0 }?.let(successMessage) + if (skipped > 0) append(skippedMessage(skipped)) + if (flakes > 0) append(flakesMessage(flakes)) + }.toString() +} + +private fun InconclusiveDetail?.formatOutcomeDetails() = when { + this == null -> "Unknown reason" + infrastructureFailure -> "Infrastructure failure" + abortedByUser -> "Test run aborted by user" + else -> "Unknown reason" +} + +private fun SkippedDetail?.formatOutcomeDetails(): String = when { + this == null -> "Unknown reason" + incompatibleAppVersion == true -> "Incompatible device/OS combination" + incompatibleArchitecture == true -> "App does not support the device architecture" + incompatibleAppVersion == true -> "App does not support the OS version" + else -> "Unknown reason" +} + +private const val NATIVE_CRASH_MESSAGE = " (Native crash)" +private val flakesMessage: (Int) -> String = { ", $it flaky" } +private val skippedMessage: (Int) -> String = { ", $it skipped" } +private val successMessage: (Int) -> String = { ", $it passed" } +private val errorMessage: (Int) -> String = { ", $it errors" } diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt index fce0c69cf6..4d866398a6 100644 --- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt +++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt @@ -7,9 +7,9 @@ import ftl.util.StepOutcome.success fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable() fun List.asPrintableTable(): String = buildTable( - TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, OUTCOME_COLUMN_SIZE, map { getOutcomeColor(it.outcome) }), - TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }, MATRIX_ID_COLUMN_SIZE), - TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails }, OUTCOME_DETAILS_COLUMN_SIZE) + TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }), + TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }), + TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails }) ) private fun getOutcomeColor(outcome: String): SystemOutColor { @@ -22,8 +22,5 @@ private fun getOutcomeColor(outcome: String): SystemOutColor { } private const val OUTCOME_COLUMN_HEADER = "OUTCOME" -private const val OUTCOME_COLUMN_SIZE = 9 -private const val MATRIX_ID_COLUMN_HEADER = "TEST_AXIS_VALUE" -private const val MATRIX_ID_COLUMN_SIZE = 24 -private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST_DETAILS" -private const val OUTCOME_DETAILS_COLUMN_SIZE = 20 +private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID" +private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"