diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt index c786e16503..50b006fcc0 100644 --- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt +++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt @@ -79,12 +79,15 @@ object MockServer { outcome.summary = failure val failureDetail = FailureDetail() failureDetail.timedOut = true + failureDetail.crashed = false + failureDetail.otherNativeCrash = false outcome.failureDetail = failureDetail } "-2" -> { outcome.summary = inconclusive val inconclusiveDetail = InconclusiveDetail() inconclusiveDetail.abortedByUser = true + inconclusiveDetail.infrastructureFailure = false outcome.inconclusiveDetail = inconclusiveDetail } "-3" -> { @@ -194,6 +197,13 @@ object MockServer { val executionId = call.parameters["executionId"] ?: "" call.respond(fakeStep(executionId)) } + // GcToolResults.listTestCases(toolResultsStep) + // GET /toolresults/v1beta3/projects/flank-open-source/histories/1/executions/1/steps/1/testCases + get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}/testCases") { + println("Responding to GET ${call.request.uri}") + val executionId = call.parameters["executionId"] ?: "" + call.respond(fakeStep(executionId)) + } // GcToolResults.getDefaultBucket(project) post("/toolresults/v1beta3/projects/{project}:initializeSettings") { diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt index a7bc9ccf34..6f9939cc1a 100644 --- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt +++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt @@ -46,7 +46,7 @@ private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestS private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String { return StringBuilder("$failures test casess failed").apply { if (errors > 0) append(errorMessage(errors)) - successCount.takeIf { it > 0 }?.let(successMessage) + successCount.takeIf { it > 0 }?.let { append(successMessage(it)) } if (skipped > 0) append(skippedMessage(skipped)) if (flakes > 0) append(flakesMessage(flakes)) }.toString() @@ -60,8 +60,8 @@ private fun InconclusiveDetail?.formatOutcomeDetails() = when { } private fun SkippedDetail?.formatOutcomeDetails(): String = when { - this == null -> "Unknown reason" - incompatibleAppVersion == true -> "Incompatible device/OS combination" + this == null -> "Unknown reason" + incompatibleDevice == true -> "Incompatible device/OS combination" incompatibleArchitecture == true -> "App does not support the device architecture" incompatibleAppVersion == true -> "App does not support the OS version" else -> "Unknown reason" diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt index 4d866398a6..4e6820c58c 100644 --- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt +++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt @@ -9,7 +9,7 @@ fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable() fun List.asPrintableTable(): String = buildTable( TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }), TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }), - TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails }) + TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, mapNotNull { it.outcomeDetails }) ) private fun getOutcomeColor(outcome: String): SystemOutColor { diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt new file mode 100644 index 0000000000..540a8b0d97 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt @@ -0,0 +1,311 @@ +package ftl.util + +import com.google.api.services.toolresults.model.Outcome +import ftl.reports.api.data.TestSuiteOverviewData +import io.mockk.every +import io.mockk.mockk +import org.junit.Assert.assertEquals +import org.junit.Test + +internal class OutcomeDetailsFormatterTest { + + @Test + fun `should return correct outcome details for success`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.success + every { successDetail } returns mockk { every { otherNativeCrash } returns false } + } + val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) + val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } + val expectedMessage = "$successCount test cases passed, " + + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" + + // when + val result = mockedOutcome.getDetails(testSuiteOverviewData) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for success with other native crash`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.success + every { successDetail } returns mockk { every { otherNativeCrash } returns true } + } + val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) + val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } + val expectedMessage = "$successCount test cases passed, " + + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" + + " (Native crash)" + + // when + val result = mockedOutcome.getDetails(testSuiteOverviewData) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for failed-other with test overview data`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns mockk { + every { crashed } returns false + every { timedOut } returns false + every { notInstalled } returns false + every { otherNativeCrash } returns false + } + } + val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0) + val expectedMessage = "${testSuiteOverviewData.failures} test casess failed, " + + "${testSuiteOverviewData.errors} errors, " + + "1 passed, " + + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" + + // when + val result = mockedOutcome.getDetails(testSuiteOverviewData) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for failed-crashed`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns mockk { + every { crashed } returns true + every { timedOut } returns false + every { notInstalled } returns false + every { otherNativeCrash } returns false + } + } + val expectedMessage = "Application crashed" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for failed-timedOut`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns mockk { + every { crashed } returns false + every { timedOut } returns true + every { notInstalled } returns false + every { otherNativeCrash } returns false + } + } + val expectedMessage = "Test timed out" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for failed-notInstalled`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns mockk { + every { crashed } returns false + every { timedOut } returns false + every { notInstalled } returns true + every { otherNativeCrash } returns false + } + } + val expectedMessage = "App failed to install" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should contains message about native crash when it happens`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns mockk { + every { crashed } returns false + every { timedOut } returns false + every { notInstalled } returns true + every { otherNativeCrash } returns true + } + } + val expectedMessage = "App failed to install (Native crash)" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for inconclusive-infrastructureFailure`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.inconclusive + every { inconclusiveDetail } returns mockk { every { infrastructureFailure } returns true } + } + val expectedMessage = "Infrastructure failure" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for inconclusive-abortedByUser`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.inconclusive + every { inconclusiveDetail } returns mockk() { + every { infrastructureFailure } returns false + every { abortedByUser } returns true + } + } + val expectedMessage = "Test run aborted by user" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for inconclusive-other reason`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.inconclusive + every { inconclusiveDetail } returns mockk { + every { infrastructureFailure } returns false + every { abortedByUser } returns false + } + } + val expectedMessage = "Unknown reason" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for skipped-incompatibleDevice`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.skipped + every { skippedDetail } returns mockk { + every { incompatibleDevice } returns true + every { incompatibleArchitecture } returns false + every { incompatibleAppVersion } returns false + } + } + val expectedMessage = "Incompatible device/OS combination" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for skipped-incompatibleArchitecture`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.skipped + every { skippedDetail } returns mockk { + every { incompatibleDevice } returns false + every { incompatibleArchitecture } returns true + every { incompatibleAppVersion } returns false + } + } + val expectedMessage = "App does not support the device architecture" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for skipped-incompatibleAppVersion`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.skipped + every { skippedDetail } returns mockk { + every { incompatibleDevice } returns false + every { incompatibleArchitecture } returns false + every { incompatibleAppVersion } returns true + } + } + val expectedMessage = "App does not support the OS version" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for skipped-other reason`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.skipped + every { skippedDetail } returns mockk { + every { incompatibleDevice } returns false + every { incompatibleArchitecture } returns false + every { incompatibleAppVersion } returns false + } + } + val expectedMessage = "Unknown reason" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } + + @Test + fun `should return correct outcome details for unset reason`() { + // given + val mockedOutcome = mockk { + every { summary } returns StepOutcome.unset + } + val expectedMessage = "unset" + + // when + val result = mockedOutcome.getDetails(null) + + // then + assertEquals(expectedMessage, result) + } +} diff --git a/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt b/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt new file mode 100644 index 0000000000..68c1556bca --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt @@ -0,0 +1,31 @@ +package ftl.util + +import com.google.common.truth.Truth.assertThat +import org.junit.Test + +internal class TableColumnTest { + + @Test + fun `Should calculate properly column width`() { + // given + val expectedColumnSize = 8 + + // when + val tableColumn = TableColumn("55555", listOf("1", "22", "333", "4444", "55555", "666666")) + + // then + assertThat(tableColumn.columnSize).isEqualTo(expectedColumnSize) + } + + @Test + fun `Should calculate properly column width when header value is the longest`() { + // given + val expectedColumnSize = 7 + + // when + val tableColumn = TableColumn("55555", listOf("1", "22", "333", "4444")) + + // then + assertThat(tableColumn.columnSize).isEqualTo(expectedColumnSize) + } +} diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index 5e2e359766..31dd6f599f 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -278,11 +278,11 @@ class UtilsTest { private fun assertOutputIsEqualReferenceTable(outputLog: String, matrixId: String) { val referenceTable = - "┌─────────┬────────────────────────┬────────────────────┐\n" + - "│ OUTCOME │ TEST_AXIS_VALUE │ TEST_DETAILS │\n" + - "├─────────┼────────────────────────┼────────────────────┤\n" + - "│ Failed │ $matrixId │ Test failed to run │\n" + - "└─────────┴────────────────────────┴────────────────────┘" + "┌─────────┬───────────┬────────────────────┐\n" + + "│ OUTCOME │ MATRIX ID │ TEST DETAILS │\n" + + "├─────────┼───────────┼────────────────────┤\n" + + "│ Failed │ $matrixId │ Test failed to run │\n" + + "└─────────┴───────────┴────────────────────┘" referenceTable.split("\n") .forEach { println(it)