diff --git a/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt index d7a2a4ff9c..992e71fe8e 100644 --- a/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt +++ b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt @@ -14,11 +14,10 @@ import ftl.util.StepOutcome.success import ftl.util.StepOutcome.unset internal fun Outcome?.getDetails( - testSuiteOverviewData: TestSuiteOverviewData? + testSuiteOverviewData: TestSuiteOverviewData?, + isRoboTest: Boolean = false ): String = when (this?.summary) { - success, flaky -> testSuiteOverviewData - ?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false) - ?: "Unknown outcome" + success, flaky -> if (isRoboTest) "---" else getSuccessOutcomeDetails(testSuiteOverviewData) failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData) inconclusive -> inconclusiveDetail.formatOutcomeDetails() skipped -> skippedDetail.formatOutcomeDetails() @@ -26,6 +25,9 @@ internal fun Outcome?.getDetails( else -> "Unknown outcome" } +private fun Outcome?.getSuccessOutcomeDetails(data: TestSuiteOverviewData?) = + data?.getSuccessOutcomeDetails(this?.successDetail?.otherNativeCrash ?: false) ?: "Unknown outcome" + private fun TestSuiteOverviewData.getSuccessOutcomeDetails( otherNativeCrash: Boolean ) = StringBuilder("$successCount test cases passed").apply { @@ -43,6 +45,7 @@ internal fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: Test crashed == true -> "Application crashed" timedOut == true -> "Test timed out" notInstalled == true -> "App failed to install" + failedRoboscript == true -> "Test failed to run" else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() ?: "Unknown failure" } + this?.takeIf { it.otherNativeCrash ?: false }?.let { NATIVE_CRASH_MESSAGE }.orEmpty() diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt index b05f4c17df..d24bf2baf2 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt @@ -2,15 +2,16 @@ package ftl.reports.outcome import com.google.api.services.toolresults.model.Environment -fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair> = - steps.calculateAndroidBillableMinutes(projectId, testTimeout) to - if (environments.hasOutcome()) - environments.createMatrixOutcomeSummaryUsingEnvironments() - else { - if (steps.isEmpty()) println("No test results found, something went wrong. Try re-running the tests.") - steps.createMatrixOutcomeSummaryUsingSteps() - } +fun TestOutcomeContext.createMatrixOutcomeSummary() = billableMinutes() to outcomeSummary() -private fun List.hasOutcome() = isNotEmpty() && any { env -> - env.environmentResult?.outcome?.summary == null -}.not() +private fun TestOutcomeContext.billableMinutes() = steps.calculateAndroidBillableMinutes(projectId, testTimeout) + +private fun TestOutcomeContext.outcomeSummary() = + if (environments.hasOutcome()) + createMatrixOutcomeSummaryUsingEnvironments() + else { + if (steps.isEmpty()) println("No test results found, something went wrong. Try re-running the tests.") + createMatrixOutcomeSummaryUsingSteps() + } + +private fun List.hasOutcome() = isNotEmpty() && all { it.environmentResult?.outcome?.summary != null } diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt index a8a708ea21..fbc8fab1ed 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt @@ -23,7 +23,7 @@ internal fun List.createTestSuiteOverviewData(): TestSuiteOverviewData = t .groupBy(Step::axisValue) .values .map { it.mapToTestSuiteOverviews().foldTestSuiteOverviewData() } - .fold(TestSuiteOverviewData()) { acc, data -> acc + data } // Fixme https://github.com/Flank/flank/issues/983 + .fold(TestSuiteOverviewData()) { acc, data -> acc + data } private fun Step.isPrimaryStep() = multiStep?.primaryStep?.rollUp != null || multiStep == null diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt index 85a2247dea..98850b4ac1 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt @@ -12,29 +12,33 @@ data class TestOutcome( val details: String = "", ) -fun List.createMatrixOutcomeSummaryUsingEnvironments(): List = - map(Environment::getTestOutcome) - -private fun Environment.getTestOutcome( - outcome: Outcome? = environmentResult?.outcome -) = TestOutcome( - device = axisValue(), - outcome = outcome?.summary ?: UNKNOWN_OUTCOME, - details = outcome.getDetails(createTestSuiteOverviewData()), -) - -fun List.createMatrixOutcomeSummaryUsingSteps() = groupBy(Step::axisValue).map { (device, steps) -> - steps.getTestOutcome(device) -} - -private fun List.getTestOutcome( - deviceModel: String, - outcome: Outcome? = getOutcomeFromSteps(), -) = TestOutcome( - device = deviceModel, - outcome = outcome?.summary ?: UNKNOWN_OUTCOME, - details = outcome.getDetails(createTestSuiteOverviewData()) -) +fun TestOutcomeContext.createMatrixOutcomeSummaryUsingEnvironments(): List = environments + .map { environment -> + TestOutcome( + device = environment.axisValue(), + outcome = environment.outcomeSummary, + details = environment.getOutcomeDetails(isRoboTest) + ) + } + +private val Environment.outcomeSummary + get() = environmentResult?.outcome?.summary ?: UNKNOWN_OUTCOME + +private fun Environment.getOutcomeDetails(isRoboTest: Boolean) = environmentResult?.outcome.getDetails(createTestSuiteOverviewData(), isRoboTest) + +fun TestOutcomeContext.createMatrixOutcomeSummaryUsingSteps() = steps + .groupBy(Step::axisValue) + .map { (device, steps) -> + TestOutcome( + device = device, + outcome = steps.getOutcomeSummary(), + details = steps.getOutcomeDetails(isRoboTest) + ) + } + +private fun List.getOutcomeSummary() = getOutcomeFromSteps()?.summary ?: UNKNOWN_OUTCOME + +private fun List.getOutcomeDetails(isRoboTest: Boolean) = getOutcomeFromSteps().getDetails(createTestSuiteOverviewData(), isRoboTest) private fun List.getOutcomeFromSteps(): Outcome? = maxByOrNull { StepOutcome.order.indexOf(it.outcome?.summary) diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt index 3d63a4477a..d81d6031d5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt +++ b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt @@ -15,7 +15,8 @@ data class TestOutcomeContext( val projectId: String, val environments: List, val steps: List, - val testTimeout: Long + val testTimeout: Long, + val isRoboTest: Boolean ) fun TestMatrix.fetchTestOutcomeContext() = getToolResultsIds().let { ids -> @@ -24,7 +25,8 @@ fun TestMatrix.fetchTestOutcomeContext() = getToolResultsIds().let { ids -> matrixId = testMatrixId, environments = GcToolResults.listAllEnvironments(ids), steps = GcToolResults.listAllSteps(ids), - testTimeout = testTimeout() + testTimeout = testTimeout(), + isRoboTest = isRoboTest() ) } @@ -44,3 +46,5 @@ private fun TestMatrix.testTimeout() = timeoutToSeconds( ?.testTimeout ?: "0s" ) + +private fun TestMatrix.isRoboTest() = testExecutions.orEmpty().any { it?.testSpecification?.androidRoboTest != null } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt index 977ce2db19..865bf4740c 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/BeforeRunMessage.kt @@ -11,7 +11,7 @@ internal fun beforeRunMessage(args: IArgs, testShardChunks: List): String val (classesCount, testsCount) = testShardChunks.partitionedTestCases.testAndClassesCount val result = StringBuilder() - val testString = if (testsCount > 0) "$testsCount test${s(testsCount)}" else "" + val testString = if (testsCount == 0 && classesCount != 0) "" else "$testsCount test${s(testsCount)}" val classString = if (classesCount > 0) "$classesCount parameterized class${es(classesCount)}" else "" result.appendLine( diff --git a/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt index c14c86c063..b3774e5d52 100644 --- a/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt @@ -6,11 +6,16 @@ import ftl.reports.api.data.TestSuiteOverviewData import ftl.util.StepOutcome import io.mockk.every import io.mockk.mockk +import io.mockk.unmockkAll +import org.junit.After import org.junit.Assert.assertEquals import org.junit.Test internal class OutcomeDetailsFormatterTest { + @After + fun tearDown() = unmockkAll() + @Test fun `should return correct outcome details for success`() { // given @@ -21,8 +26,8 @@ internal class OutcomeDetailsFormatterTest { val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } val expectedMessage = "$successCount test cases passed, " + - "${testSuiteOverviewData.skipped} skipped, " + - "${testSuiteOverviewData.flakes} flaky" + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" // when val result = mockedOutcome.getDetails(testSuiteOverviewData) @@ -41,9 +46,9 @@ internal class OutcomeDetailsFormatterTest { val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } val expectedMessage = "$successCount test cases passed, " + - "${testSuiteOverviewData.skipped} skipped, " + - "${testSuiteOverviewData.flakes} flaky" + - " (Native crash)" + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" + + " (Native crash)" // when val result = mockedOutcome.getDetails(testSuiteOverviewData) @@ -57,19 +62,14 @@ internal class OutcomeDetailsFormatterTest { // given val mockedOutcome = mockk { every { summary } returns StepOutcome.failure - every { failureDetail } returns mockk { - every { crashed } returns false - every { timedOut } returns false - every { notInstalled } returns false - every { otherNativeCrash } returns false - } + every { failureDetail } returns mockk(relaxed = true) {} } val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0) val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " + - "${testSuiteOverviewData.errors} errors, " + - "1 passed, " + - "${testSuiteOverviewData.skipped} skipped, " + - "${testSuiteOverviewData.flakes} flaky" + "${testSuiteOverviewData.errors} errors, " + + "1 passed, " + + "${testSuiteOverviewData.skipped} skipped, " + + "${testSuiteOverviewData.flakes} flaky" // when val result = mockedOutcome.getDetails(testSuiteOverviewData) @@ -146,12 +146,7 @@ internal class OutcomeDetailsFormatterTest { // given val mockedOutcome = mockk { every { summary } returns StepOutcome.failure - every { failureDetail } returns mockk { - every { crashed } returns false - every { timedOut } returns false - every { notInstalled } returns false - every { otherNativeCrash } returns false - } + every { failureDetail } returns mockk(relaxed = true) {} } val expectedMessage = "Unknown failure" @@ -354,4 +349,16 @@ internal class OutcomeDetailsFormatterTest { otherNativeCrash = null }.getFailureOutcomeDetails(null) } + + @Test + fun `should print message for failed robo test`() { + val mockedOutcome = mockk { + every { summary } returns StepOutcome.failure + every { failureDetail } returns FailureDetail().apply { failedRoboscript = true } + } + + val result = mockedOutcome.getDetails(null, true) + + assertEquals("Test failed to run", result) + } } diff --git a/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt new file mode 100644 index 0000000000..0b2851e8c7 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt @@ -0,0 +1,126 @@ +package ftl.reports.outcome + +import com.google.api.services.toolresults.model.Environment +import com.google.api.services.toolresults.model.Step +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import kotlin.reflect.full.createInstance +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test + +class CreateMatrixOutcomeSummaryKtTest { + + @Before + fun setUp() { + mockkStatic("ftl.reports.outcome.UtilKt") + mockkStatic("ftl.reports.outcome.BillableMinutesKt") + } + + @After + fun tearDown() = unmockkAll() + + @Test + fun `should create TestOutcome list for success robo test - from environments`() { + val env: Environment = make { + environmentResult = make { + outcome = make { summary = "success" } + } + } + every { env.axisValue() } returns "anyDevice" + val context = TestOutcomeContext( + matrixId = "anyMatrix", + projectId = "anyProject", + testTimeout = Long.MAX_VALUE, + steps = emptyList(), + isRoboTest = true, + environments = listOf(env) + ) + + val (_, result) = context.createMatrixOutcomeSummary() + + assertEquals("---", result[0].details) + assertEquals("anyDevice", result[0].device) + assertEquals("success", result[0].outcome) + } + + @Test + fun `should create TestOutcome list for failed robo test - from environments`() { + val env: Environment = make { + environmentResult = make { + outcome = make { + summary = "failure" + failureDetail = make { failedRoboscript = true } + } + } + } + every { env.axisValue() } returns "anyDevice" + val context = TestOutcomeContext( + matrixId = "anyMatrix", + projectId = "anyProject", + testTimeout = Long.MAX_VALUE, + steps = emptyList(), + isRoboTest = true, + environments = listOf(env) + ) + + val (_, result) = context.createMatrixOutcomeSummary() + + assertEquals("Test failed to run", result[0].details) + assertEquals("anyDevice", result[0].device) + assertEquals("failure", result[0].outcome) + } + + @Test + fun `should create TestOutcome list for success robo test - from steps`() { + val steps: List = listOf(make { + outcome = make { summary = "success" } + }) + every { steps[0].axisValue() } returns "anyDevice" + every { steps.calculateAndroidBillableMinutes(any(), any()) } returns make() + val context = TestOutcomeContext( + matrixId = "anyMatrix", + projectId = "anyProject", + testTimeout = Long.MAX_VALUE, + steps = steps, + isRoboTest = true, + environments = emptyList() + ) + + val (_, result) = context.createMatrixOutcomeSummary() + + assertEquals("---", result[0].details) + assertEquals("anyDevice", result[0].device) + assertEquals("success", result[0].outcome) + } + + @Test + fun `should create TestOutcome list for failed robo test - from steps`() { + val steps: List = listOf(make { + outcome = make { + summary = "failure" + failureDetail = make { failedRoboscript = true } + } + }) + every { steps[0].axisValue() } returns "anyDevice" + every { steps.calculateAndroidBillableMinutes(any(), any()) } returns make() + val context = TestOutcomeContext( + matrixId = "anyMatrix", + projectId = "anyProject", + testTimeout = Long.MAX_VALUE, + steps = steps, + isRoboTest = true, + environments = emptyList() + ) + + val (_, result) = context.createMatrixOutcomeSummary() + + assertEquals("Test failed to run", result[0].details) + assertEquals("anyDevice", result[0].device) + assertEquals("failure", result[0].outcome) + } +} + +private inline fun make(block: T.() -> Unit = {}): T = T::class.createInstance().apply(block)