#829 update tests

Flank · Jun 25, 2020 · 429c09e · 429c09e
1 parent 3b3adce
commit 429c09e
Show file tree

Hide file tree

Showing 6 changed files with 361 additions and 9 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt
@@ -79,12 +79,15 @@ object MockServer {
                 outcome.summary = failure
                 val failureDetail = FailureDetail()
                 failureDetail.timedOut = true
+                failureDetail.crashed = false
+                failureDetail.otherNativeCrash = false
                 outcome.failureDetail = failureDetail
             }
             "-2" -> {
                 outcome.summary = inconclusive
                 val inconclusiveDetail = InconclusiveDetail()
                 inconclusiveDetail.abortedByUser = true
+                inconclusiveDetail.infrastructureFailure = false
                 outcome.inconclusiveDetail = inconclusiveDetail
             }
             "-3" -> {
@@ -194,6 +197,13 @@ object MockServer {
                     val executionId = call.parameters["executionId"] ?: ""
                     call.respond(fakeStep(executionId))
                 }
+                // GcToolResults.listTestCases(toolResultsStep)
+                // GET /toolresults/v1beta3/projects/flank-open-source/histories/1/executions/1/steps/1/testCases
+                get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}/testCases") {
+                    println("Responding to GET ${call.request.uri}")
+                    val executionId = call.parameters["executionId"] ?: ""
+                    call.respond(fakeStep(executionId))
+                }
 
                 // GcToolResults.getDefaultBucket(project)
                 post("/toolresults/v1beta3/projects/{project}:initializeSettings") {

diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -46,7 +46,7 @@ private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestS
 private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
     return StringBuilder("$failures test casess failed").apply {
         if (errors > 0) append(errorMessage(errors))
-        successCount.takeIf { it > 0 }?.let(successMessage)
+        successCount.takeIf { it > 0 }?.let { append(successMessage(it)) }
         if (skipped > 0) append(skippedMessage(skipped))
         if (flakes > 0) append(flakesMessage(flakes))
     }.toString()
@@ -60,8 +60,8 @@ private fun InconclusiveDetail?.formatOutcomeDetails() = when {
 }
 
 private fun SkippedDetail?.formatOutcomeDetails(): String = when {
-    this == null  -> "Unknown reason"
-    incompatibleAppVersion == true -> "Incompatible device/OS combination"
+    this == null -> "Unknown reason"
+    incompatibleDevice == true -> "Incompatible device/OS combination"
     incompatibleArchitecture == true -> "App does not support the device architecture"
     incompatibleAppVersion == true -> "App does not support the OS version"
     else -> "Unknown reason"

diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -9,7 +9,7 @@ fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
 fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
     TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }),
     TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }),
-    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails })
+    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, mapNotNull { it.outcomeDetails })
 )
 
 private fun getOutcomeColor(outcome: String): SystemOutColor {

diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
@@ -0,0 +1,311 @@
+package ftl.util
+
+import com.google.api.services.toolresults.model.Outcome
+import ftl.reports.api.data.TestSuiteOverviewData
+import io.mockk.every
+import io.mockk.mockk
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+internal class OutcomeDetailsFormatterTest {
+
+    @Test
+    fun `should return correct outcome details for success`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.success
+            every { successDetail } returns mockk { every { otherNativeCrash } returns false }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
+        val expectedMessage = "$successCount test cases passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for success with other native crash`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.success
+            every { successDetail } returns mockk { every { otherNativeCrash } returns true }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
+        val expectedMessage = "$successCount test cases passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky" +
+                " (Native crash)"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-other with test overview data`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
+        val expectedMessage = "${testSuiteOverviewData.failures} test casess failed, " +
+                "${testSuiteOverviewData.errors} errors, " +
+                "1 passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-crashed`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns true
+                every { timedOut } returns false
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "Application crashed"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-timedOut`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns true
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "Test timed out"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-notInstalled`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns true
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "App failed to install"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should contains message about native crash when it happens`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns true
+                every { otherNativeCrash } returns true
+            }
+        }
+        val expectedMessage = "App failed to install (Native crash)"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-infrastructureFailure`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk { every { infrastructureFailure } returns true }
+        }
+        val expectedMessage = "Infrastructure failure"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-abortedByUser`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk() {
+                every { infrastructureFailure } returns false
+                every { abortedByUser } returns true
+            }
+        }
+        val expectedMessage = "Test run aborted by user"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-other reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk {
+                every { infrastructureFailure } returns false
+                every { abortedByUser } returns false
+            }
+        }
+        val expectedMessage = "Unknown reason"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleDevice`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns true
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "Incompatible device/OS combination"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleArchitecture`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns true
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "App does not support the device architecture"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleAppVersion`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns true
+            }
+        }
+        val expectedMessage = "App does not support the OS version"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-other reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "Unknown reason"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for unset reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.unset
+        }
+        val expectedMessage = "unset"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+}