#829 Fixed flaky outcome

Flank · Jun 29, 2020 · 2f2be85 · 2f2be85
1 parent bae8463
commit 2f2be85
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 6 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -11,6 +11,7 @@ import ftl.reports.api.prepareForJUnitResult
 import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
@@ -96,6 +97,7 @@ class SavedMatrix(matrix: TestMatrix) {
             .forEach {
                 updatedFinishedInfo(
                     stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
+                    flakyOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome.summary != it.step.outcome.summary,
                     testSuiteOverviewData = summedTestSuiteOverviewData,
                     isRoboTests = it.testExecution.testSpecification.androidRoboTest != null,
                     isVirtualDevice = isVirtualDevice(
@@ -110,21 +112,29 @@ class SavedMatrix(matrix: TestMatrix) {
 
     private fun updatedFinishedInfo(
         stepOutcome: Outcome?,
+        flakyOutcome: Boolean,
         testSuiteOverviewData: TestSuiteOverviewData?,
         isRoboTests: Boolean,
         isVirtualDevice: Boolean,
         billableMinutes: Long?
     ) {
-        updateOutcome(stepOutcome)
+        updateOutcome(stepOutcome, flakyOutcome)
         updateOutcomeDetails(stepOutcome, testSuiteOverviewData, isRoboTests)
         billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
     }
 
-    private fun updateOutcome(stepOutcome: Outcome?) {
-        // the matrix outcome is failure if any step fails
-        // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
-        // inconclusive is treated as a failure
-        if (outcome != failure && outcome != inconclusive) outcome = stepOutcome?.summary ?: outcome
+    private fun updateOutcome(
+        stepOutcome: Outcome?,
+        flakyOutcome: Boolean
+    ) {
+        outcome = when {
+            // the matrix outcome is failure if any step fails
+            // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
+            // inconclusive is treated as a failure
+            outcome == failure || outcome == inconclusive -> return
+            flakyOutcome -> flaky
+            else -> stepOutcome?.summary ?: outcome
+        }
     }
 
     private fun updateOutcomeDetails(

diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -2,6 +2,7 @@ package ftl.util
 
 import ftl.json.SavedMatrix
 import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.success
 
 fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
@@ -17,6 +18,7 @@ private fun getOutcomeColor(outcome: String): SystemOutColor {
     return when (outcome) {
         failure -> SystemOutColor.RED
         success -> SystemOutColor.GREEN
+        flaky -> SystemOutColor.BLUE
         else -> SystemOutColor.DEFAULT
     }
 }

diff --git a/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt b/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt
@@ -3,6 +3,7 @@ package ftl.util
 enum class SystemOutColor(val ansiCode: String) {
     DEFAULT("\u001B[0m"),
     RED("\u001B[31m"),
+    BLUE("\u001B[34m"),
     GREEN("\u001B[32m");
 
     fun applyTo(value: String) = ansiCode + value + DEFAULT.ansiCode