From 195efa75b928a394261543e36a2ee2e2a2021ff0 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Mon, 22 Jun 2020 17:50:09 +0200
Subject: [PATCH 01/21] #829 Added printing outcome details

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 76 +++++++++----------
 .../main/kotlin/ftl/util/LogTableBuilder.kt   |  2 +-
 .../ftl/util/OutcomeDetailsFormatter.kt       | 74 ++++++++++++++++++
 .../kotlin/ftl/util/SavedMatrixTableUtil.kt   | 13 ++--
 4 files changed, 118 insertions(+), 47 deletions(-)
 create mode 100644 test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 195e90e255..7f9d528966 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -1,22 +1,28 @@
 package ftl.json
 
-import com.google.api.client.json.GenericJson
 import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.toolresults.model.Outcome
 import ftl.android.AndroidCatalog
 import ftl.gc.GcToolResults
-import ftl.util.MatrixState.ERROR
+import ftl.reports.api.createTestExecutionDataListAsync
+import ftl.reports.api.createTestSuitOverviewData
+import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
-import ftl.util.StepOutcome.unset
-import ftl.util.billableMinutes
-import ftl.util.timeoutToSeconds
+import ftl.util.getDetails
 import ftl.util.webLink
-import kotlin.math.min
+
+private data class FinishedTestMatrixData(
+    val stepOutcome: Outcome,
+    val isVirtualDevice: Boolean,
+    val testSuiteOverviewData: TestSuiteOverviewData?,
+    val billableMinutes: Long?
+)
 
 // execution gcs paths aren't API accessible.
 class SavedMatrix(matrix: TestMatrix) {
@@ -84,30 +90,25 @@ class SavedMatrix(matrix: TestMatrix) {
         outcome = success
         if (matrix.testExecutions == null) return
 
-        matrix.testExecutions.forEach {
-            val executionResult = GcToolResults.getExecutionResult(it)
-
-            updateOutcome(executionResult.outcome)
-
-            // flank should not calculate billable minutes if an infrastructure error occurred
-            if (it.state == ERROR) return
-
-            // testExecutionStep, testTiming, etc. can all be null.
-            // sometimes testExecutionStep is present and testTiming is null
-            val stepResult = GcToolResults.getStepResult(it.toolResultsStep)
-            val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
-            val testTimeout = timeoutToSeconds(it.testSpecification?.testTimeout ?: "0s")
-
-            // if overall test duration time is higher then testTimeout flank should calculate billable minutes for testTimeout
-            val timeToBill = min(testTimeSeconds, testTimeout)
-            val billableMinutes = billableMinutes(timeToBill)
+        updateFinishedMatrixData(matrix)
+    }
 
-            if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) {
-                billableVirtualMinutes += billableMinutes
-            } else {
-                billablePhysicalMinutes += billableMinutes
+    private fun updateFinishedMatrixData(matrix: TestMatrix) {
+        matrix.testExecutions.createTestExecutionDataListAsync()
+            .map {
+                FinishedTestMatrixData(
+                    stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
+                    isVirtualDevice = AndroidCatalog.isVirtualDevice(it.testExecution.environment.androidDevice, matrix.projectId.orEmpty()),
+                    testSuiteOverviewData = it.createTestSuitOverviewData(),
+                    billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
+                        ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
+                )
+            }
+            .forEach { (stepOutcome, isVirtualDevice, testSuiteOverviewData, billableMinutes) ->
+                updateOutcome(stepOutcome)
+                updateOutcomeDetails(stepOutcome, testSuiteOverviewData)
+                billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
             }
-        }
     }
 
     private fun updateOutcome(stepOutcome: Outcome) {
@@ -120,19 +121,18 @@ class SavedMatrix(matrix: TestMatrix) {
 
         // Treat flaky outcome as a success
         if (outcome == flaky) outcome = success
+    }
 
-        outcomeDetails = when (outcome) {
-            failure -> stepOutcome.failureDetail.keysToString()
-            success -> stepOutcome.successDetail.keysToString()
-            inconclusive -> stepOutcome.inconclusiveDetail.keysToString()
-            skipped -> stepOutcome.skippedDetail.keysToString()
-            unset -> "unset"
-            else -> "unknown"
-        }
+    private fun updateOutcomeDetails(stepOutcome: Outcome, testSuiteOverviewData: TestSuiteOverviewData?) {
+        outcomeDetails = stepOutcome.getDetails(testSuiteOverviewData) ?: "---"
     }
 
-    private fun GenericJson?.keysToString(): String {
-        return this?.keys?.joinToString(",") ?: ""
+    private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) {
+        if (isVirtualDevice) {
+            billableVirtualMinutes += billableMinutes
+        } else {
+            billablePhysicalMinutes += billableMinutes
+        }
     }
 
     val gcsPathWithoutRootBucket get() = gcsPath.substringAfter('/')
diff --git a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
index 2f79e7be29..1ef93384b7 100644
--- a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
+++ b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
@@ -5,7 +5,7 @@ import com.google.common.annotations.VisibleForTesting
 data class TableColumn(
     val header: String,
     val data: List<String>,
-    val columnSize: Int = header.length + DEFAULT_COLUMN_PADDING,
+    val columnSize: Int = (data + header).maxBy { it.length }!!.length + DEFAULT_COLUMN_PADDING,
     val dataColor: List<SystemOutColor> = listOf()
 )
 
diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
new file mode 100644
index 0000000000..a7bc9ccf34
--- /dev/null
+++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -0,0 +1,74 @@
+package ftl.util
+
+import com.google.api.services.toolresults.model.FailureDetail
+import com.google.api.services.toolresults.model.InconclusiveDetail
+import com.google.api.services.toolresults.model.Outcome
+import com.google.api.services.toolresults.model.SkippedDetail
+import com.google.api.services.toolresults.model.SuccessDetail
+import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
+import ftl.util.StepOutcome.inconclusive
+import ftl.util.StepOutcome.skipped
+import ftl.util.StepOutcome.success
+import ftl.util.StepOutcome.unset
+import java.lang.StringBuilder
+
+fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (summary) {
+    success, flaky -> testSuiteOverviewData?.let { getSuccessOutcomeDetails(it, successDetail) }
+    failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData)
+    inconclusive -> inconclusiveDetail.formatOutcomeDetails()
+    skipped -> skippedDetail.formatOutcomeDetails()
+    unset -> "unset"
+    else -> "unknown"
+}
+
+private fun getSuccessOutcomeDetails(
+    testSuiteOverviewData: TestSuiteOverviewData,
+    successDetail: SuccessDetail?
+) = StringBuilder("${testSuiteOverviewData.successCount} test cases passed").apply {
+    if (testSuiteOverviewData.skipped > 0) append(skippedMessage(testSuiteOverviewData.skipped))
+    if (testSuiteOverviewData.flakes > 0) append(flakesMessage(testSuiteOverviewData.flakes))
+    if (successDetail?.otherNativeCrash == true) append(NATIVE_CRASH_MESSAGE)
+}.toString()
+
+private val TestSuiteOverviewData.successCount
+    get() = total - errors - failures - skipped - flakes
+
+private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when {
+    this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+    crashed -> "Application crashed"
+    timedOut -> "Test timed out"
+    notInstalled -> "App failed to install"
+    else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+} + this?.takeIf { it.otherNativeCrash }?.let { NATIVE_CRASH_MESSAGE }.orEmpty()
+
+private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
+    return StringBuilder("$failures test casess failed").apply {
+        if (errors > 0) append(errorMessage(errors))
+        successCount.takeIf { it > 0 }?.let(successMessage)
+        if (skipped > 0) append(skippedMessage(skipped))
+        if (flakes > 0) append(flakesMessage(flakes))
+    }.toString()
+}
+
+private fun InconclusiveDetail?.formatOutcomeDetails() = when {
+    this == null -> "Unknown reason"
+    infrastructureFailure -> "Infrastructure failure"
+    abortedByUser -> "Test run aborted by user"
+    else -> "Unknown reason"
+}
+
+private fun SkippedDetail?.formatOutcomeDetails(): String = when {
+    this == null  -> "Unknown reason"
+    incompatibleAppVersion == true -> "Incompatible device/OS combination"
+    incompatibleArchitecture == true -> "App does not support the device architecture"
+    incompatibleAppVersion == true -> "App does not support the OS version"
+    else -> "Unknown reason"
+}
+
+private const val NATIVE_CRASH_MESSAGE = " (Native crash)"
+private val flakesMessage: (Int) -> String = { ", $it flaky" }
+private val skippedMessage: (Int) -> String = { ", $it skipped" }
+private val successMessage: (Int) -> String = { ", $it passed" }
+private val errorMessage: (Int) -> String = { ", $it errors" }
diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
index fce0c69cf6..4d866398a6 100644
--- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
+++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -7,9 +7,9 @@ import ftl.util.StepOutcome.success
 fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
 
 fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
-    TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, OUTCOME_COLUMN_SIZE, map { getOutcomeColor(it.outcome) }),
-    TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }, MATRIX_ID_COLUMN_SIZE),
-    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails }, OUTCOME_DETAILS_COLUMN_SIZE)
+    TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }),
+    TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }),
+    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails })
 )
 
 private fun getOutcomeColor(outcome: String): SystemOutColor {
@@ -22,8 +22,5 @@ private fun getOutcomeColor(outcome: String): SystemOutColor {
 }
 
 private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
-private const val OUTCOME_COLUMN_SIZE = 9
-private const val MATRIX_ID_COLUMN_HEADER = "TEST_AXIS_VALUE"
-private const val MATRIX_ID_COLUMN_SIZE = 24
-private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST_DETAILS"
-private const val OUTCOME_DETAILS_COLUMN_SIZE = 20
+private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
+private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"

From 957bcfc3efb4a48b1e52414e398880c24dea02f0 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Tue, 23 Jun 2020 16:35:05 +0200
Subject: [PATCH 02/21] #829 update tests

---
 .../src/main/kotlin/ftl/mock/MockServer.kt    |  10 +
 .../ftl/util/OutcomeDetailsFormatter.kt       |   6 +-
 .../kotlin/ftl/util/SavedMatrixTableUtil.kt   |   2 +-
 .../ftl/util/OutcomeDetailsFormatterTest.kt   | 311 ++++++++++++++++++
 .../test/kotlin/ftl/util/TableColumnTest.kt   |  31 ++
 .../src/test/kotlin/ftl/util/UtilsTest.kt     |  10 +-
 6 files changed, 361 insertions(+), 9 deletions(-)
 create mode 100644 test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
 create mode 100644 test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt

diff --git a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt
index c786e16503..50b006fcc0 100644
--- a/test_runner/src/main/kotlin/ftl/mock/MockServer.kt
+++ b/test_runner/src/main/kotlin/ftl/mock/MockServer.kt
@@ -79,12 +79,15 @@ object MockServer {
                 outcome.summary = failure
                 val failureDetail = FailureDetail()
                 failureDetail.timedOut = true
+                failureDetail.crashed = false
+                failureDetail.otherNativeCrash = false
                 outcome.failureDetail = failureDetail
             }
             "-2" -> {
                 outcome.summary = inconclusive
                 val inconclusiveDetail = InconclusiveDetail()
                 inconclusiveDetail.abortedByUser = true
+                inconclusiveDetail.infrastructureFailure = false
                 outcome.inconclusiveDetail = inconclusiveDetail
             }
             "-3" -> {
@@ -194,6 +197,13 @@ object MockServer {
                     val executionId = call.parameters["executionId"] ?: ""
                     call.respond(fakeStep(executionId))
                 }
+                // GcToolResults.listTestCases(toolResultsStep)
+                // GET /toolresults/v1beta3/projects/flank-open-source/histories/1/executions/1/steps/1/testCases
+                get("/toolresults/v1beta3/projects/{project}/histories/{historyId}/executions/{executionId}/steps/{stepId}/testCases") {
+                    println("Responding to GET ${call.request.uri}")
+                    val executionId = call.parameters["executionId"] ?: ""
+                    call.respond(fakeStep(executionId))
+                }
 
                 // GcToolResults.getDefaultBucket(project)
                 post("/toolresults/v1beta3/projects/{project}:initializeSettings") {
diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
index a7bc9ccf34..6f9939cc1a 100644
--- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
+++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -46,7 +46,7 @@ private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestS
 private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
     return StringBuilder("$failures test casess failed").apply {
         if (errors > 0) append(errorMessage(errors))
-        successCount.takeIf { it > 0 }?.let(successMessage)
+        successCount.takeIf { it > 0 }?.let { append(successMessage(it)) }
         if (skipped > 0) append(skippedMessage(skipped))
         if (flakes > 0) append(flakesMessage(flakes))
     }.toString()
@@ -60,8 +60,8 @@ private fun InconclusiveDetail?.formatOutcomeDetails() = when {
 }
 
 private fun SkippedDetail?.formatOutcomeDetails(): String = when {
-    this == null  -> "Unknown reason"
-    incompatibleAppVersion == true -> "Incompatible device/OS combination"
+    this == null -> "Unknown reason"
+    incompatibleDevice == true -> "Incompatible device/OS combination"
     incompatibleArchitecture == true -> "App does not support the device architecture"
     incompatibleAppVersion == true -> "App does not support the OS version"
     else -> "Unknown reason"
diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
index 4d866398a6..4e6820c58c 100644
--- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
+++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -9,7 +9,7 @@ fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
 fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
     TableColumn(OUTCOME_COLUMN_HEADER, map { it.outcome }, dataColor = map { getOutcomeColor(it.outcome) }),
     TableColumn(MATRIX_ID_COLUMN_HEADER, map { it.matrixId }),
-    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, map { it.outcomeDetails })
+    TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, mapNotNull { it.outcomeDetails })
 )
 
 private fun getOutcomeColor(outcome: String): SystemOutColor {
diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
new file mode 100644
index 0000000000..540a8b0d97
--- /dev/null
+++ b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
@@ -0,0 +1,311 @@
+package ftl.util
+
+import com.google.api.services.toolresults.model.Outcome
+import ftl.reports.api.data.TestSuiteOverviewData
+import io.mockk.every
+import io.mockk.mockk
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+internal class OutcomeDetailsFormatterTest {
+
+    @Test
+    fun `should return correct outcome details for success`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.success
+            every { successDetail } returns mockk { every { otherNativeCrash } returns false }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
+        val expectedMessage = "$successCount test cases passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for success with other native crash`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.success
+            every { successDetail } returns mockk { every { otherNativeCrash } returns true }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
+        val expectedMessage = "$successCount test cases passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky" +
+                " (Native crash)"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-other with test overview data`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
+        val expectedMessage = "${testSuiteOverviewData.failures} test casess failed, " +
+                "${testSuiteOverviewData.errors} errors, " +
+                "1 passed, " +
+                "${testSuiteOverviewData.skipped} skipped, " +
+                "${testSuiteOverviewData.flakes} flaky"
+
+        // when
+        val result = mockedOutcome.getDetails(testSuiteOverviewData)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-crashed`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns true
+                every { timedOut } returns false
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "Application crashed"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-timedOut`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns true
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "Test timed out"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for failed-notInstalled`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns true
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "App failed to install"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should contains message about native crash when it happens`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns true
+                every { otherNativeCrash } returns true
+            }
+        }
+        val expectedMessage = "App failed to install (Native crash)"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-infrastructureFailure`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk { every { infrastructureFailure } returns true }
+        }
+        val expectedMessage = "Infrastructure failure"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-abortedByUser`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk() {
+                every { infrastructureFailure } returns false
+                every { abortedByUser } returns true
+            }
+        }
+        val expectedMessage = "Test run aborted by user"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for inconclusive-other reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.inconclusive
+            every { inconclusiveDetail } returns mockk {
+                every { infrastructureFailure } returns false
+                every { abortedByUser } returns false
+            }
+        }
+        val expectedMessage = "Unknown reason"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleDevice`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns true
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "Incompatible device/OS combination"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleArchitecture`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns true
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "App does not support the device architecture"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-incompatibleAppVersion`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns true
+            }
+        }
+        val expectedMessage = "App does not support the OS version"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for skipped-other reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.skipped
+            every { skippedDetail } returns mockk {
+                every { incompatibleDevice } returns false
+                every { incompatibleArchitecture } returns false
+                every { incompatibleAppVersion } returns false
+            }
+        }
+        val expectedMessage = "Unknown reason"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `should return correct outcome details for unset reason`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.unset
+        }
+        val expectedMessage = "unset"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+}
diff --git a/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt b/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt
new file mode 100644
index 0000000000..68c1556bca
--- /dev/null
+++ b/test_runner/src/test/kotlin/ftl/util/TableColumnTest.kt
@@ -0,0 +1,31 @@
+package ftl.util
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+
+internal class TableColumnTest {
+
+    @Test
+    fun `Should calculate properly column width`() {
+        // given
+        val expectedColumnSize = 8
+
+        // when
+        val tableColumn = TableColumn("55555", listOf("1", "22", "333", "4444", "55555", "666666"))
+
+        // then
+        assertThat(tableColumn.columnSize).isEqualTo(expectedColumnSize)
+    }
+
+    @Test
+    fun `Should calculate properly column width when header value is the longest`() {
+        // given
+        val expectedColumnSize = 7
+
+        // when
+        val tableColumn = TableColumn("55555", listOf("1", "22", "333", "4444"))
+
+        // then
+        assertThat(tableColumn.columnSize).isEqualTo(expectedColumnSize)
+    }
+}
diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt
index 5e2e359766..31dd6f599f 100644
--- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt
+++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt
@@ -278,11 +278,11 @@ class UtilsTest {
 
     private fun assertOutputIsEqualReferenceTable(outputLog: String, matrixId: String) {
         val referenceTable =
-            "┌─────────┬────────────────────────┬────────────────────┐\n" +
-            "│ OUTCOME │    TEST_AXIS_VALUE     │    TEST_DETAILS    │\n" +
-            "├─────────┼────────────────────────┼────────────────────┤\n" +
-            "│ Failed  │           $matrixId            │ Test failed to run │\n" +
-            "└─────────┴────────────────────────┴────────────────────┘"
+            "┌─────────┬───────────┬────────────────────┐\n" +
+            "│ OUTCOME │ MATRIX ID │    TEST DETAILS    │\n" +
+            "├─────────┼───────────┼────────────────────┤\n" +
+            "│ Failed  │     $matrixId     │ Test failed to run │\n" +
+            "└─────────┴───────────┴────────────────────┘"
         referenceTable.split("\n")
             .forEach {
                 println(it)

From a6e0a39b3b052891fed1d03fd14215b1ac4f52bc Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Tue, 23 Jun 2020 16:49:08 +0200
Subject: [PATCH 03/21] #829 update release_notes.md

---
 release_notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release_notes.md b/release_notes.md
index 4e854ce6f8..ff41ef9c05 100644
--- a/release_notes.md
+++ b/release_notes.md
@@ -5,7 +5,7 @@
 - [#828](https://github.com/Flank/flank/pull/828) Store test results in gcloud bucket. ([adamfilipow92](https://github.com/adamfilipow92))
 - [#865](https://github.com/Flank/flank/pull/865) Flank needs to respect the timeout value as that's a cap for billing purposes. ([adamfilipow92](https://github.com/adamfilipow92), [pawelpasterz](https://github.com/pawelpasterz))
 - [#866](https://github.com/Flank/flank/pull/866) Fix printing all matrix links. ([piotradamczyk5](https://github.com/piotradamczyk5))
--
+- [#862](https://github.com/Flank/flank/pull/862) Added printing outcome details. ([piotradamczyk5](https://github.com/piotradamczyk5))
 -
 -
 

From 0cc62e3717a1717cbedc2df5af312cd364617d0d Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Wed, 24 Jun 2020 15:22:37 +0200
Subject: [PATCH 04/21] #829 fix OutcomeDetailsFormatter.kt

---
 .../main/kotlin/ftl/util/OutcomeDetailsFormatter.kt    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
index 6f9939cc1a..d5c47d855a 100644
--- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
+++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -37,9 +37,9 @@ private val TestSuiteOverviewData.successCount
 
 private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when {
     this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
-    crashed -> "Application crashed"
-    timedOut -> "Test timed out"
-    notInstalled -> "App failed to install"
+    crashed == true -> "Application crashed"
+    timedOut == true -> "Test timed out"
+    notInstalled == true -> "App failed to install"
     else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
 } + this?.takeIf { it.otherNativeCrash }?.let { NATIVE_CRASH_MESSAGE }.orEmpty()
 
@@ -54,8 +54,8 @@ private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
 
 private fun InconclusiveDetail?.formatOutcomeDetails() = when {
     this == null -> "Unknown reason"
-    infrastructureFailure -> "Infrastructure failure"
-    abortedByUser -> "Test run aborted by user"
+    infrastructureFailure == true -> "Infrastructure failure"
+    abortedByUser == true -> "Test run aborted by user"
     else -> "Unknown reason"
 }
 

From 8d3335ef2e3f246d81fa2af5d742957908428404 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Wed, 24 Jun 2020 16:32:11 +0200
Subject: [PATCH 05/21] #829 Fixing pr comments

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 55 ++++++++++---------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 7f9d528966..d9fcffc3e4 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -14,16 +14,10 @@ import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
+import ftl.util.StepOutcome.unset
 import ftl.util.getDetails
 import ftl.util.webLink
 
-private data class FinishedTestMatrixData(
-    val stepOutcome: Outcome,
-    val isVirtualDevice: Boolean,
-    val testSuiteOverviewData: TestSuiteOverviewData?,
-    val billableMinutes: Long?
-)
-
 // execution gcs paths aren't API accessible.
 class SavedMatrix(matrix: TestMatrix) {
     val matrixId: String = matrix.testMatrixId
@@ -95,36 +89,45 @@ class SavedMatrix(matrix: TestMatrix) {
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
         matrix.testExecutions.createTestExecutionDataListAsync()
-            .map {
-                FinishedTestMatrixData(
+            .forEach {
+                updatedFinishedInfo(
                     stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
-                    isVirtualDevice = AndroidCatalog.isVirtualDevice(it.testExecution.environment.androidDevice, matrix.projectId.orEmpty()),
+                    isVirtualDevice = AndroidCatalog.isVirtualDevice(
+                        it.testExecution.environment.androidDevice,
+                        matrix.projectId.orEmpty()
+                    ),
                     testSuiteOverviewData = it.createTestSuitOverviewData(),
                     billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
                         ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
                 )
             }
-            .forEach { (stepOutcome, isVirtualDevice, testSuiteOverviewData, billableMinutes) ->
-                updateOutcome(stepOutcome)
-                updateOutcomeDetails(stepOutcome, testSuiteOverviewData)
-                billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
-            }
     }
 
-    private fun updateOutcome(stepOutcome: Outcome) {
-        // the matrix outcome is failure if any step fails
-        // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
-        // inconclusive is treated as a failure
-        if (outcome == failure || outcome == inconclusive) return
-
-        outcome = stepOutcome.summary
+    private fun updatedFinishedInfo(
+        stepOutcome: Outcome?,
+        isVirtualDevice: Boolean,
+        testSuiteOverviewData: TestSuiteOverviewData?,
+        billableMinutes: Long?
+    ) {
+        updateOutcome(stepOutcome)
+        updateOutcomeDetails(stepOutcome, testSuiteOverviewData)
+        billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
+    }
 
-        // Treat flaky outcome as a success
-        if (outcome == flaky) outcome = success
+    private fun updateOutcome(stepOutcome: Outcome?) {
+        outcome = when {
+            stepOutcome == null -> unset
+            // the matrix outcome is failure if any step fails
+            // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
+            // inconclusive is treated as a failure
+            outcome == failure || outcome == inconclusive -> return
+            stepOutcome.summary == flaky -> success
+            else -> stepOutcome.summary
+        }
     }
 
-    private fun updateOutcomeDetails(stepOutcome: Outcome, testSuiteOverviewData: TestSuiteOverviewData?) {
-        outcomeDetails = stepOutcome.getDetails(testSuiteOverviewData) ?: "---"
+    private fun updateOutcomeDetails(stepOutcome: Outcome?, testSuiteOverviewData: TestSuiteOverviewData?) {
+        outcomeDetails = stepOutcome?.getDetails(testSuiteOverviewData) ?: "---"
     }
 
     private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) {

From 943c48f2953621532815d2c5a814d6401c83c7ec Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Wed, 24 Jun 2020 19:11:28 +0200
Subject: [PATCH 06/21] #829 Handle unknown failure

---
 .../ftl/util/OutcomeDetailsFormatter.kt       | 22 ++++++-----
 .../ftl/util/OutcomeDetailsFormatterTest.kt   | 39 ++++++++++++++++++-
 2 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
index d5c47d855a..55b93766d6 100644
--- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
+++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -4,7 +4,6 @@ import com.google.api.services.toolresults.model.FailureDetail
 import com.google.api.services.toolresults.model.InconclusiveDetail
 import com.google.api.services.toolresults.model.Outcome
 import com.google.api.services.toolresults.model.SkippedDetail
-import com.google.api.services.toolresults.model.SuccessDetail
 import ftl.reports.api.data.TestSuiteOverviewData
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.flaky
@@ -12,10 +11,14 @@ import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
-import java.lang.StringBuilder
 
 fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (summary) {
-    success, flaky -> testSuiteOverviewData?.let { getSuccessOutcomeDetails(it, successDetail) }
+    success, flaky -> testSuiteOverviewData?.let {
+        getSuccessOutcomeDetails(
+            testSuiteOverviewData = it,
+            otherNativeCrash = successDetail?.otherNativeCrash ?: false
+        )
+    }
     failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData)
     inconclusive -> inconclusiveDetail.formatOutcomeDetails()
     skipped -> skippedDetail.formatOutcomeDetails()
@@ -25,32 +28,31 @@ fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (su
 
 private fun getSuccessOutcomeDetails(
     testSuiteOverviewData: TestSuiteOverviewData,
-    successDetail: SuccessDetail?
+    otherNativeCrash: Boolean
 ) = StringBuilder("${testSuiteOverviewData.successCount} test cases passed").apply {
     if (testSuiteOverviewData.skipped > 0) append(skippedMessage(testSuiteOverviewData.skipped))
     if (testSuiteOverviewData.flakes > 0) append(flakesMessage(testSuiteOverviewData.flakes))
-    if (successDetail?.otherNativeCrash == true) append(NATIVE_CRASH_MESSAGE)
+    if (otherNativeCrash) append(NATIVE_CRASH_MESSAGE)
 }.toString()
 
 private val TestSuiteOverviewData.successCount
     get() = total - errors - failures - skipped - flakes
 
 private fun FailureDetail?.getFailureOutcomeDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when {
-    this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+    this == null -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() ?: "Unknown failure"
     crashed == true -> "Application crashed"
     timedOut == true -> "Test timed out"
     notInstalled == true -> "App failed to install"
-    else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary()
+    else -> testSuiteOverviewData?.buildFailureOutcomeDetailsSummary() ?: "Unknown failure"
 } + this?.takeIf { it.otherNativeCrash }?.let { NATIVE_CRASH_MESSAGE }.orEmpty()
 
-private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary(): String {
-    return StringBuilder("$failures test casess failed").apply {
+private fun TestSuiteOverviewData.buildFailureOutcomeDetailsSummary() =
+    StringBuilder("$failures test cases failed").apply {
         if (errors > 0) append(errorMessage(errors))
         successCount.takeIf { it > 0 }?.let { append(successMessage(it)) }
         if (skipped > 0) append(skippedMessage(skipped))
         if (flakes > 0) append(flakesMessage(flakes))
     }.toString()
-}
 
 private fun InconclusiveDetail?.formatOutcomeDetails() = when {
     this == null -> "Unknown reason"
diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
index 540a8b0d97..03fcba060b 100644
--- a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
+++ b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
@@ -63,7 +63,7 @@ internal class OutcomeDetailsFormatterTest {
             }
         }
         val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
-        val expectedMessage = "${testSuiteOverviewData.failures} test casess failed, " +
+        val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " +
                 "${testSuiteOverviewData.errors} errors, " +
                 "1 passed, " +
                 "${testSuiteOverviewData.skipped} skipped, " +
@@ -139,6 +139,43 @@ internal class OutcomeDetailsFormatterTest {
         assertEquals(expectedMessage, result)
     }
 
+    @Test
+    fun `Should return unknown failure message for failed-null testSuiteOverviewData`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns mockk {
+                every { crashed } returns false
+                every { timedOut } returns false
+                every { notInstalled } returns false
+                every { otherNativeCrash } returns false
+            }
+        }
+        val expectedMessage = "Unknown failure"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
+    @Test
+    fun `Should return unknown failure message for failed-null testSuiteOverviewData and failure details present`() {
+        // given
+        val mockedOutcome = mockk<Outcome> {
+            every { summary } returns StepOutcome.failure
+            every { failureDetail } returns null
+        }
+        val expectedMessage = "Unknown failure"
+
+        // when
+        val result = mockedOutcome.getDetails(null)
+
+        // then
+        assertEquals(expectedMessage, result)
+    }
+
     @Test
     fun `should contains message about native crash when it happens`() {
         // given

From 9c170d205c969b51cebfa8b01c6132c215989650 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Thu, 25 Jun 2020 12:02:39 +0200
Subject: [PATCH 07/21] Simplify

---
 .../ftl/util/OutcomeDetailsFormatter.kt       | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
index 55b93766d6..141335f58f 100644
--- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
+++ b/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
@@ -13,25 +13,19 @@ import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
 
 fun Outcome.getDetails(testSuiteOverviewData: TestSuiteOverviewData?) = when (summary) {
-    success, flaky -> testSuiteOverviewData?.let {
-        getSuccessOutcomeDetails(
-            testSuiteOverviewData = it,
-            otherNativeCrash = successDetail?.otherNativeCrash ?: false
-        )
-    }
+    success, flaky -> testSuiteOverviewData?.getSuccessOutcomeDetails(successDetail?.otherNativeCrash ?: false)
     failure -> failureDetail.getFailureOutcomeDetails(testSuiteOverviewData)
     inconclusive -> inconclusiveDetail.formatOutcomeDetails()
     skipped -> skippedDetail.formatOutcomeDetails()
-    unset -> "unset"
-    else -> "unknown"
+    unset -> "Unset outcome"
+    else -> "Unknown outcome"
 }
 
-private fun getSuccessOutcomeDetails(
-    testSuiteOverviewData: TestSuiteOverviewData,
+private fun TestSuiteOverviewData.getSuccessOutcomeDetails(
     otherNativeCrash: Boolean
-) = StringBuilder("${testSuiteOverviewData.successCount} test cases passed").apply {
-    if (testSuiteOverviewData.skipped > 0) append(skippedMessage(testSuiteOverviewData.skipped))
-    if (testSuiteOverviewData.flakes > 0) append(flakesMessage(testSuiteOverviewData.flakes))
+) = StringBuilder("$successCount test cases passed").apply {
+    if (skipped > 0) append(skippedMessage(skipped))
+    if (flakes > 0) append(flakesMessage(flakes))
     if (otherNativeCrash) append(NATIVE_CRASH_MESSAGE)
 }.toString()
 

From 2d2d8a85bb3d75be6c576a5e99af7119b485d43b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Thu, 25 Jun 2020 12:06:08 +0200
Subject: [PATCH 08/21] Fix test

---
 .../src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
index 03fcba060b..fbe91a9d7d 100644
--- a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
+++ b/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
@@ -337,7 +337,7 @@ internal class OutcomeDetailsFormatterTest {
         val mockedOutcome = mockk<Outcome> {
             every { summary } returns StepOutcome.unset
         }
-        val expectedMessage = "unset"
+        val expectedMessage = "Unset outcome"
 
         // when
         val result = mockedOutcome.getDetails(null)

From 4e0e2666fa7ff086a91af4084c3f199dabfccdc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Thu, 25 Jun 2020 15:55:26 +0200
Subject: [PATCH 09/21] Add plant uml diagram for results_summary.py

---
 .../gcloud/firebase/test/results_summary.puml | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 docs/gcloud/firebase/test/results_summary.puml

diff --git a/docs/gcloud/firebase/test/results_summary.puml b/docs/gcloud/firebase/test/results_summary.puml
new file mode 100644
index 0000000000..ea3ac9ec13
--- /dev/null
+++ b/docs/gcloud/firebase/test/results_summary.puml
@@ -0,0 +1,57 @@
+@startuml
+start
+:CreateMatrixOutcomeSummaryUsingSteps;
+:_GetStepOutcomeDetails;
+if (outcome success) then (yes)
+    :_GetSuccessCountDetails;
+    if (successDetail and otherNativeCrash) then (yes)
+        :format details;
+    else (no)
+        :details;
+    endif
+else if (failure) then (yes)
+    if (failureDetail) then (yes)
+        :_GetFailureDetail;
+    elseif (not testExecutionStep) then (yes)
+        :Unknown failure;
+    else (no)
+        :_GetFailureOrFlakyCountDetails;
+    endif
+else if (inconclusive) then (yes)
+    :_GetInconclusiveDetail;
+elseif (skipped) then (yes)
+    :_GetSkippedDetail;
+else (no)
+    :Unknown outcome;
+endif
+end
+
+start
+:CreateMatrixOutcomeSummaryUsingEnvironments;
+if (environments and all environmentResult outcome) then (yes)
+:_GetEnvironmentOutcomeDetails;
+    if (outcome success) then (yes)
+        :_GetSuccessCountDetails;
+        if (successDetail and otherNativeCrash) then (yes)
+            :format details;
+        else (no)
+            :details;
+        endif
+    else if (failure or flaky) then (yes)
+        if (failureDetail) then (yes)
+            :_GetFailureDetail;
+        else (no)
+            :_GetFailureOrFlakyCountDetails;
+        endif
+    else if (inconclusive) then (yes)
+        :_GetInconclusiveDetail;
+    elseif (skipped) then (yes)
+        :_GetSkippedDetail;
+    else (no)
+        :Unknown outcome;
+    endif
+else (no)
+    :_GetStepOutcomeDetails;
+endif
+end
+@enduml

From 2fadd9d31909ce02d5ee46bef5194bdbfab8e936 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Thu, 25 Jun 2020 18:15:05 +0200
Subject: [PATCH 10/21] Move OutcomeDetailsFormatter.kt to same packet as
 SavedMatrix.kt

---
 .../main/kotlin/ftl/{util => json}/OutcomeDetailsFormatter.kt  | 2 +-
 test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt            | 1 -
 .../kotlin/ftl/{util => json}/OutcomeDetailsFormatterTest.kt   | 3 ++-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename test_runner/src/main/kotlin/ftl/{util => json}/OutcomeDetailsFormatter.kt (99%)
 rename test_runner/src/test/kotlin/ftl/{util => json}/OutcomeDetailsFormatterTest.kt (99%)

diff --git a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt
similarity index 99%
rename from test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
rename to test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt
index 141335f58f..0f52b1aee4 100644
--- a/test_runner/src/main/kotlin/ftl/util/OutcomeDetailsFormatter.kt
+++ b/test_runner/src/main/kotlin/ftl/json/OutcomeDetailsFormatter.kt
@@ -1,4 +1,4 @@
-package ftl.util
+package ftl.json
 
 import com.google.api.services.toolresults.model.FailureDetail
 import com.google.api.services.toolresults.model.InconclusiveDetail
diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index d9fcffc3e4..e18c4fa2e8 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -15,7 +15,6 @@ import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
-import ftl.util.getDetails
 import ftl.util.webLink
 
 // execution gcs paths aren't API accessible.
diff --git a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt
similarity index 99%
rename from test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
rename to test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt
index fbe91a9d7d..9ae0300a5e 100644
--- a/test_runner/src/test/kotlin/ftl/util/OutcomeDetailsFormatterTest.kt
+++ b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt
@@ -1,7 +1,8 @@
-package ftl.util
+package ftl.json
 
 import com.google.api.services.toolresults.model.Outcome
 import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.util.StepOutcome
 import io.mockk.every
 import io.mockk.mockk
 import org.junit.Assert.assertEquals

From 8102b715f1a35169df2ee4b26fd9335a9d8d51a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Fri, 26 Jun 2020 19:32:44 +0200
Subject: [PATCH 11/21] Fix

---
 test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt        | 7 +++----
 .../ftl/fixtures/test_app_cases/flank-multiple-flaky.yml   | 4 +++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index e18c4fa2e8..a03fdc19e0 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -7,10 +7,10 @@ import ftl.gc.GcToolResults
 import ftl.reports.api.createTestExecutionDataListAsync
 import ftl.reports.api.createTestSuitOverviewData
 import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.reports.api.prepareForJUnitResult
 import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
-import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
@@ -88,6 +88,7 @@ class SavedMatrix(matrix: TestMatrix) {
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
         matrix.testExecutions.createTestExecutionDataListAsync()
+            .prepareForJUnitResult()
             .forEach {
                 updatedFinishedInfo(
                     stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
@@ -115,13 +116,11 @@ class SavedMatrix(matrix: TestMatrix) {
 
     private fun updateOutcome(stepOutcome: Outcome?) {
         outcome = when {
-            stepOutcome == null -> unset
             // the matrix outcome is failure if any step fails
             // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
             // inconclusive is treated as a failure
             outcome == failure || outcome == inconclusive -> return
-            stepOutcome.summary == flaky -> success
-            else -> stepOutcome.summary
+            else -> stepOutcome?.summary ?: outcome
         }
     }
 
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
index 0854537738..1f2fbf1783 100644
--- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
+++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
@@ -8,9 +8,11 @@ gcloud:
   directories-to-pull: 
     - /sdcard/
   num-flaky-test-attempts: 3
+  use-orchestrator: false
+#  num-uniform-shards: 3
 
 flank:
   disable-sharding: false
-  max-test-shards: 2
+  max-test-shards: 4
   files-to-download:
     - .*/sdcard/[^/]+\.ec$

From 1949804c5857b4bf28f76575db54261c5df61aa3 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Mon, 29 Jun 2020 17:32:10 +0200
Subject: [PATCH 12/21] #829 Display outcome details based on shards sum

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 27 ++++++++++++++++---
 .../reports/api/data/TestSuiteOverviewData.kt | 13 ++++++++-
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index a03fdc19e0..800c448b5b 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -14,7 +14,6 @@ import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
-import ftl.util.StepOutcome.unset
 import ftl.util.webLink
 
 // execution gcs paths aren't API accessible.
@@ -87,7 +86,13 @@ class SavedMatrix(matrix: TestMatrix) {
     }
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
-        matrix.testExecutions.createTestExecutionDataListAsync()
+        val testExecutionData = matrix.testExecutions.createTestExecutionDataListAsync()
+        val initial = TestSuiteOverviewData(0, 0, 0, 0, 0, 0.0, 0.0)
+        // details
+        val summedTestSuiteOverviewData =
+            testExecutionData.prepareForJUnitResult().fold(initial) { sum, test -> sum + test.createTestSuitOverviewData() }
+
+        testExecutionData
             .prepareForJUnitResult()
             .forEach {
                 updatedFinishedInfo(
@@ -96,11 +101,27 @@ class SavedMatrix(matrix: TestMatrix) {
                         it.testExecution.environment.androidDevice,
                         matrix.projectId.orEmpty()
                     ),
-                    testSuiteOverviewData = it.createTestSuitOverviewData(),
+                    testSuiteOverviewData = summedTestSuiteOverviewData,
                     billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
                         ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
                 )
             }
+
+                // old method
+/*        data
+            .prepareForJUnitResult()
+            .forEach {
+                updatedFinishedInfo(
+                    stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
+                    isVirtualDevice = AndroidCatalog.isVirtualDevice(
+                        it.testExecution.environment.androidDevice,
+                        matrix.projectId.orEmpty()
+                    ),
+                    testSuiteOverviewData = it.createTestSuitOverviewData(),
+                    billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
+                        ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
+                )
+            }*/
     }
 
     private fun updatedFinishedInfo(
diff --git a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt
index 961cebab9c..b7c656bece 100644
--- a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt
+++ b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt
@@ -8,4 +8,15 @@ data class TestSuiteOverviewData(
     val skipped: Int,
     val elapsedTime: Double,
     val overheadTime: Double
-)
+) {
+    operator fun plus(nextData: TestSuiteOverviewData?) =
+        if (nextData == null) this else copy(
+            total = this.total + nextData.total,
+            errors = this.errors + nextData.errors,
+            failures = this.failures + nextData.failures,
+            flakes = this.flakes + nextData.flakes,
+            skipped = this.skipped + nextData.skipped,
+            elapsedTime = this.elapsedTime + nextData.elapsedTime,
+            overheadTime = this.overheadTime + nextData.overheadTime
+        )
+}

From e513201828342bb896431d5413ab59d700678af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Mon, 29 Jun 2020 20:23:55 +0200
Subject: [PATCH 13/21] Add summary_output.md doc

---
 docs/summary_output.md | 64 ++++++++++++++++++++++++++++++++++++++++++
 release_notes.md       |  2 +-
 2 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 docs/summary_output.md

diff --git a/docs/summary_output.md b/docs/summary_output.md
new file mode 100644
index 0000000000..a046dcc29b
--- /dev/null
+++ b/docs/summary_output.md
@@ -0,0 +1,64 @@
+# Formatted summary output 
+```
+┌─────────┬──────────────────────┬─────────────────────┐
+│ OUTCOME │      MATRIX ID       │    TEST DETAILS     │
+├─────────┼──────────────────────┼─────────────────────┤
+│ success │ matrix-1z85qtvdnvb0l │ 4 test cases passed │
+└─────────┴──────────────────────┴─────────────────────┘
+```
+
+
+## User scenario
+As a user I want to see finely formatted summary result at the end of execution output.
+
+## Motivation
+Gcloud prints summary output in the table. It looks nice and is readable. Why we wouldn't have same in flank?
+
+## Possible outputs
+Numbers are representing `OUTCOME` column, points are representing `TEST DETAILS` column.
+1. `success | flaky`
+    * `${1} test cases passed | ${2} skipped | ${3} flakes | (Native crash) | ---`
+2. `failure`
+    * `${1} test cases failed | ${2} errors | ${3} passed | ${4} skipped |  ${4} flakes  | (Native crash)`
+    * `Application crashed | (Native crash)`
+    * `Test timed out | (Native crash)`
+    * `App failed to install | (Native crash)`
+    * `Unknown failure | (Native crash)`
+3. `inconclusive`
+    * `Infrastructure failure`
+    * `Test run aborted by user`
+    * `Unknown reason`
+4. `skipped`
+    * `Incompatible device/OS combination`
+    * `App does not support the device architecture`
+    * `App does not support the OS version`
+    * `Unknown reason`
+
+## Implementation details
+
+### Outcome calculation
+It should be mentioned there are some crucial differences how flank and gcloud calculates outcome value.
+
+Gcloud is using following API calls
+1. `self._client.projects_histories_executions_environments.List(request)`
+2. `self._client.projects_histories_executions_steps.List(request)`
+
+The first one is default, but if returns any `environment` without `environmentResult.outcome`, the second one will be used to obtain `steps`. 
+Both `environemnts` and `steps` can provide `outcome`. The difference between them is the `steps` returns `success` event if tests are `flaky`.
+Currently, we don't know why `self._client.projects_histories_executions_environments.List(request)` may return empty `environmentResult.outcome`.
+
+In difference to gcloud flank uses 3 api call to obtain necessary data
+1. `TestMatrix` - `GcTesting.get.projects().testMatrices().get(projectId, testMatrixId)`
+2. `Step` - `toolsResults.projects().histories().executions().steps().get(projectId, historyId, executionId, stepId)`
+3. `ListTestCasesResponse` - `toolsResults.projects().histories().executions().steps().testCases().get(projectId, historyId, executionId, stepId)`
+
+`TestMatrix` from first call provides `ToolResultsStep` through `TestExecution` which is used to obtain arguments for next two calls.  
+
+This is part of flank legacy. Those api calls provides data for `JUnitResult`. 
+As `JUnitResult` contains all data required to generate `table` output, we can reuse it.
+In result, we are forced to calculate `flaky` outcomes on flank site because of `step`.
+Probably it is place for little improvement in the future.
+
+### Test details calculation
+When flank and gcloud implementations can be slightly different because of programming languages,
+the logic behind is the mainly same.
diff --git a/release_notes.md b/release_notes.md
index ff41ef9c05..e01794134d 100644
--- a/release_notes.md
+++ b/release_notes.md
@@ -5,7 +5,7 @@
 - [#828](https://github.com/Flank/flank/pull/828) Store test results in gcloud bucket. ([adamfilipow92](https://github.com/adamfilipow92))
 - [#865](https://github.com/Flank/flank/pull/865) Flank needs to respect the timeout value as that's a cap for billing purposes. ([adamfilipow92](https://github.com/adamfilipow92), [pawelpasterz](https://github.com/pawelpasterz))
 - [#866](https://github.com/Flank/flank/pull/866) Fix printing all matrix links. ([piotradamczyk5](https://github.com/piotradamczyk5))
-- [#862](https://github.com/Flank/flank/pull/862) Added printing outcome details. ([piotradamczyk5](https://github.com/piotradamczyk5))
+- [#862](https://github.com/Flank/flank/pull/862) Added printing outcome details. ([piotradamczyk5](https://github.com/piotradamczyk5), [jan-gogo](https://github.com/jan-gogo))
 -
 -
 

From ccbbe90e0c9966014c36b498ef8bb7e5c2d934b7 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Mon, 29 Jun 2020 20:35:01 +0200
Subject: [PATCH 14/21] #829 Show robo test information

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 62 +++++++------------
 1 file changed, 24 insertions(+), 38 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 800c448b5b..7e1f8b4bac 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -2,7 +2,7 @@ package ftl.json
 
 import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.toolresults.model.Outcome
-import ftl.android.AndroidCatalog
+import ftl.android.AndroidCatalog.isVirtualDevice
 import ftl.gc.GcToolResults
 import ftl.reports.api.createTestExecutionDataListAsync
 import ftl.reports.api.createTestSuitOverviewData
@@ -86,67 +86,53 @@ class SavedMatrix(matrix: TestMatrix) {
     }
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
-        val testExecutionData = matrix.testExecutions.createTestExecutionDataListAsync()
-        val initial = TestSuiteOverviewData(0, 0, 0, 0, 0, 0.0, 0.0)
-        // details
+        val testExecutionsData = matrix.testExecutions.createTestExecutionDataListAsync().prepareForJUnitResult()
         val summedTestSuiteOverviewData =
-            testExecutionData.prepareForJUnitResult().fold(initial) { sum, test -> sum + test.createTestSuitOverviewData() }
-
-        testExecutionData
-            .prepareForJUnitResult()
-            .forEach {
-                updatedFinishedInfo(
-                    stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
-                    isVirtualDevice = AndroidCatalog.isVirtualDevice(
-                        it.testExecution.environment.androidDevice,
-                        matrix.projectId.orEmpty()
-                    ),
-                    testSuiteOverviewData = summedTestSuiteOverviewData,
-                    billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
-                        ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
-                )
+            testExecutionsData.fold(TestSuiteOverviewData(0, 0, 0, 0, 0, 0.0, 0.0)) { sum, test ->
+                sum + test.createTestSuitOverviewData()
             }
 
-                // old method
-/*        data
-            .prepareForJUnitResult()
+        testExecutionsData
             .forEach {
                 updatedFinishedInfo(
                     stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
-                    isVirtualDevice = AndroidCatalog.isVirtualDevice(
-                        it.testExecution.environment.androidDevice,
-                        matrix.projectId.orEmpty()
+                    testSuiteOverviewData = summedTestSuiteOverviewData,
+                    isRoboTests = it.testExecution.testSpecification.androidRoboTest != null,
+                    isVirtualDevice = isVirtualDevice(
+                        device = it.testExecution.environment.androidDevice,
+                        projectId = matrix.projectId.orEmpty()
                     ),
-                    testSuiteOverviewData = it.createTestSuitOverviewData(),
                     billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
                         ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
                 )
-            }*/
+            }
     }
 
     private fun updatedFinishedInfo(
         stepOutcome: Outcome?,
-        isVirtualDevice: Boolean,
         testSuiteOverviewData: TestSuiteOverviewData?,
+        isRoboTests: Boolean,
+        isVirtualDevice: Boolean,
         billableMinutes: Long?
     ) {
         updateOutcome(stepOutcome)
-        updateOutcomeDetails(stepOutcome, testSuiteOverviewData)
+        updateOutcomeDetails(stepOutcome, testSuiteOverviewData, isRoboTests)
         billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
     }
 
     private fun updateOutcome(stepOutcome: Outcome?) {
-        outcome = when {
-            // the matrix outcome is failure if any step fails
-            // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
-            // inconclusive is treated as a failure
-            outcome == failure || outcome == inconclusive -> return
-            else -> stepOutcome?.summary ?: outcome
-        }
+        // the matrix outcome is failure if any step fails
+        // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
+        // inconclusive is treated as a failure
+        if (outcome != failure && outcome != inconclusive) outcome = stepOutcome?.summary ?: outcome
     }
 
-    private fun updateOutcomeDetails(stepOutcome: Outcome?, testSuiteOverviewData: TestSuiteOverviewData?) {
-        outcomeDetails = stepOutcome?.getDetails(testSuiteOverviewData) ?: "---"
+    private fun updateOutcomeDetails(
+        stepOutcome: Outcome?,
+        testSuiteOverviewData: TestSuiteOverviewData?,
+        isRoboTests: Boolean
+    ) {
+        outcomeDetails = if (isRoboTests) "Robo test" else (stepOutcome?.getDetails(testSuiteOverviewData) ?: "---")
     }
 
     private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) {

From 5ba7230970629ecf6507a7e4c611a40ab7bb6ee0 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Mon, 29 Jun 2020 21:37:52 +0200
Subject: [PATCH 15/21] #829 Fixed flaky outcome

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 22 ++++++++++++++-----
 .../kotlin/ftl/util/SavedMatrixTableUtil.kt   |  2 ++
 .../main/kotlin/ftl/util/SystemOutColor.kt    |  1 +
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 7e1f8b4bac..3bb914899f 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -11,6 +11,7 @@ import ftl.reports.api.prepareForJUnitResult
 import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
@@ -96,6 +97,7 @@ class SavedMatrix(matrix: TestMatrix) {
             .forEach {
                 updatedFinishedInfo(
                     stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
+                    flakyOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome.summary != it.step.outcome.summary,
                     testSuiteOverviewData = summedTestSuiteOverviewData,
                     isRoboTests = it.testExecution.testSpecification.androidRoboTest != null,
                     isVirtualDevice = isVirtualDevice(
@@ -110,21 +112,29 @@ class SavedMatrix(matrix: TestMatrix) {
 
     private fun updatedFinishedInfo(
         stepOutcome: Outcome?,
+        flakyOutcome: Boolean,
         testSuiteOverviewData: TestSuiteOverviewData?,
         isRoboTests: Boolean,
         isVirtualDevice: Boolean,
         billableMinutes: Long?
     ) {
-        updateOutcome(stepOutcome)
+        updateOutcome(stepOutcome, flakyOutcome)
         updateOutcomeDetails(stepOutcome, testSuiteOverviewData, isRoboTests)
         billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
     }
 
-    private fun updateOutcome(stepOutcome: Outcome?) {
-        // the matrix outcome is failure if any step fails
-        // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
-        // inconclusive is treated as a failure
-        if (outcome != failure && outcome != inconclusive) outcome = stepOutcome?.summary ?: outcome
+    private fun updateOutcome(
+        stepOutcome: Outcome?,
+        flakyOutcome: Boolean
+    ) {
+        outcome = when {
+            // the matrix outcome is failure if any step fails
+            // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
+            // inconclusive is treated as a failure
+            outcome == failure || outcome == inconclusive -> return
+            flakyOutcome -> flaky
+            else -> stepOutcome?.summary ?: outcome
+        }
     }
 
     private fun updateOutcomeDetails(
diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
index 4e6820c58c..910a5601c1 100644
--- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
+++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -2,6 +2,7 @@ package ftl.util
 
 import ftl.json.SavedMatrix
 import ftl.util.StepOutcome.failure
+import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.success
 
 fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable()
@@ -17,6 +18,7 @@ private fun getOutcomeColor(outcome: String): SystemOutColor {
     return when (outcome) {
         failure -> SystemOutColor.RED
         success -> SystemOutColor.GREEN
+        flaky -> SystemOutColor.BLUE
         else -> SystemOutColor.DEFAULT
     }
 }
diff --git a/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt b/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt
index c74037b889..d6490abf91 100644
--- a/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt
+++ b/test_runner/src/main/kotlin/ftl/util/SystemOutColor.kt
@@ -3,6 +3,7 @@ package ftl.util
 enum class SystemOutColor(val ansiCode: String) {
     DEFAULT("\u001B[0m"),
     RED("\u001B[31m"),
+    BLUE("\u001B[34m"),
     GREEN("\u001B[32m");
 
     fun applyTo(value: String) = ansiCode + value + DEFAULT.ansiCode

From 418e1f925d0f9c73e9faf90eb06534a5bef47c4d Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Tue, 30 Jun 2020 16:40:59 +0200
Subject: [PATCH 16/21] #829 Fixed tests and change flaky outcome logic

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 3bb914899f..b0e05e8cb1 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -87,19 +87,22 @@ class SavedMatrix(matrix: TestMatrix) {
     }
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
-        val testExecutionsData = matrix.testExecutions.createTestExecutionDataListAsync().prepareForJUnitResult()
+        val testExecutionsData = matrix.testExecutions.createTestExecutionDataListAsync()
         val summedTestSuiteOverviewData =
-            testExecutionsData.fold(TestSuiteOverviewData(0, 0, 0, 0, 0, 0.0, 0.0)) { sum, test ->
-                sum + test.createTestSuitOverviewData()
-            }
+            testExecutionsData
+                .prepareForJUnitResult()
+                .fold(TestSuiteOverviewData(0, 0, 0, 0, 0, 0.0, 0.0)) { sum, test ->
+                    sum + test.createTestSuitOverviewData()
+                }
 
         testExecutionsData
             .forEach {
+                val stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome
                 updatedFinishedInfo(
-                    stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome,
-                    flakyOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome.summary != it.step.outcome.summary,
+                    stepOutcome = stepOutcome,
+                    flakyOutcome = stepOutcome.summary != it.step.outcome.summary,
                     testSuiteOverviewData = summedTestSuiteOverviewData,
-                    isRoboTests = it.testExecution.testSpecification.androidRoboTest != null,
+                    isRoboTests = it.testExecution.testSpecification?.androidRoboTest != null,
                     isVirtualDevice = isVirtualDevice(
                         device = it.testExecution.environment.androidDevice,
                         projectId = matrix.projectId.orEmpty()
@@ -131,8 +134,9 @@ class SavedMatrix(matrix: TestMatrix) {
             // the matrix outcome is failure if any step fails
             // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
             // inconclusive is treated as a failure
-            outcome == failure || outcome == inconclusive -> return
             flakyOutcome -> flaky
+            outcome == failure || outcome == inconclusive -> return
+            outcome == flaky -> stepOutcome?.summary?.takeIf { it == failure || it == inconclusive } ?: outcome
             else -> stepOutcome?.summary ?: outcome
         }
     }

From 3ca25debfe6642d14a359df28f6a208f33db26a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Tue, 30 Jun 2020 17:12:25 +0200
Subject: [PATCH 17/21] Fix total count

---
 .../main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt
index 1cf0c696cc..92afd4f902 100644
--- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt
+++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt
@@ -12,7 +12,7 @@ internal fun TestExecutionData.createTestSuitOverviewData(): TestSuiteOverviewDa
     ?.let { overview ->
         val skipped: Int = overview.skippedCount ?: 0
         TestSuiteOverviewData(
-            total = testCases.size + skipped,
+            total = testCases.size,
             errors = testCases.countErrors(),
             failures = testCases.countFailures(),
             flakes = testCases.countFlakes(),

From d6e49ac5b2e2eca73a445df010de2a3f3ab61e28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janek=20G=C3=B3ral?= <jan.goral@gogoapps.io>
Date: Tue, 30 Jun 2020 17:17:07 +0200
Subject: [PATCH 18/21] Fix total count test

---
 .../kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt
index 7da6cb8534..d0b20e4321 100644
--- a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt
+++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt
@@ -48,7 +48,7 @@ class CreateTestSuitOverviewDataKtTest {
 
         // when
         val expected = TestSuiteOverviewData(
-            total = 5,
+            total = 4,
             errors = 1,
             failures = 1,
             skipped = 1,

From 1d58a1ab89a39288bfafd1354caf83c1e6b08ab4 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Tue, 30 Jun 2020 17:21:55 +0200
Subject: [PATCH 19/21] #829 small refactor

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 31 ++++++++++++++-----
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index b0e05e8cb1..3c13a2d82f 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -1,5 +1,6 @@
 package ftl.json
 
+import com.google.api.services.testing.model.TestExecution
 import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.toolresults.model.Outcome
 import ftl.android.AndroidCatalog.isVirtualDevice
@@ -8,14 +9,17 @@ import ftl.reports.api.createTestExecutionDataListAsync
 import ftl.reports.api.createTestSuitOverviewData
 import ftl.reports.api.data.TestSuiteOverviewData
 import ftl.reports.api.prepareForJUnitResult
-import ftl.util.Billing
+import ftl.util.MatrixState.ERROR
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
+import ftl.util.billableMinutes
+import ftl.util.timeoutToSeconds
 import ftl.util.webLink
+import kotlin.math.min
 
 // execution gcs paths aren't API accessible.
 class SavedMatrix(matrix: TestMatrix) {
@@ -107,12 +111,25 @@ class SavedMatrix(matrix: TestMatrix) {
                         device = it.testExecution.environment.androidDevice,
                         projectId = matrix.projectId.orEmpty()
                     ),
-                    billableMinutes = it.step.testExecutionStep?.testTiming?.testProcessDuration?.seconds
-                        ?.let { testTimeSeconds -> Billing.billableMinutes(testTimeSeconds) }
+                    billableMinutes = it.testExecution.getBillableMinutes()
                 )
             }
     }
 
+    private fun TestExecution.getBillableMinutes() =
+        takeIf { testExecution -> testExecution.state != ERROR }
+            ?.run {
+                // testExecutionStep, testTiming, etc. can all be null.
+                // sometimes testExecutionStep is present and testTiming is null
+                val testTimeSeconds =
+                    GcToolResults.getStepResult(toolResultsStep).testExecutionStep?.testTiming?.testProcessDuration?.seconds
+                        ?: return@run null
+                val testTimeout = timeoutToSeconds(testSpecification?.testTimeout ?: "0s")
+
+                // if overall test duration time is higher then testTimeout flank should calculate billable minutes for testTimeout
+                billableMinutes(min(testTimeSeconds, testTimeout))
+            }
+
     private fun updatedFinishedInfo(
         stepOutcome: Outcome?,
         flakyOutcome: Boolean,
@@ -131,14 +148,14 @@ class SavedMatrix(matrix: TestMatrix) {
         flakyOutcome: Boolean
     ) {
         outcome = when {
+            flakyOutcome -> flaky
             // the matrix outcome is failure if any step fails
             // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
             // inconclusive is treated as a failure
-            flakyOutcome -> flaky
             outcome == failure || outcome == inconclusive -> return
-            outcome == flaky -> stepOutcome?.summary?.takeIf { it == failure || it == inconclusive } ?: outcome
-            else -> stepOutcome?.summary ?: outcome
-        }
+            outcome == flaky -> stepOutcome?.summary?.takeIf { it == failure || it == inconclusive }
+            else -> stepOutcome?.summary
+        } ?: outcome
     }
 
     private fun updateOutcomeDetails(

From 45b46296f929753eed8eee4ddcb8067f3b93f176 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Tue, 30 Jun 2020 17:56:21 +0200
Subject: [PATCH 20/21] #829 PR fixes

---
 .../main/kotlin/ftl/util/SavedMatrixTableUtil.kt    | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
index 910a5601c1..19c461e8c9 100644
--- a/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
+++ b/test_runner/src/main/kotlin/ftl/util/SavedMatrixTableUtil.kt
@@ -13,14 +13,11 @@ fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
     TableColumn(OUTCOME_DETAILS_COLUMN_HEADER, mapNotNull { it.outcomeDetails })
 )
 
-private fun getOutcomeColor(outcome: String): SystemOutColor {
-    // inconclusive is treated as a failure, flaky as a success
-    return when (outcome) {
-        failure -> SystemOutColor.RED
-        success -> SystemOutColor.GREEN
-        flaky -> SystemOutColor.BLUE
-        else -> SystemOutColor.DEFAULT
-    }
+private fun getOutcomeColor(outcome: String) = when (outcome) {
+    failure -> SystemOutColor.RED
+    success -> SystemOutColor.GREEN
+    flaky -> SystemOutColor.BLUE
+    else -> SystemOutColor.DEFAULT
 }
 
 private const val OUTCOME_COLUMN_HEADER = "OUTCOME"

From d911dc5713be1d228d386e38f4cb6ae09cb39ff7 Mon Sep 17 00:00:00 2001
From: Piotr Adamczyk <piotr.adamczyk@gogoapps.io>
Date: Thu, 2 Jul 2020 15:51:14 +0200
Subject: [PATCH 21/21] #829 PR fixes

---
 .../src/main/kotlin/ftl/json/SavedMatrix.kt   | 58 ++++++++-----------
 .../main/kotlin/ftl/util/LogTableBuilder.kt   |  2 +-
 .../test_app_cases/flank-multiple-flaky.yml   |  2 -
 3 files changed, 24 insertions(+), 38 deletions(-)

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
index 3c13a2d82f..079d092b48 100644
--- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
+++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -85,13 +85,12 @@ class SavedMatrix(matrix: TestMatrix) {
         billableVirtualMinutes = 0
         billablePhysicalMinutes = 0
         outcome = success
-        if (matrix.testExecutions == null) return
 
         updateFinishedMatrixData(matrix)
     }
 
     private fun updateFinishedMatrixData(matrix: TestMatrix) {
-        val testExecutionsData = matrix.testExecutions.createTestExecutionDataListAsync()
+        val testExecutionsData = matrix.testExecutions?.createTestExecutionDataListAsync() ?: return
         val summedTestSuiteOverviewData =
             testExecutionsData
                 .prepareForJUnitResult()
@@ -101,18 +100,22 @@ class SavedMatrix(matrix: TestMatrix) {
 
         testExecutionsData
             .forEach {
-                val stepOutcome = GcToolResults.getExecutionResult(it.testExecution).outcome
-                updatedFinishedInfo(
-                    stepOutcome = stepOutcome,
-                    flakyOutcome = stepOutcome.summary != it.step.outcome.summary,
-                    testSuiteOverviewData = summedTestSuiteOverviewData,
-                    isRoboTests = it.testExecution.testSpecification?.androidRoboTest != null,
-                    isVirtualDevice = isVirtualDevice(
-                        device = it.testExecution.environment.androidDevice,
-                        projectId = matrix.projectId.orEmpty()
-                    ),
-                    billableMinutes = it.testExecution.getBillableMinutes()
-                )
+                with(GcToolResults.getExecutionResult(it.testExecution).outcome) {
+                    updateOutcome(it.step.outcome?.summary != this?.summary)
+                    updateOutcomeDetails(
+                        testSuiteOverviewData = summedTestSuiteOverviewData,
+                        isRoboTests = it.testExecution.testSpecification?.androidRoboTest != null
+                    )
+                }
+                it.testExecution.getBillableMinutes()?.let { billableMinutes ->
+                    updateBillableMinutes(
+                        billableMinutes = billableMinutes,
+                        isVirtualDevice = isVirtualDevice(
+                            it.testExecution.environment.androidDevice,
+                            matrix.projectId.orEmpty()
+                        )
+                    )
+                }
             }
     }
 
@@ -123,28 +126,14 @@ class SavedMatrix(matrix: TestMatrix) {
                 // sometimes testExecutionStep is present and testTiming is null
                 val testTimeSeconds =
                     GcToolResults.getStepResult(toolResultsStep).testExecutionStep?.testTiming?.testProcessDuration?.seconds
-                        ?: return@run null
+                        ?: return null
                 val testTimeout = timeoutToSeconds(testSpecification?.testTimeout ?: "0s")
 
                 // if overall test duration time is higher then testTimeout flank should calculate billable minutes for testTimeout
                 billableMinutes(min(testTimeSeconds, testTimeout))
             }
 
-    private fun updatedFinishedInfo(
-        stepOutcome: Outcome?,
-        flakyOutcome: Boolean,
-        testSuiteOverviewData: TestSuiteOverviewData?,
-        isRoboTests: Boolean,
-        isVirtualDevice: Boolean,
-        billableMinutes: Long?
-    ) {
-        updateOutcome(stepOutcome, flakyOutcome)
-        updateOutcomeDetails(stepOutcome, testSuiteOverviewData, isRoboTests)
-        billableMinutes?.let { updateBillableMinutes(it, isVirtualDevice) }
-    }
-
-    private fun updateOutcome(
-        stepOutcome: Outcome?,
+    private fun Outcome?.updateOutcome(
         flakyOutcome: Boolean
     ) {
         outcome = when {
@@ -153,17 +142,16 @@ class SavedMatrix(matrix: TestMatrix) {
             // if the matrix outcome is already set to failure then we can ignore the other step outcomes.
             // inconclusive is treated as a failure
             outcome == failure || outcome == inconclusive -> return
-            outcome == flaky -> stepOutcome?.summary?.takeIf { it == failure || it == inconclusive }
-            else -> stepOutcome?.summary
+            outcome == flaky -> this?.summary?.takeIf { it == failure || it == inconclusive }
+            else -> this?.summary
         } ?: outcome
     }
 
-    private fun updateOutcomeDetails(
-        stepOutcome: Outcome?,
+    private fun Outcome?.updateOutcomeDetails(
         testSuiteOverviewData: TestSuiteOverviewData?,
         isRoboTests: Boolean
     ) {
-        outcomeDetails = if (isRoboTests) "Robo test" else (stepOutcome?.getDetails(testSuiteOverviewData) ?: "---")
+        outcomeDetails = if (isRoboTests) "Robo test" else (this?.getDetails(testSuiteOverviewData) ?: "---")
     }
 
     private fun updateBillableMinutes(billableMinutes: Long, isVirtualDevice: Boolean) {
diff --git a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
index 1ef93384b7..d50ecb97b9 100644
--- a/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
+++ b/test_runner/src/main/kotlin/ftl/util/LogTableBuilder.kt
@@ -5,7 +5,7 @@ import com.google.common.annotations.VisibleForTesting
 data class TableColumn(
     val header: String,
     val data: List<String>,
-    val columnSize: Int = (data + header).maxBy { it.length }!!.length + DEFAULT_COLUMN_PADDING,
+    val columnSize: Int = ((data + header).maxBy { it.length }?.length ?: 0) + DEFAULT_COLUMN_PADDING,
     val dataColor: List<SystemOutColor> = listOf()
 )
 
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
index 1f2fbf1783..4fc27d41b5 100644
--- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
+++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml
@@ -9,8 +9,6 @@ gcloud:
     - /sdcard/
   num-flaky-test-attempts: 3
   use-orchestrator: false
-#  num-uniform-shards: 3
-
 flank:
   disable-sharding: false
   max-test-shards: 4