Add overhead time to junit test case report

release_notes.md

@@ -1,5 +1,6 @@
 ## next (unreleased)
 - [#687](https://github.com/Flank/flank/pull/687) Debug message printed after every command. ([pawelpasterz](https://github.com/pawelpasterz))
+- [#684](https://github.com/Flank/flank/pull/684) Add overhead time to junit test case report. ([jan-gogo](https://github.com/jan-gogo))
 - [#666](https://github.com/Flank/flank/pull/666) Use API instead of XML for result parsing for android. ([jan-gogo](https://github.com/jan-gogo))
 - [#678](https://github.com/Flank/flank/pull/678) Skip Bugsnag initialization if user disabled gcloud analytics. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#672](https://github.com/Flank/flank/pull/672) Flank timeout feature. ([pawelpasterz](https://github.com/pawelpasterz))

test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt

@@ -7,22 +7,28 @@ import ftl.reports.xml.model.JUnitTestCase
 
 internal fun createJUnitTestCases(
     testCases: List<TestCase>,
-    toolResultsStep: ToolResultsStep
+    toolResultsStep: ToolResultsStep,
+    overheadTime: Double
 ): List<JUnitTestCase> = testCases.map { testCase ->
-    createJUnitTestCase(testCase, toolResultsStep)
+    createJUnitTestCase(
+        testCase = testCase,
+        toolResultsStep = toolResultsStep,
+        overheadTime = overheadTime
+    )
 }
 
 private fun createJUnitTestCase(
     testCase: TestCase,
-    toolResultsStep: ToolResultsStep
+    toolResultsStep: ToolResultsStep,
+    overheadTime: Double
 ): JUnitTestCase {
     val stackTraces = mapOf(
         testCase.status to testCase.stackTraces?.map(StackTrace::getException)
     )
     return JUnitTestCase(
         name = testCase.testCaseReference.name,
         classname = testCase.testCaseReference.className,
-        time = testCase.elapsedTime.format(),
+        time = (testCase.elapsedTime.millis() + overheadTime).format(),
         failures = stackTraces["failed"],
         errors = stackTraces["error"],
         // skipped = true is represented by null. skipped = false is "absent"

test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt

@@ -3,24 +3,18 @@ package ftl.reports.api
 import com.google.api.services.toolresults.model.Step
 import ftl.reports.api.data.TestExecutionData
 import ftl.reports.api.data.TestSuiteOverviewData
-import ftl.reports.xml.model.JUnitTestCase
 import ftl.reports.xml.model.JUnitTestSuite
 
 internal fun List<TestExecutionData>.createJUnitTestSuites() = map { data: TestExecutionData ->
     createJUnitTestSuite(
         data = data,
-        overview = data.createTestSuitOverviewData(),
-        testCases = createJUnitTestCases(
-            testCases = data.testCases,
-            toolResultsStep = data.testExecution.toolResultsStep
-        )
+        overview = data.createTestSuitOverviewData()
     )
 }
 
 private fun createJUnitTestSuite(
     data: TestExecutionData,
-    overview: TestSuiteOverviewData,
-    testCases: List<JUnitTestCase>
+    overview: TestSuiteOverviewData
 ) = JUnitTestSuite(
     name = data.step.testSuiteName(),
     timestamp = data.timestamp.asUnixTimestamp().formatUtcDate(),
@@ -29,15 +23,15 @@ private fun createJUnitTestSuite(
     errors = overview.errors.toString(),
     skipped = overview.skipped.toString(),
     flakes = overview.flakes,
-    testcases = testCases.toMutableList(),
-    time = testCases.sumTime().format() // FIXME include also setup and teardown duration https://github.com/Flank/flank/issues/557
+    testcases = createJUnitTestCases(
+        testCases = data.testCases,
+        toolResultsStep = data.testExecution.toolResultsStep,
+        overheadTime = overview.overheadTime
+    ).toMutableList(),
+    time = overview.elapsedTime.format()
 )
 
 private fun Step.testSuiteName(): String {
     val map = dimensionValue.map { it.key to it.value }.toMap()
     return listOf(map["Model"], map["Version"], map["Locale"], map["Orientation"]).joinToString("-")
 }
-
-private fun List<JUnitTestCase>.sumTime(): Double = this
-    .map { it.time?.toDouble() ?: 0.0 }
-    .reduce { acc, d -> acc + d }

test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt

@@ -1,20 +1,33 @@
 package ftl.reports.api
 
 import com.google.api.services.toolresults.model.TestCase
+import com.google.api.services.toolresults.model.TestSuiteOverview
 import ftl.reports.api.data.TestExecutionData
 import ftl.reports.api.data.TestSuiteOverviewData
 
 internal fun TestExecutionData.createTestSuitOverviewData(): TestSuiteOverviewData {
-    val skipped = step.testExecutionStep.testSuiteOverviews.first().skippedCount ?: 0
+    val overview: TestSuiteOverview = step.testExecutionStep.testSuiteOverviews.first()
+    val skipped: Int = overview.skippedCount ?: 0
     return TestSuiteOverviewData(
         total = testCases.size + skipped,
         errors = testCases.countErrors(),
         failures = testCases.countFailures(),
         flakes = testCases.countFlakes(),
-        skipped = skipped
+        skipped = skipped,
+        elapsedTime = overview.elapsedTime.millis(),
+        overheadTime = getOverheadTime(overview, testCases)
     )
 }
 
 private fun List<TestCase>.countErrors() = count { !it.flaky && it.status == "error" }
 private fun List<TestCase>.countFailures() = count { !it.flaky && it.status == "failed" }
 private fun List<TestCase>.countFlakes() = count { it.flaky }
+
+private fun getOverheadTime(
+    overview: TestSuiteOverview,
+    testCases: List<TestCase>
+) = if (testCases.isEmpty())
+    0.0 else
+    (overview.elapsedTime.millis() - testCases.sumTime()) / testCases.size
+
+private fun List<TestCase>.sumTime(): Double = sumByDouble { it.elapsedTime.millis() }

test_runner/src/main/kotlin/ftl/reports/api/Utils.kt

@@ -12,10 +12,10 @@ internal fun Double.format(): String = "%.3f".format(Locale.ROOT, this)
 
 internal fun Int?.format() = (this ?: 0).toString()
 
-internal fun Duration?.format(): String =
+internal fun Duration?.millis(): Double =
     if (this == null)
-        "0.0" else
-        ((seconds ?: 0) + nanosToSeconds(nanos)).format()
+        0.0 else
+        ((seconds ?: 0) + nanosToSeconds(nanos))
 
 //   manually divide to keep fractional precision
 private fun nanosToSeconds(nanos: Int?): Double =

test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt

@@ -5,5 +5,7 @@ data class TestSuiteOverviewData(
     val errors: Int,
     val failures: Int,
     val flakes: Int,
-    val skipped: Int
+    val skipped: Int,
+    val elapsedTime: Double,
+    val overheadTime: Double
 )

test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt

@@ -62,7 +62,11 @@ object ReportManager {
         return matchResult?.groupValues?.last().orEmpty()
     }
 
-    private fun processXmlFromFile(matrices: MatrixMap, args: IArgs, process: (file: File) -> JUnitTestResult): JUnitTestResult? {
+    private fun processXmlFromFile(
+        matrices: MatrixMap,
+        args: IArgs,
+        process: (file: File) -> JUnitTestResult
+    ): JUnitTestResult? {
         var mergedXml: JUnitTestResult? = null
 
         findXmlFiles(matrices, args).forEach { xmlFile ->
@@ -132,13 +136,11 @@ object ReportManager {
         newResult: JUnitTestResult,
         args: IArgs,
         testShardChunks: ShardChunks
-    ):
-            List<ShardEfficiency> {
+    ): List<ShardEfficiency> {
         val oldDurations = Shard.createTestMethodDurationMap(oldResult, args)
         val newDurations = Shard.createTestMethodDurationMap(newResult, args)
 
-        val timeList = mutableListOf<ShardEfficiency>()
-        testShardChunks.forEachIndexed { index, testSuite ->
+        return testShardChunks.mapIndexed { index, testSuite ->
 
             var expectedTime = 0.0
             var finalTime = 0.0
@@ -148,10 +150,8 @@ object ReportManager {
             }
 
             val timeDiff = (finalTime - expectedTime)
-            timeList.add(ShardEfficiency("Shard $index", expectedTime, finalTime, timeDiff))
+            ShardEfficiency("Shard $index", expectedTime, finalTime, timeDiff)
         }
-
-        return timeList
     }
 
     private fun printActual(
@@ -176,7 +176,9 @@ object ReportManager {
 
         val oldTestResult = GcStorage.downloadJunitXml(args)
 
-        newTestResult.mergeTestTimes(oldTestResult)
+        if (args.useLegacyJUnitResult) {
+            newTestResult.mergeTestTimes(oldTestResult)
+        }
 
         if (oldTestResult != null) {
             printActual(oldTestResult, newTestResult, args, testShardChunks)

test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt

@@ -48,34 +48,34 @@ class CreateJUnitTestCaseKtTest {
             JUnitTestCase(
                 name = "test1",
                 classname = "TestClassName",
-                time = "1.100"
+                time = "2.200"
             ),
             JUnitTestCase(
                 name = "test2",
                 classname = "TestClassName",
-                time = "1.100",
+                time = "2.200",
                 skipped = null
             ),
             JUnitTestCase(
                 name = "test3",
                 classname = "TestClassName",
-                time = "1.100",
+                time = "2.200",
                 errors = listOf("exception")
             ).apply {
                 webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test3"
             },
             JUnitTestCase(
                 name = "test4",
                 classname = "TestClassName",
-                time = "1.100",
+                time = "2.200",
                 failures = listOf("exception")
             ).apply {
                 webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test4"
             },
             JUnitTestCase(
                 name = "test5",
                 classname = "TestClassName",
-                time = "1.100",
+                time = "2.200",
                 failures = listOf("exception")
             ).apply {
                 webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test5"
@@ -85,7 +85,8 @@ class CreateJUnitTestCaseKtTest {
 
         val actual = createJUnitTestCases(
             testCases = testCases,
-            toolResultsStep = toolResultsStep
+            toolResultsStep = toolResultsStep,
+            overheadTime = 1.1
         )
 
         // then

test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt

@@ -39,12 +39,12 @@ class CreateJUnitTestSuiteKtTest {
         // given
         every {
             any<TestExecutionData>().createTestSuitOverviewData()
-        } returns TestSuiteOverviewData(1, 1, 1, 1, 1)
+        } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1, 1.1)
 
         val jUnitTestCase = JUnitTestCase(null, null, "1.1")
 
         every {
-            createJUnitTestCases(any(), any())
+            createJUnitTestCases(any(), any(), any())
         } returns listOf(jUnitTestCase)
 
         val testExecutionDataList = listOf(

test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt

@@ -1,6 +1,7 @@
 package ftl.reports.api
 
 import com.google.api.services.testing.model.TestExecution
+import com.google.api.services.toolresults.model.Duration
 import com.google.api.services.toolresults.model.Step
 import com.google.api.services.toolresults.model.TestCase
 import com.google.api.services.toolresults.model.TestExecutionStep
@@ -24,12 +25,21 @@ class CreateTestSuitOverviewDataKtTest {
                 TestCase().apply { status = "error" },
                 TestCase().apply { status = "failed" },
                 TestCase().apply { flaky = true }
-            ),
+            ).apply {
+                forEach {
+                    it.elapsedTime = Duration().apply {
+                        seconds = 1
+                    }
+                }
+            },
             step = Step().apply {
                 testExecutionStep = TestExecutionStep().apply {
                     testSuiteOverviews = listOf(
                         TestSuiteOverview().apply {
                             skippedCount = 1
+                            elapsedTime = Duration().apply {
+                                seconds = 8
+                            }
                         }
                     )
                 }
@@ -42,7 +52,9 @@ class CreateTestSuitOverviewDataKtTest {
             errors = 1,
             failures = 1,
             skipped = 1,
-            flakes = 1
+            flakes = 1,
+            elapsedTime = 8.0,
+            overheadTime = 1.0
         )
         val actual = testExecutionData.createTestSuitOverviewData()