Removed test case time overhead calculation, as it was incorrectly pr…

…ocessing deflake reruns & reporting negative test case times. (#2364) Overhead (as described in #557) no longer exists in FTL, so this feature is not useful anymore. Fixes #2363
Flank · Apr 13, 2023 · f31dd9d · f31dd9d
1 parent d410932
commit f31dd9d
Show file tree

Hide file tree

Showing 10 changed files with 13 additions and 33 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt
@@ -61,7 +61,6 @@ fun TestOutcome.toApiModel() = Outcome(
         testSuiteOverview.flakes,
         testSuiteOverview.skipped,
         testSuiteOverview.elapsedTime,
-        testSuiteOverview.overheadTime
     )
 )
 

diff --git a/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt
@@ -70,7 +70,6 @@ object TestMatrix {
         val flakes: Int = 0,
         val skipped: Int = 0,
         val elapsedTime: Double = 0.0,
-        val overheadTime: Double = 0.0
     )
 
     data class BillableMinutes(

diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt
@@ -10,27 +10,24 @@ import ftl.reports.api.millis
 internal fun createJUnitTestCases(
     testCases: List<TestCase>,
     toolResultsStep: ToolResultsStep,
-    overheadTime: Double
 ): List<JUnitTest.Case> = testCases.map { testCase ->
     createJUnitTestCase(
         testCase = testCase,
         toolResultsStep = toolResultsStep,
-        overheadTime = overheadTime
     )
 }
 
 private fun createJUnitTestCase(
     testCase: TestCase,
     toolResultsStep: ToolResultsStep,
-    overheadTime: Double
 ): JUnitTest.Case {
     val stackTraces = mapOf(
         testCase.status to testCase.stackTraces?.map(StackTrace::getException)
     )
     return JUnitTest.Case(
         name = testCase.testCaseReference.name,
         classname = testCase.testCaseReference.className,
-        time = (testCase.elapsedTime.millis() + overheadTime).format(),
+        time = testCase.elapsedTime.millis().format(),
         failures = stackTraces["failed"],
         errors = stackTraces["error"],
         // skipped = true is represented by null. skipped = false is "absent"

diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt
@@ -28,7 +28,6 @@ private fun createJUnitTestSuite(
     testcases = createJUnitTestCases(
         testCases = data.testCases,
         toolResultsStep = data.testExecution.toolResultsStep,
-        overheadTime = overview.overheadTime
     ).toMutableList(),
     time = overview.elapsedTime.format()
 )
diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt
@@ -18,19 +18,9 @@ internal fun TestExecutionData.createTestSuiteOverviewData(): TestSuiteOverviewD
             flakes = testCases.countFlakes(),
             skipped = skipped,
             elapsedTime = overview.elapsedTime.millis(),
-            overheadTime = getOverheadTime(overview, testCases)
         )
     }
 
 private fun List<TestCase>.countErrors() = count { !it.flaky && it.status == "error" }
 private fun List<TestCase>.countFailures() = count { !it.flaky && it.status == "failed" }
 private fun List<TestCase>.countFlakes() = count { it.flaky }
-
-private fun getOverheadTime(
-    overview: TestSuiteOverview,
-    testCases: List<TestCase>
-) = if (testCases.isEmpty())
-    0.0 else
-    (overview.elapsedTime.millis() - testCases.sumTime()) / testCases.size
-
-private fun List<TestCase>.sumTime(): Double = sumOf { it.elapsedTime.millis() }
diff --git a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt
@@ -10,7 +10,6 @@ data class TestSuiteOverviewData(
     val flakes: Int = 0,
     val skipped: Int = 0,
     val elapsedTime: Double = 0.0,
-    val overheadTime: Double = 0.0
 ) {
     operator fun plus(nextData: TestSuiteOverviewData?) =
         if (nextData == null) this else copy(
@@ -20,7 +19,6 @@ data class TestSuiteOverviewData(
             flakes = this.flakes + nextData.flakes,
             skipped = this.skipped + nextData.skipped,
             elapsedTime = this.elapsedTime + nextData.elapsedTime,
-            overheadTime = this.overheadTime + nextData.overheadTime
         )
 
     operator fun plus(data: TestSuiteOverview) = copy(

diff --git a/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt
@@ -23,7 +23,7 @@ internal class OutcomeDetailsFormatterTest {
             every { summary } returns StepOutcome.success
             every { successDetail } returns mockk { every { otherNativeCrash } returns false }
         }
-        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0)
         val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
         val expectedMessage = "$successCount test cases passed, " +
             "${testSuiteOverviewData.skipped} skipped, " +
@@ -43,7 +43,7 @@ internal class OutcomeDetailsFormatterTest {
             every { summary } returns StepOutcome.success
             every { successDetail } returns mockk { every { otherNativeCrash } returns true }
         }
-        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0)
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0)
         val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped }
         val expectedMessage = "$successCount test cases passed, " +
             "${testSuiteOverviewData.skipped} skipped, " +
@@ -64,7 +64,7 @@ internal class OutcomeDetailsFormatterTest {
             every { summary } returns StepOutcome.failure
             every { failureDetail } returns mockk(relaxed = true) {}
         }
-        val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0)
+        val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0)
         val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " +
             "${testSuiteOverviewData.errors} errors, " +
             "1 passed, " +

diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt
@@ -50,15 +50,15 @@ class CreateJUnitTestCaseKtTest {
             JUnitTest.Case(
                 name = "test1",
                 classname = "TestClassName",
-                time = "2.200"
+                time = "1.100"
             ).apply {
                 webLink =
                     "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test1"
             },
             JUnitTest.Case(
                 name = "test2",
                 classname = "TestClassName",
-                time = "2.200",
+                time = "1.100",
                 skipped = null
             ).apply {
                 webLink =
@@ -67,7 +67,7 @@ class CreateJUnitTestCaseKtTest {
             JUnitTest.Case(
                 name = "test3",
                 classname = "TestClassName",
-                time = "2.200",
+                time = "1.100",
                 errors = listOf("exception")
             ).apply {
                 webLink =
@@ -76,7 +76,7 @@ class CreateJUnitTestCaseKtTest {
             JUnitTest.Case(
                 name = "test4",
                 classname = "TestClassName",
-                time = "2.200",
+                time = "1.100",
                 failures = listOf("exception")
             ).apply {
                 webLink =
@@ -85,7 +85,7 @@ class CreateJUnitTestCaseKtTest {
             JUnitTest.Case(
                 name = "test5",
                 classname = "TestClassName",
-                time = "2.200",
+                time = "1.100",
                 failures = listOf("exception")
             ).apply {
                 webLink =
@@ -97,7 +97,6 @@ class CreateJUnitTestCaseKtTest {
         val actual = createJUnitTestCases(
             testCases = testCases,
             toolResultsStep = toolResultsStep,
-            overheadTime = 1.1
         )
 
         // then

diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt
@@ -42,12 +42,12 @@ class CreateJUnitTestSuiteKtTest {
         // given
         every {
             any<TestExecutionData>().createTestSuiteOverviewData()
-        } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1, 1.1)
+        } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1)
 
         val jUnitTestCase = JUnitTest.Case(null, null, "1.1")
 
         every {
-            createJUnitTestCases(any(), any(), any())
+            createJUnitTestCases(any(), any())
         } returns listOf(jUnitTestCase)
 
         val testExecutionDataList = listOf(

diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt
@@ -40,7 +40,7 @@ class CreateTestSuiteOverviewDataKtTest {
                         TestSuiteOverview().apply {
                             skippedCount = 1
                             elapsedTime = Duration().apply {
-                                seconds = 8
+                                seconds = 4
                             }
                         }
                     )
@@ -55,8 +55,7 @@ class CreateTestSuiteOverviewDataKtTest {
             failures = 1,
             skipped = 1,
             flakes = 1,
-            elapsedTime = 8.0,
-            overheadTime = 1.0
+            elapsedTime = 4.0,
         )
         val actual = testExecutionData.createTestSuiteOverviewData()