From f31dd9d1fb13b1a38bdcc8de81a87c7fe7d2084c Mon Sep 17 00:00:00 2001 From: Fry-kun Date: Thu, 13 Apr 2023 11:24:15 -0700 Subject: [PATCH] Removed test case time overhead calculation, as it was incorrectly processing deflake reruns & reporting negative test case times. (#2364) Overhead (as described in #557) no longer exists in FTL, so this feature is not useful anymore. Fixes #2363 --- .../ftl/adapter/google/GoogleTestMatrixAdapter.kt | 1 - test_runner/src/main/kotlin/ftl/api/TestMatrix.kt | 1 - .../kotlin/ftl/client/junit/CreateJUnitTestCase.kt | 5 +---- .../kotlin/ftl/client/junit/CreateJUnitTestSuite.kt | 1 - .../ftl/client/junit/CreateTestSuiteOverviewData.kt | 10 ---------- .../ftl/reports/api/data/TestSuiteOverviewData.kt | 2 -- .../kotlin/ftl/json/OutcomeDetailsFormatterTest.kt | 6 +++--- .../ftl/reports/api/CreateJUnitTestCaseKtTest.kt | 11 +++++------ .../ftl/reports/api/CreateJUnitTestSuiteKtTest.kt | 4 ++-- .../reports/api/CreateTestSuiteOverviewDataKtTest.kt | 5 ++--- 10 files changed, 13 insertions(+), 33 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt index 0a14a2a1c8..67770c7512 100644 --- a/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt +++ b/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt @@ -61,7 +61,6 @@ fun TestOutcome.toApiModel() = Outcome( testSuiteOverview.flakes, testSuiteOverview.skipped, testSuiteOverview.elapsedTime, - testSuiteOverview.overheadTime ) ) diff --git a/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt index eb2f942d9e..e93898cc41 100644 --- a/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt @@ -70,7 +70,6 @@ object TestMatrix { val flakes: Int = 0, val skipped: Int = 0, val elapsedTime: Double = 0.0, - val overheadTime: Double = 0.0 ) data class BillableMinutes( diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt index ee62a4d42e..a650e85ded 100644 --- a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt @@ -10,19 +10,16 @@ import ftl.reports.api.millis internal fun createJUnitTestCases( testCases: List, toolResultsStep: ToolResultsStep, - overheadTime: Double ): List = testCases.map { testCase -> createJUnitTestCase( testCase = testCase, toolResultsStep = toolResultsStep, - overheadTime = overheadTime ) } private fun createJUnitTestCase( testCase: TestCase, toolResultsStep: ToolResultsStep, - overheadTime: Double ): JUnitTest.Case { val stackTraces = mapOf( testCase.status to testCase.stackTraces?.map(StackTrace::getException) @@ -30,7 +27,7 @@ private fun createJUnitTestCase( return JUnitTest.Case( name = testCase.testCaseReference.name, classname = testCase.testCaseReference.className, - time = (testCase.elapsedTime.millis() + overheadTime).format(), + time = testCase.elapsedTime.millis().format(), failures = stackTraces["failed"], errors = stackTraces["error"], // skipped = true is represented by null. skipped = false is "absent" diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt index 002a488a26..e4c8d0de84 100644 --- a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt @@ -28,7 +28,6 @@ private fun createJUnitTestSuite( testcases = createJUnitTestCases( testCases = data.testCases, toolResultsStep = data.testExecution.toolResultsStep, - overheadTime = overview.overheadTime ).toMutableList(), time = overview.elapsedTime.format() ) diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt index 3f948e5125..cbf41ed14d 100644 --- a/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt @@ -18,19 +18,9 @@ internal fun TestExecutionData.createTestSuiteOverviewData(): TestSuiteOverviewD flakes = testCases.countFlakes(), skipped = skipped, elapsedTime = overview.elapsedTime.millis(), - overheadTime = getOverheadTime(overview, testCases) ) } private fun List.countErrors() = count { !it.flaky && it.status == "error" } private fun List.countFailures() = count { !it.flaky && it.status == "failed" } private fun List.countFlakes() = count { it.flaky } - -private fun getOverheadTime( - overview: TestSuiteOverview, - testCases: List -) = if (testCases.isEmpty()) - 0.0 else - (overview.elapsedTime.millis() - testCases.sumTime()) / testCases.size - -private fun List.sumTime(): Double = sumOf { it.elapsedTime.millis() } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt index 0554947aa1..04ec0beb73 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt @@ -10,7 +10,6 @@ data class TestSuiteOverviewData( val flakes: Int = 0, val skipped: Int = 0, val elapsedTime: Double = 0.0, - val overheadTime: Double = 0.0 ) { operator fun plus(nextData: TestSuiteOverviewData?) = if (nextData == null) this else copy( @@ -20,7 +19,6 @@ data class TestSuiteOverviewData( flakes = this.flakes + nextData.flakes, skipped = this.skipped + nextData.skipped, elapsedTime = this.elapsedTime + nextData.elapsedTime, - overheadTime = this.overheadTime + nextData.overheadTime ) operator fun plus(data: TestSuiteOverview) = copy( diff --git a/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt index 6fea0c2cf9..def4aefb86 100644 --- a/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/OutcomeDetailsFormatterTest.kt @@ -23,7 +23,7 @@ internal class OutcomeDetailsFormatterTest { every { summary } returns StepOutcome.success every { successDetail } returns mockk { every { otherNativeCrash } returns false } } - val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) + val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0) val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } val expectedMessage = "$successCount test cases passed, " + "${testSuiteOverviewData.skipped} skipped, " + @@ -43,7 +43,7 @@ internal class OutcomeDetailsFormatterTest { every { summary } returns StepOutcome.success every { successDetail } returns mockk { every { otherNativeCrash } returns true } } - val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0, 0.0) + val testSuiteOverviewData = TestSuiteOverviewData(12, 0, 0, 3, 2, 0.0) val successCount = with(testSuiteOverviewData) { total - errors - failures - flakes - skipped } val expectedMessage = "$successCount test cases passed, " + "${testSuiteOverviewData.skipped} skipped, " + @@ -64,7 +64,7 @@ internal class OutcomeDetailsFormatterTest { every { summary } returns StepOutcome.failure every { failureDetail } returns mockk(relaxed = true) {} } - val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0, 0.0) + val testSuiteOverviewData = TestSuiteOverviewData(12, 3, 3, 3, 2, 0.0) val expectedMessage = "${testSuiteOverviewData.failures} test cases failed, " + "${testSuiteOverviewData.errors} errors, " + "1 passed, " + diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt index 9449e95708..4aaf163417 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt @@ -50,7 +50,7 @@ class CreateJUnitTestCaseKtTest { JUnitTest.Case( name = "test1", classname = "TestClassName", - time = "2.200" + time = "1.100" ).apply { webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test1" @@ -58,7 +58,7 @@ class CreateJUnitTestCaseKtTest { JUnitTest.Case( name = "test2", classname = "TestClassName", - time = "2.200", + time = "1.100", skipped = null ).apply { webLink = @@ -67,7 +67,7 @@ class CreateJUnitTestCaseKtTest { JUnitTest.Case( name = "test3", classname = "TestClassName", - time = "2.200", + time = "1.100", errors = listOf("exception") ).apply { webLink = @@ -76,7 +76,7 @@ class CreateJUnitTestCaseKtTest { JUnitTest.Case( name = "test4", classname = "TestClassName", - time = "2.200", + time = "1.100", failures = listOf("exception") ).apply { webLink = @@ -85,7 +85,7 @@ class CreateJUnitTestCaseKtTest { JUnitTest.Case( name = "test5", classname = "TestClassName", - time = "2.200", + time = "1.100", failures = listOf("exception") ).apply { webLink = @@ -97,7 +97,6 @@ class CreateJUnitTestCaseKtTest { val actual = createJUnitTestCases( testCases = testCases, toolResultsStep = toolResultsStep, - overheadTime = 1.1 ) // then diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt index 4e3b003f63..e85e84e301 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt @@ -42,12 +42,12 @@ class CreateJUnitTestSuiteKtTest { // given every { any().createTestSuiteOverviewData() - } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1, 1.1) + } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1) val jUnitTestCase = JUnitTest.Case(null, null, "1.1") every { - createJUnitTestCases(any(), any(), any()) + createJUnitTestCases(any(), any()) } returns listOf(jUnitTestCase) val testExecutionDataList = listOf( diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt index 1a2611858c..e6ddd845e4 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt @@ -40,7 +40,7 @@ class CreateTestSuiteOverviewDataKtTest { TestSuiteOverview().apply { skippedCount = 1 elapsedTime = Duration().apply { - seconds = 8 + seconds = 4 } } ) @@ -55,8 +55,7 @@ class CreateTestSuiteOverviewDataKtTest { failures = 1, skipped = 1, flakes = 1, - elapsedTime = 8.0, - overheadTime = 1.0 + elapsedTime = 4.0, ) val actual = testExecutionData.createTestSuiteOverviewData()