From b5a56fa29dead9938ea8e5ff3be17cd64daf0dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Mon, 30 Mar 2020 10:58:08 +0200 Subject: [PATCH] Add overhead time to junit test case report (#684) * Add overhead time to junit test case report * Omit mergeTestTimes if used new api results --- release_notes.md | 1 + .../ftl/reports/api/CreateJUnitTestCase.kt | 14 ++++++++---- .../ftl/reports/api/CreateJUnitTestSuite.kt | 22 +++++++------------ .../reports/api/CreateTestSuitOverviewData.kt | 17 ++++++++++++-- .../src/main/kotlin/ftl/reports/api/Utils.kt | 6 ++--- .../reports/api/data/TestSuiteOverviewData.kt | 4 +++- .../kotlin/ftl/reports/util/ReportManager.kt | 20 +++++++++-------- .../reports/api/CreateJUnitTestCaseKtTest.kt | 13 ++++++----- .../reports/api/CreateJUnitTestSuiteKtTest.kt | 4 ++-- .../api/CreateTestSuitOverviewDataKtTest.kt | 16 ++++++++++++-- 10 files changed, 74 insertions(+), 43 deletions(-) diff --git a/release_notes.md b/release_notes.md index 067d7ef867..be9f5f1c2d 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,6 @@ ## next (unreleased) - [#687](https://github.com/Flank/flank/pull/687) Debug message printed after every command. ([pawelpasterz](https://github.com/pawelpasterz)) +- [#684](https://github.com/Flank/flank/pull/684) Add overhead time to junit test case report. ([jan-gogo](https://github.com/jan-gogo)) - [#666](https://github.com/Flank/flank/pull/666) Use API instead of XML for result parsing for android. ([jan-gogo](https://github.com/jan-gogo)) - [#678](https://github.com/Flank/flank/pull/678) Skip Bugsnag initialization if user disabled gcloud analytics. ([pawelpasterz](https://github.com/pawelpasterz)) - [#672](https://github.com/Flank/flank/pull/672) Flank timeout feature. ([pawelpasterz](https://github.com/pawelpasterz)) diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt index c626323cdd..7b89c705d7 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt @@ -7,14 +7,20 @@ import ftl.reports.xml.model.JUnitTestCase internal fun createJUnitTestCases( testCases: List, - toolResultsStep: ToolResultsStep + toolResultsStep: ToolResultsStep, + overheadTime: Double ): List = testCases.map { testCase -> - createJUnitTestCase(testCase, toolResultsStep) + createJUnitTestCase( + testCase = testCase, + toolResultsStep = toolResultsStep, + overheadTime = overheadTime + ) } private fun createJUnitTestCase( testCase: TestCase, - toolResultsStep: ToolResultsStep + toolResultsStep: ToolResultsStep, + overheadTime: Double ): JUnitTestCase { val stackTraces = mapOf( testCase.status to testCase.stackTraces?.map(StackTrace::getException) @@ -22,7 +28,7 @@ private fun createJUnitTestCase( return JUnitTestCase( name = testCase.testCaseReference.name, classname = testCase.testCaseReference.className, - time = testCase.elapsedTime.format(), + time = (testCase.elapsedTime.millis() + overheadTime).format(), failures = stackTraces["failed"], errors = stackTraces["error"], // skipped = true is represented by null. skipped = false is "absent" diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt index bb5e9c05a7..5e5b78b6ca 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt @@ -3,24 +3,18 @@ package ftl.reports.api import com.google.api.services.toolresults.model.Step import ftl.reports.api.data.TestExecutionData import ftl.reports.api.data.TestSuiteOverviewData -import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestSuite internal fun List.createJUnitTestSuites() = map { data: TestExecutionData -> createJUnitTestSuite( data = data, - overview = data.createTestSuitOverviewData(), - testCases = createJUnitTestCases( - testCases = data.testCases, - toolResultsStep = data.testExecution.toolResultsStep - ) + overview = data.createTestSuitOverviewData() ) } private fun createJUnitTestSuite( data: TestExecutionData, - overview: TestSuiteOverviewData, - testCases: List + overview: TestSuiteOverviewData ) = JUnitTestSuite( name = data.step.testSuiteName(), timestamp = data.timestamp.asUnixTimestamp().formatUtcDate(), @@ -29,15 +23,15 @@ private fun createJUnitTestSuite( errors = overview.errors.toString(), skipped = overview.skipped.toString(), flakes = overview.flakes, - testcases = testCases.toMutableList(), - time = testCases.sumTime().format() // FIXME include also setup and teardown duration https://github.com/Flank/flank/issues/557 + testcases = createJUnitTestCases( + testCases = data.testCases, + toolResultsStep = data.testExecution.toolResultsStep, + overheadTime = overview.overheadTime + ).toMutableList(), + time = overview.elapsedTime.format() ) private fun Step.testSuiteName(): String { val map = dimensionValue.map { it.key to it.value }.toMap() return listOf(map["Model"], map["Version"], map["Locale"], map["Orientation"]).joinToString("-") } - -private fun List.sumTime(): Double = this - .map { it.time?.toDouble() ?: 0.0 } - .reduce { acc, d -> acc + d } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt index 1df7106864..63f19398af 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuitOverviewData.kt @@ -1,20 +1,33 @@ package ftl.reports.api import com.google.api.services.toolresults.model.TestCase +import com.google.api.services.toolresults.model.TestSuiteOverview import ftl.reports.api.data.TestExecutionData import ftl.reports.api.data.TestSuiteOverviewData internal fun TestExecutionData.createTestSuitOverviewData(): TestSuiteOverviewData { - val skipped = step.testExecutionStep.testSuiteOverviews.first().skippedCount ?: 0 + val overview: TestSuiteOverview = step.testExecutionStep.testSuiteOverviews.first() + val skipped: Int = overview.skippedCount ?: 0 return TestSuiteOverviewData( total = testCases.size + skipped, errors = testCases.countErrors(), failures = testCases.countFailures(), flakes = testCases.countFlakes(), - skipped = skipped + skipped = skipped, + elapsedTime = overview.elapsedTime.millis(), + overheadTime = getOverheadTime(overview, testCases) ) } private fun List.countErrors() = count { !it.flaky && it.status == "error" } private fun List.countFailures() = count { !it.flaky && it.status == "failed" } private fun List.countFlakes() = count { it.flaky } + +private fun getOverheadTime( + overview: TestSuiteOverview, + testCases: List +) = if (testCases.isEmpty()) + 0.0 else + (overview.elapsedTime.millis() - testCases.sumTime()) / testCases.size + +private fun List.sumTime(): Double = sumByDouble { it.elapsedTime.millis() } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt b/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt index 33fee7b8d2..b7fdf124e5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt @@ -12,10 +12,10 @@ internal fun Double.format(): String = "%.3f".format(Locale.ROOT, this) internal fun Int?.format() = (this ?: 0).toString() -internal fun Duration?.format(): String = +internal fun Duration?.millis(): Double = if (this == null) - "0.0" else - ((seconds ?: 0) + nanosToSeconds(nanos)).format() + 0.0 else + ((seconds ?: 0) + nanosToSeconds(nanos)) // manually divide to keep fractional precision private fun nanosToSeconds(nanos: Int?): Double = diff --git a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt index fb07567bca..961cebab9c 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/data/TestSuiteOverviewData.kt @@ -5,5 +5,7 @@ data class TestSuiteOverviewData( val errors: Int, val failures: Int, val flakes: Int, - val skipped: Int + val skipped: Int, + val elapsedTime: Double, + val overheadTime: Double ) diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 4470da94b7..90b7c146b5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -62,7 +62,11 @@ object ReportManager { return matchResult?.groupValues?.last().orEmpty() } - private fun processXmlFromFile(matrices: MatrixMap, args: IArgs, process: (file: File) -> JUnitTestResult): JUnitTestResult? { + private fun processXmlFromFile( + matrices: MatrixMap, + args: IArgs, + process: (file: File) -> JUnitTestResult + ): JUnitTestResult? { var mergedXml: JUnitTestResult? = null findXmlFiles(matrices, args).forEach { xmlFile -> @@ -132,13 +136,11 @@ object ReportManager { newResult: JUnitTestResult, args: IArgs, testShardChunks: ShardChunks - ): - List { + ): List { val oldDurations = Shard.createTestMethodDurationMap(oldResult, args) val newDurations = Shard.createTestMethodDurationMap(newResult, args) - val timeList = mutableListOf() - testShardChunks.forEachIndexed { index, testSuite -> + return testShardChunks.mapIndexed { index, testSuite -> var expectedTime = 0.0 var finalTime = 0.0 @@ -148,10 +150,8 @@ object ReportManager { } val timeDiff = (finalTime - expectedTime) - timeList.add(ShardEfficiency("Shard $index", expectedTime, finalTime, timeDiff)) + ShardEfficiency("Shard $index", expectedTime, finalTime, timeDiff) } - - return timeList } private fun printActual( @@ -176,7 +176,9 @@ object ReportManager { val oldTestResult = GcStorage.downloadJunitXml(args) - newTestResult.mergeTestTimes(oldTestResult) + if (args.useLegacyJUnitResult) { + newTestResult.mergeTestTimes(oldTestResult) + } if (oldTestResult != null) { printActual(oldTestResult, newTestResult, args, testShardChunks) diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt index 31a168cd59..df58eb811e 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt @@ -48,18 +48,18 @@ class CreateJUnitTestCaseKtTest { JUnitTestCase( name = "test1", classname = "TestClassName", - time = "1.100" + time = "2.200" ), JUnitTestCase( name = "test2", classname = "TestClassName", - time = "1.100", + time = "2.200", skipped = null ), JUnitTestCase( name = "test3", classname = "TestClassName", - time = "1.100", + time = "2.200", errors = listOf("exception") ).apply { webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test3" @@ -67,7 +67,7 @@ class CreateJUnitTestCaseKtTest { JUnitTestCase( name = "test4", classname = "TestClassName", - time = "1.100", + time = "2.200", failures = listOf("exception") ).apply { webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test4" @@ -75,7 +75,7 @@ class CreateJUnitTestCaseKtTest { JUnitTestCase( name = "test5", classname = "TestClassName", - time = "1.100", + time = "2.200", failures = listOf("exception") ).apply { webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test5" @@ -85,7 +85,8 @@ class CreateJUnitTestCaseKtTest { val actual = createJUnitTestCases( testCases = testCases, - toolResultsStep = toolResultsStep + toolResultsStep = toolResultsStep, + overheadTime = 1.1 ) // then diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt index 9a48d80520..449bc8e535 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt @@ -39,12 +39,12 @@ class CreateJUnitTestSuiteKtTest { // given every { any().createTestSuitOverviewData() - } returns TestSuiteOverviewData(1, 1, 1, 1, 1) + } returns TestSuiteOverviewData(1, 1, 1, 1, 1, 1.1, 1.1) val jUnitTestCase = JUnitTestCase(null, null, "1.1") every { - createJUnitTestCases(any(), any()) + createJUnitTestCases(any(), any(), any()) } returns listOf(jUnitTestCase) val testExecutionDataList = listOf( diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt index dce0530ba8..7da6cb8534 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuitOverviewDataKtTest.kt @@ -1,6 +1,7 @@ package ftl.reports.api import com.google.api.services.testing.model.TestExecution +import com.google.api.services.toolresults.model.Duration import com.google.api.services.toolresults.model.Step import com.google.api.services.toolresults.model.TestCase import com.google.api.services.toolresults.model.TestExecutionStep @@ -24,12 +25,21 @@ class CreateTestSuitOverviewDataKtTest { TestCase().apply { status = "error" }, TestCase().apply { status = "failed" }, TestCase().apply { flaky = true } - ), + ).apply { + forEach { + it.elapsedTime = Duration().apply { + seconds = 1 + } + } + }, step = Step().apply { testExecutionStep = TestExecutionStep().apply { testSuiteOverviews = listOf( TestSuiteOverview().apply { skippedCount = 1 + elapsedTime = Duration().apply { + seconds = 8 + } } ) } @@ -42,7 +52,9 @@ class CreateTestSuitOverviewDataKtTest { errors = 1, failures = 1, skipped = 1, - flakes = 1 + flakes = 1, + elapsedTime = 8.0, + overheadTime = 1.0 ) val actual = testExecutionData.createTestSuitOverviewData()