diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt index be8ea976a2..358dd8fec3 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt @@ -5,11 +5,16 @@ import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir import flank.corellium.domain.RunTestCorelliumAndroid.LoadPreviousDurations import flank.corellium.domain.step import flank.junit.calculateTestCaseDurations +import flank.junit.mergeTestCases /** * The step is searching result directory for JUnitReport.xml. * Collected reports are used for calculating test cases durations. * + * For test cases represented by their class, the duration will be based on summary durations from methods of this class. + * Basically, this is necessary for parameterized tests that are treated as class instead of method. + * + * * require: * * [RunTestCorelliumAndroid.Context.parseTestCasesFromApks] * @@ -21,14 +26,18 @@ internal fun RunTestCorelliumAndroid.Context.loadPreviousDurations() = step(Load if (args.outputDir.startsWith(DefaultOutputDir.ROOT)) DefaultOutputDir.ROOT else args.outputDir + val classCases: Set = + testCases.values.flatten().filter { '#' !in it }.toSet() + copy( previousDurations = junit.parseTestResults(directoryToScan) .take(args.scanPreviousDurations).toList() .apply { LoadPreviousDurations.Searching(size).out() } + .map { suite -> suite.mergeTestCases(classCases) } .flatten() .calculateTestCaseDurations() .withDefault { previousDurations.getValue(it) } - .also { durations -> printStats(durations.keys).out() } + .apply { printStats(keys).out() } ) } diff --git a/tool/junit/src/main/kotlin/flank/junit/JUnit.kt b/tool/junit/src/main/kotlin/flank/junit/JUnit.kt index f609463f04..888cbb4cff 100644 --- a/tool/junit/src/main/kotlin/flank/junit/JUnit.kt +++ b/tool/junit/src/main/kotlin/flank/junit/JUnit.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement import flank.junit.internal.calculateMedianDurations +import flank.junit.internal.mergeDurations import flank.junit.internal.parseJUnitTestResults import flank.junit.mapper.TimeSerializer import flank.junit.mapper.mapToTestSuites @@ -36,6 +37,12 @@ fun JUnit.Report.writeAsXml(writer: Writer) { xmlPrettyWriter.writeValue(writer, this) } +/** + * Merge [JUnit.TestResult] test methods by class names to accumulate duration and return them as classes. + */ +fun List.mergeTestCases(byClasses: Set): List = + mergeDurations(byClasses) + /** * Calculate associate full test cases names to calculated duration. */ diff --git a/tool/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt b/tool/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt index 004036d891..0a435f27ce 100644 --- a/tool/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt +++ b/tool/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt @@ -10,7 +10,7 @@ internal fun List.calculateMedianDurations(): Map::first, Pair::second) .mapValues { (_, durations) -> durations.median() } -private val JUnit.TestResult.fullName get() = "$className#$testName" +private val JUnit.TestResult.fullName get() = className + if (testName.isBlank()) "" else "#$testName" private val JUnit.TestResult.duration get() = (endsAt - startAt) diff --git a/tool/junit/src/main/kotlin/flank/junit/internal/MergeDurations.kt b/tool/junit/src/main/kotlin/flank/junit/internal/MergeDurations.kt new file mode 100644 index 0000000000..ab0d39eed4 --- /dev/null +++ b/tool/junit/src/main/kotlin/flank/junit/internal/MergeDurations.kt @@ -0,0 +1,14 @@ +package flank.junit.internal + +import flank.junit.JUnit + +internal fun List.mergeDurations( + forClasses: Set +): List = + if (isEmpty() || forClasses.isEmpty()) this // Nothing to merge. + else groupBy { method -> method.className in forClasses }.run { // separate test cases methods to merge. + getOrDefault(false, emptyList()) + getOrDefault(true, emptyList()) // Sum test cases methods with classes. + .groupBy { method -> method.className }.values // Group tests cases to merge by class name. + .map { methods -> methods.sortedBy(JUnit.TestResult::startAt) } // Ensure correct order. + .map { methods -> methods.first().copy(testName = "", endsAt = methods.last().endsAt) } // Merge into class. + } diff --git a/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt b/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt index 7fd8fa2f24..b26d67736a 100644 --- a/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt +++ b/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt @@ -31,13 +31,17 @@ internal fun List.mapToTestSuites(): List = this internal fun List.mapToTestResults(): List = flatMap { suite -> + var startAt: Long = JUnit.dateFormat.parse(suite.timestamp).time + var endAt: Long = startAt suite.testcases.map { case -> + startAt = endAt + endAt = startAt + (case.time * 1000).toLong() JUnit.TestResult( suiteName = suite.name, testName = case.name, className = case.classname, - startAt = 0, - endsAt = (case.time * 1000).toLong(), + startAt = startAt, + endsAt = endAt, status = when { case.error.isNotEmpty() -> JUnit.TestResult.Status.Error case.failure.isNotEmpty() -> JUnit.TestResult.Status.Failed diff --git a/tool/junit/src/test/kotlin/flank/junit/internal/MergeDurationsKtTest.kt b/tool/junit/src/test/kotlin/flank/junit/internal/MergeDurationsKtTest.kt new file mode 100644 index 0000000000..cc828d107c --- /dev/null +++ b/tool/junit/src/test/kotlin/flank/junit/internal/MergeDurationsKtTest.kt @@ -0,0 +1,134 @@ +package flank.junit.internal + +import flank.junit.JUnit +import flank.junit.mergeTestCases +import org.junit.Assert +import org.junit.Test + +class MergeDurationsKtTest { + + /** + * Only test case methods of classes specified in set will be merged into classes. + */ + @Test + fun mixed() { + // given + val classes = setOf("b", "d") + + val results = listOf( + result("a", "a", 0, 10), + result("b", "b[0]", 10, 110), + result("b", "b[1]", 110, 120), + result("b", "b[2]", 120, 140), + result("c", "c", 140, 440), + result("d", "d[name: a]", 440, 470), + result("d", "d[name: b]", 470, 500), + ) + + val expected = listOf( + result("a", "a", 0, 10), + result("c", "c", 140, 440), + result("b", "", 10, 140), + result("d", "", 440, 500), + ) + + // when + val actual = results.mergeTestCases(classes) + + // then + Assert.assertEquals(expected, actual) + } + + /** + * All test case methods will be merged into test case classes. + */ + @Test + fun allToMerge() { + // given + val classes = setOf("a", "b", "c", "d") + + val results = listOf( + result("a", "a", 0, 10), + result("b", "b[0]", 10, 110), + result("b", "b[1]", 110, 120), + result("b", "b[2]", 120, 140), + result("c", "c", 140, 440), + result("d", "d[name: a]", 440, 470), + result("d", "d[name: b]", 470, 500), + ) + + val expected = listOf( + result("a", "", 0, 10), + result("b", "", 10, 140), + result("c", "", 140, 440), + result("d", "", 440, 500), + ) + + // when + val actual = results.mergeTestCases(classes) + + // then + Assert.assertEquals(expected, actual) + } + + /** + * No test cases will be merged, because of missing class name. + */ + @Test + fun missingClassName() { + // given + val results = listOf( + result("a", "a", 0, 10), + result("c", "c", 20, 60), + ) + + val expected = listOf( + result("a", "a", 0, 10), + result("c", "c", 20, 60), + ) + + // when + val actual = results.mergeTestCases(setOf("e")) + + // then + Assert.assertEquals(expected, actual) + } + + /** + * No test cases will be merged, because of empty set. + */ + @Test + fun emptySet() { + // given + val results = listOf( + result("a", "a", 0, 10), + result("c", "c", 20, 60), + ) + + val expected = listOf( + result("a", "a", 0, 10), + result("c", "c", 20, 60), + ) + + // when + val actual = results.mergeTestCases(setOf()) + + // then + Assert.assertEquals(expected, actual) + } +} + +private fun result( + className: String, + name: String, + start: Long, + end: Long, +) = JUnit.TestResult( + className = className, + testName = name, + startAt = start, + endsAt = end, + stack = emptyList(), + status = JUnit.TestResult.Status.Passed, + suiteName = "", +) diff --git a/tool/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt b/tool/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt index 42cbbf20e1..4cd0522acf 100644 --- a/tool/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt +++ b/tool/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt @@ -6,99 +6,113 @@ import org.junit.Test class StructuralKtTest { - @Test - fun mapToTestSuitesTest() { - val expected = listOf( - JUnit.Suite( - name = "suite1", - tests = 3, - failures = 1, - errors = 1, - skipped = 0, - time = 8.0, - timestamp = JUnit.dateFormat.format(1_000), - testcases = listOf( - JUnit.Case( - name = "test1", - classname = "test1.Test1", - time = 4.0, - error = listOf("some error") - ), - JUnit.Case( - name = "test2", - classname = "test1.Test1", - time = 2.0, - failure = listOf("some assertion failed"), - ), - JUnit.Case( - name = "test1", - classname = "test1.Test2", - time = 3.0, - ) + private val suites = listOf( + JUnit.Suite( + name = "suite1", + tests = 3, + failures = 1, + errors = 1, + skipped = 0, + time = 9.0, + timestamp = JUnit.dateFormat.format(1_000), + testcases = listOf( + JUnit.Case( + name = "test1", + classname = "test1.Test1", + time = 4.0, + error = listOf("some error") + ), + JUnit.Case( + name = "test2", + classname = "test1.Test1", + time = 2.0, + failure = listOf("some assertion failed"), + ), + JUnit.Case( + name = "test1", + classname = "test1.Test2", + time = 3.0, ) - ), - JUnit.Suite( - name = "suite2", - tests = 1, - failures = 0, - errors = 0, - skipped = 1, - time = 0.0, - timestamp = JUnit.dateFormat.format(0), - testcases = listOf( - JUnit.Case( - name = "test1", - classname = "test1.Test1", - time = 0.0, - skipped = null, - ), - ) - ), - ) + ) + ), + JUnit.Suite( + name = "suite2", + tests = 1, + failures = 0, + errors = 0, + skipped = 1, + time = 0.0, + timestamp = JUnit.dateFormat.format(0), + testcases = listOf( + JUnit.Case( + name = "test1", + classname = "test1.Test1", + time = 0.0, + skipped = null, + ), + ) + ), + ) - val testCases = listOf( - JUnit.TestResult( - testName = "test1", - className = "test1.Test1", - suiteName = "suite1", - startAt = 1_000, - endsAt = 5_000, - status = JUnit.TestResult.Status.Error, - stack = listOf("some error") - ), - JUnit.TestResult( - testName = "test2", - className = "test1.Test1", - suiteName = "suite1", - startAt = 6_000, - endsAt = 8_000, - status = JUnit.TestResult.Status.Failed, - stack = listOf("some assertion failed") - ), - JUnit.TestResult( - testName = "test1", - className = "test1.Test2", - suiteName = "suite1", - startAt = 6_000, - endsAt = 9_000, - status = JUnit.TestResult.Status.Passed, - stack = emptyList() - ), - JUnit.TestResult( - testName = "test1", - className = "test1.Test1", - suiteName = "suite2", - startAt = 0, - endsAt = 0, - status = JUnit.TestResult.Status.Skipped, - stack = emptyList() - ), - ) + val testCases = listOf( + JUnit.TestResult( + testName = "test1", + className = "test1.Test1", + suiteName = "suite1", + startAt = 1_000, + endsAt = 5_000, + status = JUnit.TestResult.Status.Error, + stack = listOf("some error") + ), + JUnit.TestResult( + testName = "test2", + className = "test1.Test1", + suiteName = "suite1", + startAt = 5_000, + endsAt = 7_000, + status = JUnit.TestResult.Status.Failed, + stack = listOf("some assertion failed") + ), + JUnit.TestResult( + testName = "test1", + className = "test1.Test2", + suiteName = "suite1", + startAt = 7_000, + endsAt = 10_000, + status = JUnit.TestResult.Status.Passed, + stack = emptyList() + ), + JUnit.TestResult( + testName = "test1", + className = "test1.Test1", + suiteName = "suite2", + startAt = 0, + endsAt = 0, + status = JUnit.TestResult.Status.Skipped, + stack = emptyList() + ), + ) + + @Test + fun mapToTestSuitesTest() { + val expected = suites val actual = testCases.mapToTestSuites() + println(xmlPrettyWriter.writeValueAsString(testCases)) println(xmlPrettyWriter.writeValueAsString(actual)) assertEquals(expected, actual) } + + @Test + fun mapToTestResultsTest() { + val expected = testCases + + val actual = suites.mapToTestResults() + + println(xmlPrettyWriter.writeValueAsString(suites)) + + assertEquals(expected, actual) + } }