Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Merge test method duration for parameterized classes #2062

Merged
merged 7 commits into from
Jul 9, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,16 @@ import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir
import flank.corellium.domain.RunTestCorelliumAndroid.LoadPreviousDurations
import flank.corellium.domain.step
import flank.junit.calculateTestCaseDurations
import flank.junit.mergeTestCases

/**
* The step is searching result directory for JUnitReport.xml.
* Collected reports are used for calculating test cases durations.
*
* For test cases represented by their class, the duration will be based on summary durations from methods of this class.
* Basically, this is necessary for parameterized tests that are treated as class instead of method.
*
*
* require:
* * [RunTestCorelliumAndroid.Context.parseTestCasesFromApks]
*
Expand All @@ -21,14 +26,18 @@ internal fun RunTestCorelliumAndroid.Context.loadPreviousDurations() = step(Load
if (args.outputDir.startsWith(DefaultOutputDir.ROOT)) DefaultOutputDir.ROOT
else args.outputDir

val classCases: Set<String> =
testCases.values.flatten().filter { '#' !in it }.toSet()

copy(
previousDurations = junit.parseTestResults(directoryToScan)
.take(args.scanPreviousDurations).toList()
.apply { LoadPreviousDurations.Searching(size).out() }
.map { suite -> suite.mergeTestCases(classCases) }
.flatten()
.calculateTestCaseDurations()
.withDefault { previousDurations.getValue(it) }
.also { durations -> printStats(durations.keys).out() }
.apply { printStats(keys).out() }
)
}

Expand Down
7 changes: 7 additions & 0 deletions tool/junit/src/main/kotlin/flank/junit/JUnit.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement
import flank.junit.internal.calculateMedianDurations
import flank.junit.internal.mergeDurations
import flank.junit.internal.parseJUnitTestResults
import flank.junit.mapper.TimeSerializer
import flank.junit.mapper.mapToTestSuites
Expand Down Expand Up @@ -36,6 +37,12 @@ fun JUnit.Report.writeAsXml(writer: Writer) {
xmlPrettyWriter.writeValue(writer, this)
}

/**
* Merge [JUnit.TestResult] test methods by class names to accumulate duration and return them as classes.
*/
fun List<JUnit.TestResult>.mergeTestCases(byClasses: Set<String>): List<JUnit.TestResult> =
mergeDurations(byClasses)

/**
* Calculate associate full test cases names to calculated duration.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ internal fun List<JUnit.TestResult>.calculateMedianDurations(): Map<String, Long
.groupBy(Pair<String, Long>::first, Pair<String, Long>::second)
.mapValues { (_, durations) -> durations.median() }

private val JUnit.TestResult.fullName get() = "$className#$testName"
private val JUnit.TestResult.fullName get() = className + if (testName.isBlank()) "" else "#$testName"

private val JUnit.TestResult.duration get() = (endsAt - startAt)

Expand Down
14 changes: 14 additions & 0 deletions tool/junit/src/main/kotlin/flank/junit/internal/MergeDurations.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package flank.junit.internal

import flank.junit.JUnit

internal fun List<JUnit.TestResult>.mergeDurations(
forClasses: Set<String>
): List<JUnit.TestResult> =
if (isEmpty() || forClasses.isEmpty()) this // Nothing to merge.
else groupBy { method -> method.className in forClasses }.run { // separate test cases methods to merge.
getOrDefault(false, emptyList()) + getOrDefault(true, emptyList()) // Sum test cases methods with classes.
.groupBy { method -> method.className }.values // Group tests cases to merge by class name.
.map { methods -> methods.sortedBy(JUnit.TestResult::startAt) } // Ensure correct order.
.map { methods -> methods.first().copy(testName = "", endsAt = methods.last().endsAt) } // Merge into class.
}
8 changes: 6 additions & 2 deletions tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,17 @@ internal fun List<JUnit.TestResult>.mapToTestSuites(): List<JUnit.Suite> = this

internal fun List<JUnit.Suite>.mapToTestResults(): List<JUnit.TestResult> =
flatMap { suite ->
var startAt: Long = JUnit.dateFormat.parse(suite.timestamp).time
var endAt: Long = startAt
suite.testcases.map { case ->
startAt = endAt
endAt = startAt + (case.time * 1000).toLong()
JUnit.TestResult(
suiteName = suite.name,
testName = case.name,
className = case.classname,
startAt = 0,
endsAt = (case.time * 1000).toLong(),
startAt = startAt,
endsAt = endAt,
status = when {
case.error.isNotEmpty() -> JUnit.TestResult.Status.Error
case.failure.isNotEmpty() -> JUnit.TestResult.Status.Failed
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package flank.junit.internal

import flank.junit.JUnit
import flank.junit.mergeTestCases
import org.junit.Assert
import org.junit.Test

class MergeDurationsKtTest {

/**
* Only test case methods of classes specified in set will be merged into classes.
*/
@Test
fun mixed() {
// given
val classes = setOf("b", "d")

val results = listOf(
result("a", "a", 0, 10),
result("b", "b[0]", 10, 110),
result("b", "b[1]", 110, 120),
result("b", "b[2]", 120, 140),
result("c", "c", 140, 440),
result("d", "d[name: a]", 440, 470),
result("d", "d[name: b]", 470, 500),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 140, 440),
result("b", "", 10, 140),
result("d", "", 440, 500),
)

// when
val actual = results.mergeTestCases(classes)

// then
Assert.assertEquals(expected, actual)
}

/**
* All test case methods will be merged into test case classes.
*/
@Test
fun allToMerge() {
// given
val classes = setOf("a", "b", "c", "d")

val results = listOf(
result("a", "a", 0, 10),
result("b", "b[0]", 10, 110),
result("b", "b[1]", 110, 120),
result("b", "b[2]", 120, 140),
result("c", "c", 140, 440),
result("d", "d[name: a]", 440, 470),
result("d", "d[name: b]", 470, 500),
)

val expected = listOf(
result("a", "", 0, 10),
result("b", "", 10, 140),
result("c", "", 140, 440),
result("d", "", 440, 500),
)

// when
val actual = results.mergeTestCases(classes)

// then
Assert.assertEquals(expected, actual)
}

/**
* No test cases will be merged, because of missing class name.
*/
@Test
fun missingClassName() {
// given
val results = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

// when
val actual = results.mergeTestCases(setOf("e"))

// then
Assert.assertEquals(expected, actual)
}

/**
* No test cases will be merged, because of empty set.
*/
@Test
fun emptySet() {
// given
val results = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

// when
val actual = results.mergeTestCases(setOf())

// then
Assert.assertEquals(expected, actual)
}
}

private fun result(
className: String,
name: String,
start: Long,
end: Long,
) = JUnit.TestResult(
className = className,
testName = name,
startAt = start,
endsAt = end,
stack = emptyList(),
status = JUnit.TestResult.Status.Passed,
suiteName = "",
)
Loading