Skip to content

Commit

Permalink
feat: Merge test method duration for parameterized classes (#2062)
Browse files Browse the repository at this point in the history
Fixes #2052 

## Changes

* Adds to module `:tool:junit` function for merging test methods into test-class with accumulated duration.
* Merges test methods for given class names while loading previous durations.

## Test Plan
> How do we know the code works?

Build flank:
```shell
. .env
flankScripts assemble flank -d
```
Run the following command twice:
```shell
flank corellium test android run -c="./test_configs/flank-corellium.yml"
```
The second run should calculate durations for parameterized classes.
Check generated `android-shards.json` each parameterized class should have a duration different than the default (120).

## Checklist

- [x] Documented
- [x] Unit tested
  • Loading branch information
jan-goral authored Jul 9, 2021
1 parent d8a97ea commit 4ce447a
Show file tree
Hide file tree
Showing 7 changed files with 272 additions and 90 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,16 @@ import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir
import flank.corellium.domain.RunTestCorelliumAndroid.LoadPreviousDurations
import flank.corellium.domain.step
import flank.junit.calculateTestCaseDurations
import flank.junit.mergeTestCases

/**
* The step is searching result directory for JUnitReport.xml.
* Collected reports are used for calculating test cases durations.
*
* For test cases represented by their class, the duration will be based on summary durations from methods of this class.
* Basically, this is necessary for parameterized tests that are treated as class instead of method.
*
*
* require:
* * [RunTestCorelliumAndroid.Context.parseTestCasesFromApks]
*
Expand All @@ -21,14 +26,18 @@ internal fun RunTestCorelliumAndroid.Context.loadPreviousDurations() = step(Load
if (args.outputDir.startsWith(DefaultOutputDir.ROOT)) DefaultOutputDir.ROOT
else args.outputDir

val classCases: Set<String> =
testCases.values.flatten().filter { '#' !in it }.toSet()

copy(
previousDurations = junit.parseTestResults(directoryToScan)
.take(args.scanPreviousDurations).toList()
.apply { LoadPreviousDurations.Searching(size).out() }
.map { suite -> suite.mergeTestCases(classCases) }
.flatten()
.calculateTestCaseDurations()
.withDefault { previousDurations.getValue(it) }
.also { durations -> printStats(durations.keys).out() }
.apply { printStats(keys).out() }
)
}

Expand Down
7 changes: 7 additions & 0 deletions tool/junit/src/main/kotlin/flank/junit/JUnit.kt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement
import flank.junit.internal.calculateMedianDurations
import flank.junit.internal.mergeDurations
import flank.junit.internal.parseJUnitTestResults
import flank.junit.mapper.TimeSerializer
import flank.junit.mapper.mapToTestSuites
Expand Down Expand Up @@ -36,6 +37,12 @@ fun JUnit.Report.writeAsXml(writer: Writer) {
xmlPrettyWriter.writeValue(writer, this)
}

/**
* Merge [JUnit.TestResult] test methods by class names to accumulate duration and return them as classes.
*/
fun List<JUnit.TestResult>.mergeTestCases(byClasses: Set<String>): List<JUnit.TestResult> =
mergeDurations(byClasses)

/**
* Calculate associate full test cases names to calculated duration.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ internal fun List<JUnit.TestResult>.calculateMedianDurations(): Map<String, Long
.groupBy(Pair<String, Long>::first, Pair<String, Long>::second)
.mapValues { (_, durations) -> durations.median() }

private val JUnit.TestResult.fullName get() = "$className#$testName"
private val JUnit.TestResult.fullName get() = className + if (testName.isBlank()) "" else "#$testName"

private val JUnit.TestResult.duration get() = (endsAt - startAt)

Expand Down
14 changes: 14 additions & 0 deletions tool/junit/src/main/kotlin/flank/junit/internal/MergeDurations.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package flank.junit.internal

import flank.junit.JUnit

internal fun List<JUnit.TestResult>.mergeDurations(
forClasses: Set<String>
): List<JUnit.TestResult> =
if (isEmpty() || forClasses.isEmpty()) this // Nothing to merge.
else groupBy { method -> method.className in forClasses }.run { // separate test cases methods to merge.
getOrDefault(false, emptyList()) + getOrDefault(true, emptyList()) // Sum test cases methods with classes.
.groupBy { method -> method.className }.values // Group tests cases to merge by class name.
.map { methods -> methods.sortedBy(JUnit.TestResult::startAt) } // Ensure correct order.
.map { methods -> methods.first().copy(testName = "", endsAt = methods.last().endsAt) } // Merge into class.
}
8 changes: 6 additions & 2 deletions tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,17 @@ internal fun List<JUnit.TestResult>.mapToTestSuites(): List<JUnit.Suite> = this

internal fun List<JUnit.Suite>.mapToTestResults(): List<JUnit.TestResult> =
flatMap { suite ->
var startAt: Long = JUnit.dateFormat.parse(suite.timestamp).time
var endAt: Long = startAt
suite.testcases.map { case ->
startAt = endAt
endAt = startAt + (case.time * 1000).toLong()
JUnit.TestResult(
suiteName = suite.name,
testName = case.name,
className = case.classname,
startAt = 0,
endsAt = (case.time * 1000).toLong(),
startAt = startAt,
endsAt = endAt,
status = when {
case.error.isNotEmpty() -> JUnit.TestResult.Status.Error
case.failure.isNotEmpty() -> JUnit.TestResult.Status.Failed
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package flank.junit.internal

import flank.junit.JUnit
import flank.junit.mergeTestCases
import org.junit.Assert
import org.junit.Test

class MergeDurationsKtTest {

/**
* Only test case methods of classes specified in set will be merged into classes.
*/
@Test
fun mixed() {
// given
val classes = setOf("b", "d")

val results = listOf(
result("a", "a", 0, 10),
result("b", "b[0]", 10, 110),
result("b", "b[1]", 110, 120),
result("b", "b[2]", 120, 140),
result("c", "c", 140, 440),
result("d", "d[name: a]", 440, 470),
result("d", "d[name: b]", 470, 500),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 140, 440),
result("b", "", 10, 140),
result("d", "", 440, 500),
)

// when
val actual = results.mergeTestCases(classes)

// then
Assert.assertEquals(expected, actual)
}

/**
* All test case methods will be merged into test case classes.
*/
@Test
fun allToMerge() {
// given
val classes = setOf("a", "b", "c", "d")

val results = listOf(
result("a", "a", 0, 10),
result("b", "b[0]", 10, 110),
result("b", "b[1]", 110, 120),
result("b", "b[2]", 120, 140),
result("c", "c", 140, 440),
result("d", "d[name: a]", 440, 470),
result("d", "d[name: b]", 470, 500),
)

val expected = listOf(
result("a", "", 0, 10),
result("b", "", 10, 140),
result("c", "", 140, 440),
result("d", "", 440, 500),
)

// when
val actual = results.mergeTestCases(classes)

// then
Assert.assertEquals(expected, actual)
}

/**
* No test cases will be merged, because of missing class name.
*/
@Test
fun missingClassName() {
// given
val results = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

// when
val actual = results.mergeTestCases(setOf("e"))

// then
Assert.assertEquals(expected, actual)
}

/**
* No test cases will be merged, because of empty set.
*/
@Test
fun emptySet() {
// given
val results = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

val expected = listOf(
result("a", "a", 0, 10),
result("c", "c", 20, 60),
)

// when
val actual = results.mergeTestCases(setOf())

// then
Assert.assertEquals(expected, actual)
}
}

private fun result(
className: String,
name: String,
start: Long,
end: Long,
) = JUnit.TestResult(
className = className,
testName = name,
startAt = start,
endsAt = end,
stack = emptyList(),
status = JUnit.TestResult.Status.Passed,
suiteName = "",
)
Loading

0 comments on commit 4ce447a

Please sign in to comment.