diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt index 9ef8876ead..7b59b67b3d 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt @@ -150,6 +150,27 @@ class TestAndroidCommand : ) @set:JsonProperty("num-flaky-test-attempts") var flakyTestAttempts: Int? by data + + @set:JsonProperty("junit-report-config") + var junitReport: Map>? by data + + @CommandLine.Option( + names = ["--junit-report-config"], + split = ";", + description = [ + "A map of name suffixes related to set of result types required to include in custom junit report. " + + "As results, this option will generate additional amount of junit reports named `JUnitReport-\$suffix.xml`." + + "Available result types to include are: [Skipped, Passed, Failed, Flaky]." + + "Default value is `--junit-report-config=failures=Failed,Flaky;`" + ] + ) + fun setJUnitReport(map: Map) { + junitReport = map.mapValues { (_, types) -> + types.split(",") + .map { type -> type.lowercase().replaceFirstChar(Char::uppercaseChar) } + .map(Args.Report.JUnit.Type::valueOf).toSet() + } + } } @CommandLine.Option( diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt index daa6551d79..84c861eaef 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt @@ -18,5 +18,6 @@ internal val args = Args from setOf(Config) using context { gpuAcceleration = config.gpuAcceleration!!, scanPreviousDurations = config.scanPreviousDurations!!, flakyTestsAttempts = config.flakyTestAttempts!!, + junitReport = config.junitReport!! ) } diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt index 20dcbb9d66..4c966d1911 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt @@ -30,6 +30,7 @@ private operator fun Config.plusAssign(args: TestAndroid.Args) { gpuAcceleration = args.gpuAcceleration scanPreviousDurations = args.scanPreviousDurations flakyTestAttempts = args.flakyTestsAttempts + junitReport = args.junitReport } internal fun yamlConfig(path: String?): Config = diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt index 1934414d59..e3f251e7cb 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt @@ -40,6 +40,7 @@ class ArgsKtTest { gpuAcceleration = gpuAcceleration!!, scanPreviousDurations = scanPreviousDurations!!, flakyTestsAttempts = flakyTestAttempts!!, + junitReport = junitReport!!, ) } diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt index 23a41e725e..98743415fe 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt @@ -37,7 +37,8 @@ class ConfigKtTest { "--obfuscate=$obfuscate", "--gpu-acceleration=$gpuAcceleration", "--scan-previous-durations=$scanPreviousDurations", - "--num-flaky-test-attempts=$flakyTestAttempts" + "--num-flaky-test-attempts=$flakyTestAttempts", + "--junit-report-config=test1=Passed;test2=FAILED,flaky" ) } @@ -62,6 +63,9 @@ class ConfigKtTest { gpu-acceleration: $gpuAcceleration scan-previous-durations: $scanPreviousDurations num-flaky-test-attempts: $flakyTestAttempts + junit-report-config: + test1: [Passed] + test2: [FAILED, flaky] """.trimIndent() } diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt index 0f14a9c775..f398aa7782 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt @@ -2,6 +2,9 @@ package flank.corellium.cli.test.android.task import flank.corellium.cli.TestAndroidCommand import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Failed +import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Flaky +import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Passed /** * Apply test values to config. Each value should be different than default. @@ -34,4 +37,8 @@ fun TestAndroidCommand.Config.applyTestValues() = apply { gpuAcceleration = false scanPreviousDurations = 123 flakyTestAttempts = Int.MAX_VALUE + junitReport = mapOf( + "test1" to setOf(Passed), + "test2" to setOf(Failed, Flaky), + ) } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt index b27137d7d0..9c66700076 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt @@ -27,6 +27,7 @@ import flank.corellium.domain.test.android.task.loadPreviousDurations import flank.corellium.domain.test.android.task.parseApksInfo import flank.corellium.domain.test.android.task.parseTestCasesFromApks import flank.corellium.domain.test.android.task.prepareShards +import flank.corellium.domain.test.android.task.processResults import flank.exection.parallel.Parallel import flank.exection.parallel.type import flank.instrument.log.Instrument @@ -67,6 +68,7 @@ object TestAndroid { val gpuAcceleration: Boolean = true, val scanPreviousDurations: Int = 10, val flakyTestsAttempts: Int = 0, + val junitReport: JUnitReportConfig = Report.JUnit.Default, ) { companion object : Parallel.Type { @@ -78,7 +80,7 @@ object TestAndroid { /** * Default output directory scheme. * - * @property new Directory name in format: `results/corellium/android/yyyy-MM-dd_HH-mm-ss-SSS`. + * @property new A directory name in format: `results/corellium/android/yyyy-MM-dd_HH-mm-ss-SSS`. */ object DefaultOutputDir { internal const val ROOT = "results/corellium/android/" @@ -111,6 +113,19 @@ object TestAndroid { override val path: String ) : Apk() } + + /** + * Report configuration file. + */ + object Report { + object JUnit { + enum class Type { Skipped, Passed, Failed, Flaky } + + val Default = mapOf( + "failures" to setOf(Type.Failed, Type.Flaky) + ) + } + } } // Context @@ -132,7 +147,8 @@ object TestAndroid { * @property dispatch Channel for dispatching test shards to execute. * @property devices Channel for providing devices that are available and ready to use. * @property ids the ids of corellium device instances. - * @property testResult Execution results. + * @property rawResults Execution results. + * @property processResults Results processed according to [Args.junitReport] configuration. */ internal class Context : Parallel.Context() { val api by !type() @@ -147,7 +163,8 @@ object TestAndroid { val dispatch: Channel by -Dispatch.Shards val devices: Channel by -AvailableDevices val ids: List by -InvokeDevices - val testResult: List by -ExecuteTests + val rawResults: List by -ExecuteTests + val processedResults: Map> by -ProcessedResults } internal val context = Parallel.Function(::Context) @@ -226,6 +243,8 @@ object TestAndroid { } object ReleaseDevice : Parallel.Type + object ProcessedResults : Parallel.Type>> + object CleanUp : Parallel.Type object GenerateReport : Parallel.Type object CompleteTests : Parallel.Type @@ -292,6 +311,7 @@ object TestAndroid { val id: String, val data: Dispatch.Data, val value: List, + val flakes: Set = emptySet(), ) internal val execute by lazy { @@ -307,24 +327,34 @@ object TestAndroid { // Evaluate lazy to avoid strange NullPointerException. val execute by lazy { + // Keep alphabetic order. setOf( context.validate, authorize, + availableDevices, createOutputDir, + dispatchFailedTests, dispatchShards, dispatchTests, - dispatchFailedTests, dumpShards, executeTestQueue, finish, generateReport, initResultsChannel, - availableDevices, invokeDevices, loadPreviousDurations, parseApksInfo, parseTestCasesFromApks, prepareShards, + processResults, ) } } + +/** + * JUnit report configuration. + * + * key - suffix that will be added to [JUnit.REPORT_FILE_NAME] for creating custom JUnitReport file name. + * value - set of required results to include in report. + */ +typealias JUnitReportConfig = Map> diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt index 098a0c88b0..0da1da6792 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt @@ -24,12 +24,9 @@ internal val dispatchFailedTests = Dispatch.Failed from setOf( var index = 0 results.consumeEach { result -> if (result.status is Instrument.Status) { - if (result.status.code in errorCodes) { - val shard = result.shard - .reduceTo(result.status.details.fullTestName) - val attempt = runs - .getOrPut(shard, counter) - .getAndIncrement() + if (result.status.code in Instrument.Code.errors) { + val shard = result.shard.reduceTo(result.status.name) + val attempt = runs.getOrPut(shard, counter).getAndIncrement() if (attempt < args.flakyTestsAttempts) dispatch.send( Dispatch.Data( @@ -44,14 +41,6 @@ internal val dispatchFailedTests = Dispatch.Failed from setOf( runs.mapValues { (_, value) -> value.get() } } -/** - * Set of [Instrument] error codes. - */ -private val errorCodes = setOf( - Instrument.Code.FAILED, - Instrument.Code.EXCEPTION, -) - /** * Creates new [InstanceShard] that contains only one test basing on the given [name]. * @@ -73,5 +62,3 @@ private fun InstanceShard.reduceTo( }.takeIf { it.isNotEmpty() }?.let { test.copy(cases = it) } }.takeIf { it.isNotEmpty() }?.let { app.copy(tests = it) } } - -private val Instrument.Status.Details.fullTestName get() = "$className#$testName" diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt index 87712c49be..fdb6f13384 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt @@ -6,11 +6,13 @@ import flank.corellium.domain.TestAndroid.Dispatch import flank.corellium.domain.TestAndroid.ExecuteTests import flank.corellium.domain.TestAndroid.GenerateReport import flank.corellium.domain.TestAndroid.OutputDir +import flank.corellium.domain.TestAndroid.ProcessedResults import flank.corellium.domain.TestAndroid.context import flank.exection.parallel.from import flank.exection.parallel.using import flank.instrument.log.Instrument import flank.junit.JUnit +import flank.junit.JUnit.REPORT_FILE_NAME import flank.junit.generateJUnitReport import flank.junit.writeAsXml import java.io.File @@ -21,10 +23,24 @@ import java.io.File */ internal val generateReport = GenerateReport from setOf( ExecuteTests, - OutputDir + ProcessedResults, + OutputDir, ) using context { - val file = File(args.outputDir, JUnit.REPORT_FILE_NAME) - testResult + // Generate default junit report from raw results. + generateReport(rawResults, REPORT_FILE_NAME) + + // Generate junit reports from processed results. + processedResults.forEach { (suffix, results) -> + generateReport(results, REPORT_FILE_NAME.replace(".", "_$suffix.")) + } +} + +private fun TestAndroid.Context.generateReport( + results: List, + fileName: String +) { + val file = File(args.outputDir, fileName) + results .prepareInputForJUnit() .generateJUnitReport() .writeAsXml(file.bufferedWriter()) @@ -49,6 +65,7 @@ private fun List.prepareInputForJUnit(): List = startAt = status.startTime, endsAt = status.endTime, stack = listOfNotNull(status.details.stack), + flaky = status.name in result.flakes, status = when (status.code) { Instrument.Code.PASSED -> JUnit.TestResult.Status.Passed Instrument.Code.FAILED -> JUnit.TestResult.Status.Failed diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ProcessResults.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ProcessResults.kt new file mode 100644 index 0000000000..8a958a7219 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ProcessResults.kt @@ -0,0 +1,77 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.Args.Report.JUnit +import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.ProcessedResults +import flank.corellium.domain.TestAndroid.context +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.instrument.log.Instrument +import flank.instrument.log.Instrument.Code.PASSED +import flank.instrument.log.Instrument.Code.SKIPPED +import flank.instrument.log.Instrument.Code.errors + +/** + * Process raw test results according to [TestAndroid.Args.Report] requirements. + */ +val processResults = ProcessedResults from setOf( + ExecuteTests, +) using context { + val codes = rawResults.mapTestCodes() + val shards = rawResults.filter { result -> result.data.type == Dispatch.Type.Shard } + + args.junitReport.mapValues { (_, types) -> + shards.filter(codes.testNamesBy(types)).run { + if (JUnit.Type.Flaky !in types) this + else update(codes.testNamesBy(JUnit.Type.Flaky)) + } + } +} + +private typealias TestResults = List +private typealias TestCodes = Map> + +private fun TestResults.mapTestCodes(): TestCodes = this + .flatMap { result -> + result.value + .filterIsInstance() + .map { status -> status.run { name to code } } + } + .groupBy( + keySelector = { (name, _) -> name }, + valueTransform = { (_, codes) -> codes } + ) + +private fun TestCodes.testNamesBy(types: Set): Set = + types.flatMap { testNamesBy(it) }.toSet() + +private fun TestCodes.testNamesBy(type: JUnit.Type): Set { + fun Map.keysByValues(predicate: V.() -> Boolean): Set = filterValues(predicate).keys + return when (type) { + JUnit.Type.Skipped -> keysByValues { all { code -> code == SKIPPED } } + JUnit.Type.Passed -> keysByValues { all { code -> code == PASSED } } + JUnit.Type.Failed -> keysByValues { all { code -> code in errors } } + JUnit.Type.Flaky -> keysByValues { groupBy { code -> code in errors }.size == 2 } + } +} + +private fun TestResults.filter(testNames: Set): TestResults = map { result -> + result.copy( + value = result.value + .filterIsInstance() + .filter { status -> status.name in testNames } + ) +} + +private fun TestResults.update(flakyTests: Set): TestResults = + map { result -> + result.copy( + flakes = result.value + .filterIsInstance() + .map(Instrument.Status::name) + .filter(flakyTests::contains) + .toSet() + ) + } diff --git a/test_configs/flank-corellium-many.yml b/test_configs/flank-corellium-many.yml index 37b1895d79..d18c1ae16c 100644 --- a/test_configs/flank-corellium-many.yml +++ b/test_configs/flank-corellium-many.yml @@ -11,4 +11,9 @@ apks: max-test-shards: 6 gpu-acceleration: false scan-previous-durations: 3 -num-flaky-test-attempts: 3 +num-flaky-test-attempts: 6 + +junit-report-config: + skipped: [Skipped] + passed: [Passed] + failures: [Failed, Flaky] diff --git a/test_configs/flank-corellium.yml b/test_configs/flank-corellium.yml index 6a61b2c8e5..726538ec4a 100644 --- a/test_configs/flank-corellium.yml +++ b/test_configs/flank-corellium.yml @@ -11,3 +11,4 @@ apks: max-test-shards: 3 gpu-acceleration: false scan-previous-durations: 3 +num-flaky-test-attempts: 6 diff --git a/tool/config/src/main/kotlin/flank/config/Config.kt b/tool/config/src/main/kotlin/flank/config/Config.kt index 03cbbda73d..f21221cf26 100644 --- a/tool/config/src/main/kotlin/flank/config/Config.kt +++ b/tool/config/src/main/kotlin/flank/config/Config.kt @@ -1,5 +1,6 @@ package flank.config +import com.fasterxml.jackson.databind.MapperFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.registerKotlinModule @@ -36,5 +37,7 @@ inline fun loadYaml(path: String): T = yamlMapper.readValue(File(path), T::class.java) val yamlMapper: ObjectMapper by lazy { - ObjectMapper(YAMLFactory()).registerKotlinModule() + ObjectMapper(YAMLFactory()) + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + .registerKotlinModule() } diff --git a/tool/instrument/log/src/main/kotlin/flank/instrument/log/Parser.kt b/tool/instrument/log/src/main/kotlin/flank/instrument/log/Parser.kt index d9fa2b73ab..2b83d0cc08 100644 --- a/tool/instrument/log/src/main/kotlin/flank/instrument/log/Parser.kt +++ b/tool/instrument/log/src/main/kotlin/flank/instrument/log/Parser.kt @@ -48,12 +48,16 @@ sealed class Instrument { val details: Details, ) : Instrument() { + val name = details.name + data class Details( val raw: Map, val className: String, val testName: String, val stack: String?, - ) + ) { + val name get() = "$className#$testName" + } } /** @@ -94,5 +98,10 @@ sealed class Instrument { const val FAILED = -2 const val EXCEPTION = -1 const val SKIPPED = -3 + + val errors = setOf( + FAILED, + EXCEPTION, + ) } } diff --git a/tool/junit/src/main/kotlin/flank/junit/JUnit.kt b/tool/junit/src/main/kotlin/flank/junit/JUnit.kt index 888cbb4cff..85a976d247 100644 --- a/tool/junit/src/main/kotlin/flank/junit/JUnit.kt +++ b/tool/junit/src/main/kotlin/flank/junit/JUnit.kt @@ -76,6 +76,7 @@ object JUnit { val endsAt: Long, val stack: List, val status: Status, + val flaky: Boolean = false, ) { enum class Status { Passed, Failed, Error, Skipped } diff --git a/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt b/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt index b26d67736a..1b327a88c0 100644 --- a/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt +++ b/tool/junit/src/main/kotlin/flank/junit/mapper/Structural.kt @@ -11,6 +11,7 @@ internal fun List.mapToTestSuites(): List = this failures = cases.count { it.status == JUnit.TestResult.Status.Failed }, errors = cases.count { it.status == JUnit.TestResult.Status.Error }, skipped = cases.count { it.status == JUnit.TestResult.Status.Skipped }, + flakes = cases.count { it.flaky }, timestamp = JUnit.dateFormat.format(cases.map { it.startAt }.minOrNull() ?: 0), time = cases.filterNot { it.status == JUnit.TestResult.Status.Skipped }.run { if (isEmpty()) 0.0 @@ -23,7 +24,8 @@ internal fun List.mapToTestSuites(): List = this time = (case.endsAt - case.startAt).toDouble() / 1000, error = if (case.status == JUnit.TestResult.Status.Error) case.stack else emptyList(), failure = if (case.status == JUnit.TestResult.Status.Failed) case.stack else emptyList(), - skipped = if (case.status == JUnit.TestResult.Status.Skipped) null else Unit + skipped = if (case.status == JUnit.TestResult.Status.Skipped) null else Unit, + flaky = case.flaky, ) } ) @@ -48,7 +50,8 @@ internal fun List.mapToTestResults(): List = case.skipped == null -> JUnit.TestResult.Status.Skipped else -> JUnit.TestResult.Status.Passed }, - stack = case.run { error + failure } + stack = case.run { error + failure }, + flaky = case.flaky, ) } }