From 6b82e1ab4955a5fc3086f24fd45e350ff629a590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 2 Jun 2021 08:31:50 +0200 Subject: [PATCH 1/4] Load test cases durations from previous run and use for sharding --- corellium/cli/build.gradle.kts | 1 + .../cli/RunTestCorelliumAndroidCommand.kt | 17 ++- .../cli/RunTestCorelliumAndroidCommandTest.kt | 4 + .../domain/RunTestCorelliumAndroid.kt | 16 ++- .../run/test/android/step/GenerateReport.kt | 4 +- .../android/step/LoadPreviousDurations.kt | 42 +++++++ .../run/test/android/step/PrepareShards.kt | 11 +- .../domain/RunTestAndroidCorelliumExample.kt | 2 + ...nTestAndroidCorelliumTestMockApiAndroid.kt | 3 + ...nTestAndroidCorelliumTestParsingAndroid.kt | 3 + .../src/main/kotlin/flank/junit/JUnit.kt | 42 +++---- .../internal/CalculateMedianDurations.kt | 21 ++++ .../junit/internal/ParseJUnitTestResults.kt | 17 +++ .../kotlin/flank/junit/mapper/JacksonXml.kt | 15 +-- .../flank/junit/mapper/ParseJUnitReport.kt | 33 ++++++ .../kotlin/flank/junit/mapper/Structural.kt | 23 +++- .../flank/junit/GenerateJUnitReportTest.kt | 105 ------------------ .../junit/src/test/kotlin/flank/junit/Util.kt | 1 - .../CalculateMedianDurationsKtTest.kt | 48 ++++++++ .../FailOnInvalidJUnitReportTest.kt | 9 +- .../{ => mapper}/ParseEmptyJUnitReportTest.kt | 7 +- .../{ => mapper}/ParseJUnitReportTest.kt | 10 +- .../flank/junit/mapper/StructuralKtTest.kt | 104 +++++++++++++++++ 23 files changed, 379 insertions(+), 159 deletions(-) create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt create mode 100644 corellium/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt create mode 100644 corellium/junit/src/main/kotlin/flank/junit/internal/ParseJUnitTestResults.kt create mode 100644 corellium/junit/src/main/kotlin/flank/junit/mapper/ParseJUnitReport.kt delete mode 100644 corellium/junit/src/test/kotlin/flank/junit/GenerateJUnitReportTest.kt create mode 100644 corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt rename corellium/junit/src/test/kotlin/flank/junit/{ => mapper}/FailOnInvalidJUnitReportTest.kt (59%) rename corellium/junit/src/test/kotlin/flank/junit/{ => mapper}/ParseEmptyJUnitReportTest.kt (81%) rename corellium/junit/src/test/kotlin/flank/junit/{ => mapper}/ParseJUnitReportTest.kt (59%) create mode 100644 corellium/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt diff --git a/corellium/cli/build.gradle.kts b/corellium/cli/build.gradle.kts index 782c2625b2..ab6654e72b 100644 --- a/corellium/cli/build.gradle.kts +++ b/corellium/cli/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { implementation(project(":corellium:domain")) implementation(project(":corellium:adapter")) implementation(project(":corellium:apk")) + implementation(project(":corellium:junit")) implementation(Dependencies.JACKSON_KOTLIN) implementation(Dependencies.JACKSON_YAML) implementation(Dependencies.JACKSON_XML) diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt index 4032a4cdce..69bfd4db84 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt @@ -13,6 +13,7 @@ import flank.corellium.corelliumApi import flank.corellium.domain.RunTestCorelliumAndroid import flank.corellium.domain.RunTestCorelliumAndroid.Args import flank.corellium.domain.invoke +import flank.junit.JUnit import picocli.CommandLine @CommandLine.Command( @@ -101,6 +102,16 @@ class RunTestCorelliumAndroidCommand : ) @set:JsonProperty("gpu-acceleration") var gpuAcceleration: Boolean? by data + + @set:CommandLine.Option( + names = ["--scan-previous-durations"], + description = [ + "Scan the specified amount of JUnitReport.xml files to obtain test cases durations necessary for optimized sharding." + + "The `local-result-dir` is used for searching JUnit reports." + ] + ) + @set:JsonProperty("scan-previous-durations") + var scanPreviousDurations: Int? by data } @CommandLine.Mixin @@ -118,6 +129,8 @@ class RunTestCorelliumAndroidCommand : override val apk = Apk.Api() + override val junit = JUnit.Api() + override val args by lazy { createArgs() } override fun run() = invoke() @@ -131,6 +144,7 @@ private fun defaultConfig() = Config().apply { localResultsDir = null obfuscate = false gpuAcceleration = true + scanPreviousDurations = 10 } private fun RunTestCorelliumAndroidCommand.yamlConfig(): Config = @@ -142,5 +156,6 @@ private fun RunTestCorelliumAndroidCommand.createArgs() = Args( maxShardsCount = config.maxTestShards!!, outputDir = config.localResultsDir ?: Args.DefaultOutputDir.new, obfuscateDumpShards = config.obfuscate!!, - gpuAcceleration = config.gpuAcceleration!! + gpuAcceleration = config.gpuAcceleration!!, + scanPreviousDurations = config.scanPreviousDurations!!, ) diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt index 50a47caec4..57c9ad5351 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt @@ -35,6 +35,7 @@ class RunTestCorelliumAndroidCommandTest { localResultsDir = "test_result_dir" obfuscate = true gpuAcceleration = false + scanPreviousDurations = 123 } /** @@ -59,6 +60,7 @@ class RunTestCorelliumAndroidCommandTest { "--local-result-dir=$localResultsDir", "--obfuscate=$obfuscate", "--gpu-acceleration=$gpuAcceleration", + "--scan-previous-durations=$scanPreviousDurations", ) } @@ -78,6 +80,7 @@ max-test-shards: $maxTestShards local-result-dir: $localResultsDir obfuscate: $obfuscate gpu-acceleration: $gpuAcceleration +scan-previous-durations: $scanPreviousDurations """.trimIndent() } @@ -141,6 +144,7 @@ gpu-acceleration: $gpuAcceleration outputDir = localResultsDir!!, obfuscateDumpShards = obfuscate!!, gpuAcceleration = gpuAcceleration!!, + scanPreviousDurations = scanPreviousDurations!!, ) } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt index 9d4fe3776e..398519ea36 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt @@ -14,6 +14,7 @@ import flank.corellium.domain.run.test.android.step.finish import flank.corellium.domain.run.test.android.step.generateReport import flank.corellium.domain.run.test.android.step.installApks import flank.corellium.domain.run.test.android.step.invokeDevices +import flank.corellium.domain.run.test.android.step.loadPreviousDurations import flank.corellium.domain.run.test.android.step.parseApksInfo import flank.corellium.domain.run.test.android.step.parseTestCasesFromApks import flank.corellium.domain.run.test.android.step.prepareShards @@ -21,6 +22,7 @@ import flank.corellium.domain.util.CreateTransformation import flank.corellium.domain.util.execute import flank.corellium.shard.Shard import flank.instrument.log.Instrument +import flank.junit.JUnit import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.runBlocking import java.lang.System.currentTimeMillis @@ -38,6 +40,7 @@ object RunTestCorelliumAndroid { interface Context { val api: CorelliumApi val apk: Apk.Api + val junit: JUnit.Api val args: Args } @@ -50,6 +53,7 @@ object RunTestCorelliumAndroid { * @param obfuscateDumpShards Obfuscate the test names in shards before dumping to file. * @param outputDir Set output dir. Default value is [DefaultOutputDir.new] * @param gpuAcceleration Enable gpu acceleration for newly created virtual devices. + * @param scanPreviousDurations Scan the specified amount of JUnitReport.xml files to obtain test cases durations necessary for optimized sharding. The [outputDir] is used for searching JUnit reports. */ data class Args( val credentials: Authorization.Credentials, @@ -58,6 +62,7 @@ object RunTestCorelliumAndroid { val obfuscateDumpShards: Boolean = false, val outputDir: String = DefaultOutputDir.new, val gpuAcceleration: Boolean = true, + val scanPreviousDurations: Int = 10, ) { /** * Default output directory scheme. @@ -65,9 +70,9 @@ object RunTestCorelliumAndroid { * @property new Directory name in format: `results/corellium/android/yyyy-MM-dd_HH-mm-ss-SSS`. */ object DefaultOutputDir { - private const val PATH = "results/corellium/android/" + internal const val ROOT = "results/corellium/android/" private val date = SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS") - val new get() = PATH + date.format(currentTimeMillis()) + val new get() = ROOT + date.format(currentTimeMillis()) } /** @@ -102,6 +107,7 @@ object RunTestCorelliumAndroid { * For convenience the properties are sorted in order equal to its initialization. * * @param testCases key - path to the test apk, value - list of test method names. + * @param previousDurations key - test case name, value - calculated previous duration. * @param shards each item is representing list of apps to run on another device instance. * @param ids the ids of corellium device instances. * @param packageNames key - path to the test apk, value - package name. @@ -109,6 +115,7 @@ object RunTestCorelliumAndroid { */ internal data class State( val testCases: Map> = emptyMap(), + val previousDurations: Map = defaultPreviousDurations, val shards: List> = emptyList(), val ids: List = emptyList(), val packageNames: Map = emptyMap(), @@ -116,6 +123,10 @@ object RunTestCorelliumAndroid { val testResult: List> = emptyList(), ) + private const val DEFAULT_TEST_CASE_DURATION = 120L + + private val defaultPreviousDurations = emptyMap().withDefault { DEFAULT_TEST_CASE_DURATION } + /** * The reference to the step factory. * Invoke it to generate new execution step. @@ -127,6 +138,7 @@ operator fun Context.invoke(): Unit = runBlocking { State() execute flowOf( authorize(), parseTestCasesFromApks(), + loadPreviousDurations(), prepareShards(), createOutputDir(), dumpShards(), diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt index 387e58e6a0..400bffcd43 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt @@ -18,7 +18,7 @@ import java.io.File */ internal fun RunTestCorelliumAndroid.Context.generateReport() = RunTestCorelliumAndroid.step { println("* Generating report") - val file = File(args.outputDir, JUNIT_REPORT_FILENAME) + val file = File(args.outputDir, JUnit.REPORT_FILE_NAME) testResult .prepareInputForJUnit() .generateJUnitReport() @@ -27,8 +27,6 @@ internal fun RunTestCorelliumAndroid.Context.generateReport() = RunTestCorellium this } -private const val JUNIT_REPORT_FILENAME = "JUnitReport.xml" - /** * Simple mapper, no logical operations or API calls, * just converting one structure to another. diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt new file mode 100644 index 0000000000..7e8cdba77a --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/LoadPreviousDurations.kt @@ -0,0 +1,42 @@ +package flank.corellium.domain.run.test.android.step + +import flank.corellium.domain.RunTestCorelliumAndroid +import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir +import flank.junit.calculateTestCaseDurations + +/** + * The step is searching result directory for JUnitReport.xml. + * Collected reports are used for calculating test cases durations. + * + * require: + * * [RunTestCorelliumAndroid.Context.parseTestCasesFromApks] + * + * updates: + * * [RunTestCorelliumAndroid.State.previousDurations] + */ +internal fun RunTestCorelliumAndroid.Context.loadPreviousDurations() = RunTestCorelliumAndroid.step { + println("* Obtaining previous test cases durations") + + val directoryToScan: String = + if (args.outputDir.startsWith(DefaultOutputDir.ROOT)) DefaultOutputDir.ROOT + else args.outputDir + + copy( + previousDurations = junit.parseTestResults(directoryToScan) + .take(args.scanPreviousDurations).toList() + .apply { println("Searching in $size JUnitReport.xml files...") } + .flatten() + .calculateTestCaseDurations() + .withDefault { previousDurations.getValue(it) } + .also { durations -> printStats(durations.keys) } + ) +} + +private fun RunTestCorelliumAndroid.State.printStats(obtainedDurations: Set) { + val testCasesNames = testCases.flatMap { (_, cases) -> cases }.toSet() + val unknown = (obtainedDurations - testCasesNames).size + val matching = obtainedDurations.size - unknown + val required = testCasesNames.size + + println("For $required test cases, found $matching matching and $unknown unknown") +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/PrepareShards.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/PrepareShards.kt index 1cfbd5a3c8..c421609bc8 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/PrepareShards.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/PrepareShards.kt @@ -20,6 +20,7 @@ internal fun RunTestCorelliumAndroid.Context.prepareShards() = RunTestCorelliumA apps = prepareDataForSharding( apks = args.apks, testCases = testCases, + durations = previousDurations, ), maxCount = args.maxShardsCount ) @@ -32,7 +33,8 @@ internal fun RunTestCorelliumAndroid.Context.prepareShards() = RunTestCorelliumA */ private fun prepareDataForSharding( apks: List, - testCases: Map> + testCases: Map>, + durations: Map, ): List = apks.map { app -> Shard.App( @@ -42,7 +44,12 @@ private fun prepareDataForSharding( name = test.path, cases = testCases .getValue(test.path) - .map(Shard.Test::Case) + .map { name -> + Shard.Test.Case( + name = name, + duration = durations.getValue(name) + ) + } ) } ) diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumExample.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumExample.kt index 3f0daaae9b..a61ccde09b 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumExample.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumExample.kt @@ -2,10 +2,12 @@ package flank.corellium.domain import flank.apk.Apk import flank.corellium.corelliumApi +import flank.junit.JUnit object RunTestAndroidCorelliumExample : RunTestCorelliumAndroid.Context { override val api = corelliumApi("Default Project") override val apk = Apk.Api() + override val junit = JUnit.Api() override val args = RunTestCorelliumAndroid.Args( credentials = loadedCredentials, apks = fewTestArtifactsApks(APK_PATH_MAIN), diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt index 89139495cd..13fe57dedb 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt @@ -3,6 +3,7 @@ package flank.corellium.domain import flank.apk.Apk import flank.corellium.api.CorelliumApi import flank.corellium.domain.RunTestCorelliumAndroid.Args +import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.map import org.junit.After @@ -84,6 +85,8 @@ class RunTestAndroidCorelliumTestMockApiAndroid : RunTestCorelliumAndroid.Contex }, ) + override val junit = JUnit.Api() + @Test fun test(): Unit = invoke() diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt index 1f7c44daea..45464dc40e 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt @@ -2,6 +2,7 @@ package flank.corellium.domain import flank.apk.Apk import flank.corellium.api.CorelliumApi +import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.map import org.junit.After @@ -55,6 +56,8 @@ class RunTestAndroidCorelliumTestParsingAndroid : RunTestCorelliumAndroid.Contex }, ) + override val junit = JUnit.Api() + override val apk = Apk.Api() @Test diff --git a/corellium/junit/src/main/kotlin/flank/junit/JUnit.kt b/corellium/junit/src/main/kotlin/flank/junit/JUnit.kt index 6e487f9b1f..f609463f04 100644 --- a/corellium/junit/src/main/kotlin/flank/junit/JUnit.kt +++ b/corellium/junit/src/main/kotlin/flank/junit/JUnit.kt @@ -4,13 +4,11 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement -import com.fasterxml.jackson.module.kotlin.readValue +import flank.junit.internal.calculateMedianDurations +import flank.junit.internal.parseJUnitTestResults import flank.junit.mapper.TimeSerializer import flank.junit.mapper.mapToTestSuites -import flank.junit.mapper.readEmptyTestSuites -import flank.junit.mapper.xmlMapper import flank.junit.mapper.xmlPrettyWriter -import java.io.File import java.io.Writer import java.text.SimpleDateFormat @@ -22,35 +20,28 @@ import java.text.SimpleDateFormat * * This early implementation doesn't support flaky tests. * * The list of test case results should be sorted in same order as received from console output * - * @receiver list of raw test cases results. - * @return structural representation of XML JUnit report + * @receiver List of raw test cases results. + * @return Structural representation of XML JUnit report */ fun List.generateJUnitReport(): JUnit.Report = JUnit.Report(mapToTestSuites()) -/** - * Parse [JUnit.Report] from file path. - * - * @receiver path to XML JUnit report - * @return parsed structural representation of XML JUnit report - */ -fun String.parseJUnitReportFromFile(): JUnit.Report = - File(this).let { file -> - xmlMapper.readValue(file) - ?: file.readEmptyTestSuites() - ?: throw IllegalArgumentException("cannot parse JUnitReport from: $this") - } - /** * Write JUnite report as formatted XML string. * - * @receiver structural representation of XML JUnit report. + * @receiver Structural representation of XML JUnit report. * @param writer The output where report will be written. */ fun JUnit.Report.writeAsXml(writer: Writer) { xmlPrettyWriter.writeValue(writer, this) } +/** + * Calculate associate full test cases names to calculated duration. + */ +fun List.calculateTestCaseDurations(): Map = + calculateMedianDurations() + // ========================= Structures ========================= /** @@ -62,6 +53,10 @@ fun JUnit.Report.writeAsXml(writer: Writer) { */ object JUnit { + class Api( + val parseTestResults: TestResult.Parse = parseJUnitTestResults + ) + /** * Compact representation of test case execution result. * Contains all data required to generate JUnitReport. @@ -76,6 +71,11 @@ object JUnit { val status: Status, ) { enum class Status { Passed, Failed, Error, Skipped } + + /** + * Search given file or directory for [REPORT_FILE_NAME] and parse as TestResults + */ + fun interface Parse : (String) -> Sequence> } /** @@ -200,4 +200,6 @@ object JUnit { private const val ISO8601_DATETIME_PATTERN = "yyyy-MM-dd'T'HH:mm:ssXXX" internal val dateFormat = SimpleDateFormat(ISO8601_DATETIME_PATTERN) + + const val REPORT_FILE_NAME = "JUnitReport.xml" } diff --git a/corellium/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt b/corellium/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt new file mode 100644 index 0000000000..004036d891 --- /dev/null +++ b/corellium/junit/src/main/kotlin/flank/junit/internal/CalculateMedianDurations.kt @@ -0,0 +1,21 @@ +package flank.junit.internal + +import flank.junit.JUnit + +/** + * Group test case results by full name and associate with calculated median of durations in group. + */ +internal fun List.calculateMedianDurations(): Map = this + .map { result -> result.fullName to result.duration } + .groupBy(Pair::first, Pair::second) + .mapValues { (_, durations) -> durations.median() } + +private val JUnit.TestResult.fullName get() = "$className#$testName" + +private val JUnit.TestResult.duration get() = (endsAt - startAt) + +private fun List.median(): Long = when { + isEmpty() -> throw IllegalArgumentException("Cannot calculate median of empty list") + size % 2 == 0 -> (size / 2).let { get(it - 1) + get(it) } / 2 + else -> get(size / 2) +} diff --git a/corellium/junit/src/main/kotlin/flank/junit/internal/ParseJUnitTestResults.kt b/corellium/junit/src/main/kotlin/flank/junit/internal/ParseJUnitTestResults.kt new file mode 100644 index 0000000000..83caddfe98 --- /dev/null +++ b/corellium/junit/src/main/kotlin/flank/junit/internal/ParseJUnitTestResults.kt @@ -0,0 +1,17 @@ +package flank.junit.internal + +import flank.junit.JUnit +import flank.junit.mapper.mapToTestResults +import flank.junit.mapper.parseJUnitReport +import java.io.File + +internal val parseJUnitTestResults = JUnit.TestResult.Parse { path -> + val file = File(path) + when { + file.isFile -> sequenceOf(file) + file.isDirectory -> file.walkTopDown().filter { next -> next.name == JUnit.REPORT_FILE_NAME } + else -> emptySequence() + }.map { next: File -> + next.reader().parseJUnitReport().testsuites.mapToTestResults() + } +} diff --git a/corellium/junit/src/main/kotlin/flank/junit/mapper/JacksonXml.kt b/corellium/junit/src/main/kotlin/flank/junit/mapper/JacksonXml.kt index 3c50a00a48..f55d9fcb0e 100644 --- a/corellium/junit/src/main/kotlin/flank/junit/mapper/JacksonXml.kt +++ b/corellium/junit/src/main/kotlin/flank/junit/mapper/JacksonXml.kt @@ -9,11 +9,9 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper import com.fasterxml.jackson.dataformat.xml.deser.FromXmlParser import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator import com.fasterxml.jackson.module.kotlin.KotlinModule -import flank.junit.JUnit -import java.io.File import java.util.Locale -internal val xmlModule = JacksonXmlModule().apply { setDefaultUseWrapper(false) } +private val xmlModule = JacksonXmlModule().apply { setDefaultUseWrapper(false) } internal val xmlMapper = XmlMapper(xmlModule) .apply { @@ -33,14 +31,3 @@ internal class TimeSerializer : JsonSerializer() { ) } } - -/** - * For objectMapper is returning null. - * This is helper method for fixing this behaviour. - */ -internal fun File.readEmptyTestSuites(): JUnit.Report? = - JUnit.Report().takeIf { - readLines() - .filter { it.isNotBlank() } - .run { size < 3 && last() == "" } - } diff --git a/corellium/junit/src/main/kotlin/flank/junit/mapper/ParseJUnitReport.kt b/corellium/junit/src/main/kotlin/flank/junit/mapper/ParseJUnitReport.kt new file mode 100644 index 0000000000..c2f3b6bc06 --- /dev/null +++ b/corellium/junit/src/main/kotlin/flank/junit/mapper/ParseJUnitReport.kt @@ -0,0 +1,33 @@ +package flank.junit.mapper + +import com.fasterxml.jackson.module.kotlin.readValue +import flank.junit.JUnit +import java.io.BufferedReader +import java.io.Reader + +/** + * Parse [JUnit.Report] from reader. + * + * @receiver Reader of XML JUnit report + * @return Parsed structural representation of XML JUnit report + */ +internal fun Reader.parseJUnitReport(): JUnit.Report = + buffered().run { + readEmptyTestSuites() + ?: xmlMapper.readValue(this) + ?: throw IllegalArgumentException("cannot parse JUnitReport from: $this") + } + +/** + * For objectMapper is returning null. + * This is helper method for fixing this behaviour. + */ +private fun BufferedReader.readEmptyTestSuites(): JUnit.Report? = + JUnit.Report().takeIf { + mark(DEFAULT_BUFFER_SIZE) + val string = String(CharArray(DEFAULT_BUFFER_SIZE).let { buffer -> buffer.copyOf(read(buffer)) }) + reset() + string.lines() + .filter { it.isNotBlank() } + .run { size < 3 && last() == "" } + } diff --git a/corellium/junit/src/main/kotlin/flank/junit/mapper/Structural.kt b/corellium/junit/src/main/kotlin/flank/junit/mapper/Structural.kt index 674c014274..7fd8fa2f24 100644 --- a/corellium/junit/src/main/kotlin/flank/junit/mapper/Structural.kt +++ b/corellium/junit/src/main/kotlin/flank/junit/mapper/Structural.kt @@ -2,7 +2,7 @@ package flank.junit.mapper import flank.junit.JUnit -internal fun List.mapToTestSuites() = this +internal fun List.mapToTestSuites(): List = this .groupBy { case -> case.suiteName } .map { (suiteName, cases: List) -> JUnit.Suite( @@ -23,7 +23,28 @@ internal fun List.mapToTestSuites() = this time = (case.endsAt - case.startAt).toDouble() / 1000, error = if (case.status == JUnit.TestResult.Status.Error) case.stack else emptyList(), failure = if (case.status == JUnit.TestResult.Status.Failed) case.stack else emptyList(), + skipped = if (case.status == JUnit.TestResult.Status.Skipped) null else Unit ) } ) } + +internal fun List.mapToTestResults(): List = + flatMap { suite -> + suite.testcases.map { case -> + JUnit.TestResult( + suiteName = suite.name, + testName = case.name, + className = case.classname, + startAt = 0, + endsAt = (case.time * 1000).toLong(), + status = when { + case.error.isNotEmpty() -> JUnit.TestResult.Status.Error + case.failure.isNotEmpty() -> JUnit.TestResult.Status.Failed + case.skipped == null -> JUnit.TestResult.Status.Skipped + else -> JUnit.TestResult.Status.Passed + }, + stack = case.run { error + failure } + ) + } + } diff --git a/corellium/junit/src/test/kotlin/flank/junit/GenerateJUnitReportTest.kt b/corellium/junit/src/test/kotlin/flank/junit/GenerateJUnitReportTest.kt deleted file mode 100644 index 9e155a0842..0000000000 --- a/corellium/junit/src/test/kotlin/flank/junit/GenerateJUnitReportTest.kt +++ /dev/null @@ -1,105 +0,0 @@ -package flank.junit - -import flank.junit.mapper.xmlPrettyWriter -import org.junit.Assert -import org.junit.Test - -class GenerateJUnitReportTest { - - @Test - fun test() { - val expected = JUnit.Report( - testsuites = listOf( - JUnit.Suite( - name = "suite1", - tests = 3, - failures = 1, - errors = 1, - skipped = 0, - time = 8.0, - timestamp = JUnit.dateFormat.format(1_000), - testcases = listOf( - JUnit.Case( - name = "test1", - classname = "test1.Test1", - time = 4.0, - error = listOf("some error") - ), - JUnit.Case( - name = "test2", - classname = "test1.Test1", - time = 2.0, - failure = listOf("some assertion failed"), - ), - JUnit.Case( - name = "test1", - classname = "test1.Test2", - time = 3.0, - ) - ) - ), - JUnit.Suite( - name = "suite2", - tests = 1, - failures = 0, - errors = 0, - skipped = 1, - time = 0.0, - timestamp = JUnit.dateFormat.format(0), - testcases = listOf( - JUnit.Case( - name = "test1", - classname = "test1.Test1", - time = 0.0, - ), - ) - ), - ) - ) - - val testCases = listOf( - JUnit.TestResult( - testName = "test1", - className = "test1.Test1", - suiteName = "suite1", - startAt = 1_000, - endsAt = 5_000, - status = JUnit.TestResult.Status.Error, - stack = listOf("some error") - ), - JUnit.TestResult( - testName = "test2", - className = "test1.Test1", - suiteName = "suite1", - startAt = 6_000, - endsAt = 8_000, - status = JUnit.TestResult.Status.Failed, - stack = listOf("some assertion failed") - ), - JUnit.TestResult( - testName = "test1", - className = "test1.Test2", - suiteName = "suite1", - startAt = 6_000, - endsAt = 9_000, - status = JUnit.TestResult.Status.Passed, - stack = emptyList() - ), - JUnit.TestResult( - testName = "test1", - className = "test1.Test1", - suiteName = "suite2", - startAt = 0, - endsAt = 0, - status = JUnit.TestResult.Status.Skipped, - stack = emptyList() - ), - ) - - val actual = testCases.generateJUnitReport() - - println(xmlPrettyWriter.writeValueAsString(actual)) - - Assert.assertEquals(expected, actual) - } -} diff --git a/corellium/junit/src/test/kotlin/flank/junit/Util.kt b/corellium/junit/src/test/kotlin/flank/junit/Util.kt index dd35976333..03da888965 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/Util.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/Util.kt @@ -1,4 +1,3 @@ package flank.junit const val RESOURCES = "./src/test/resources/" -const val JUNIT_REPORT = "" diff --git a/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt new file mode 100644 index 0000000000..b2fb50e1ef --- /dev/null +++ b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt @@ -0,0 +1,48 @@ +package flank.junit.internal + +import flank.junit.JUnit +import flank.junit.calculateTestCaseDurations +import org.junit.Assert +import org.junit.Test + +class CalculateMedianDurationsKtTest { + + @Test + fun test() { + // given + val results = listOf( + result("a", 5), + result("a", 6), + result("a", 100), + result("b", 10), + result("b", 20), + result("c", 300), + ) + + val expected = mapOf( + "a#a" to 6L, + "b#b" to 15L, + "c#c" to 300L, + ) + + // when + val actual = results.calculateTestCaseDurations() + + // then + Assert.assertEquals(expected, actual) + } +} + +private fun result( + name: String, + duration: Long +) = JUnit.TestResult( + className = name, + testName = name, + startAt = 0, + endsAt = duration, + stack = emptyList(), + status = JUnit.TestResult.Status.Passed, + suiteName = "" +) + diff --git a/corellium/junit/src/test/kotlin/flank/junit/FailOnInvalidJUnitReportTest.kt b/corellium/junit/src/test/kotlin/flank/junit/mapper/FailOnInvalidJUnitReportTest.kt similarity index 59% rename from corellium/junit/src/test/kotlin/flank/junit/FailOnInvalidJUnitReportTest.kt rename to corellium/junit/src/test/kotlin/flank/junit/mapper/FailOnInvalidJUnitReportTest.kt index 9a0b809a4f..f3af629936 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/FailOnInvalidJUnitReportTest.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/mapper/FailOnInvalidJUnitReportTest.kt @@ -1,15 +1,18 @@ -package flank.junit +package flank.junit.mapper +import flank.junit.JUnit +import flank.junit.RESOURCES import org.junit.Assert import org.junit.Test +import java.io.File import java.lang.IllegalArgumentException class FailOnInvalidJUnitReportTest { @Test(expected = IllegalArgumentException::class) fun test() { - val path = RESOURCES + "JUnitReport_invalid.xml" - val parsed = path.parseJUnitReportFromFile() + val file = File(RESOURCES + "JUnitReport_invalid.xml") + val parsed = file.reader().parseJUnitReport() println(parsed) Assert.assertEquals( diff --git a/corellium/junit/src/test/kotlin/flank/junit/ParseEmptyJUnitReportTest.kt b/corellium/junit/src/test/kotlin/flank/junit/mapper/ParseEmptyJUnitReportTest.kt similarity index 81% rename from corellium/junit/src/test/kotlin/flank/junit/ParseEmptyJUnitReportTest.kt rename to corellium/junit/src/test/kotlin/flank/junit/mapper/ParseEmptyJUnitReportTest.kt index 85e8b4d009..d1e1bf6a6e 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/ParseEmptyJUnitReportTest.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/mapper/ParseEmptyJUnitReportTest.kt @@ -1,9 +1,12 @@ -package flank.junit +package flank.junit.mapper +import flank.junit.JUnit +import flank.junit.RESOURCES import org.junit.Assert import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized +import java.io.File /** * Various tests for parsing empty test reports in different shapes. @@ -27,7 +30,7 @@ class ParseEmptyJUnitReportTest( @Test fun test() { val path = RESOURCES + name - val parsed = path.parseJUnitReportFromFile() + val parsed = File(path).reader().parseJUnitReport() println(parsed) Assert.assertEquals( diff --git a/corellium/junit/src/test/kotlin/flank/junit/ParseJUnitReportTest.kt b/corellium/junit/src/test/kotlin/flank/junit/mapper/ParseJUnitReportTest.kt similarity index 59% rename from corellium/junit/src/test/kotlin/flank/junit/ParseJUnitReportTest.kt rename to corellium/junit/src/test/kotlin/flank/junit/mapper/ParseJUnitReportTest.kt index fa3ae0cb55..78147084b7 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/ParseJUnitReportTest.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/mapper/ParseJUnitReportTest.kt @@ -1,6 +1,6 @@ -package flank.junit +package flank.junit.mapper -import flank.junit.mapper.xmlPrettyWriter +import flank.junit.RESOURCES import org.junit.Assert.assertArrayEquals import org.junit.Test import java.io.File @@ -9,10 +9,10 @@ class ParseJUnitReportTest { @Test fun test() { - val path = RESOURCES + "JUnitReport.xml" - val expected = File(path).readLines().filter(String::isNotBlank).toTypedArray() + val file = File(RESOURCES + "JUnitReport.xml") + val expected = file.readLines().filter(String::isNotBlank).toTypedArray() - val parsed = path.parseJUnitReportFromFile() + val parsed = file.reader().parseJUnitReport() val formatted = xmlPrettyWriter.writeValueAsString(parsed).lines() val actual = formatted.filter(String::isNotBlank).toTypedArray() diff --git a/corellium/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt b/corellium/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt new file mode 100644 index 0000000000..42cbbf20e1 --- /dev/null +++ b/corellium/junit/src/test/kotlin/flank/junit/mapper/StructuralKtTest.kt @@ -0,0 +1,104 @@ +package flank.junit.mapper + +import flank.junit.JUnit +import org.junit.Assert.assertEquals +import org.junit.Test + +class StructuralKtTest { + + @Test + fun mapToTestSuitesTest() { + val expected = listOf( + JUnit.Suite( + name = "suite1", + tests = 3, + failures = 1, + errors = 1, + skipped = 0, + time = 8.0, + timestamp = JUnit.dateFormat.format(1_000), + testcases = listOf( + JUnit.Case( + name = "test1", + classname = "test1.Test1", + time = 4.0, + error = listOf("some error") + ), + JUnit.Case( + name = "test2", + classname = "test1.Test1", + time = 2.0, + failure = listOf("some assertion failed"), + ), + JUnit.Case( + name = "test1", + classname = "test1.Test2", + time = 3.0, + ) + ) + ), + JUnit.Suite( + name = "suite2", + tests = 1, + failures = 0, + errors = 0, + skipped = 1, + time = 0.0, + timestamp = JUnit.dateFormat.format(0), + testcases = listOf( + JUnit.Case( + name = "test1", + classname = "test1.Test1", + time = 0.0, + skipped = null, + ), + ) + ), + ) + + val testCases = listOf( + JUnit.TestResult( + testName = "test1", + className = "test1.Test1", + suiteName = "suite1", + startAt = 1_000, + endsAt = 5_000, + status = JUnit.TestResult.Status.Error, + stack = listOf("some error") + ), + JUnit.TestResult( + testName = "test2", + className = "test1.Test1", + suiteName = "suite1", + startAt = 6_000, + endsAt = 8_000, + status = JUnit.TestResult.Status.Failed, + stack = listOf("some assertion failed") + ), + JUnit.TestResult( + testName = "test1", + className = "test1.Test2", + suiteName = "suite1", + startAt = 6_000, + endsAt = 9_000, + status = JUnit.TestResult.Status.Passed, + stack = emptyList() + ), + JUnit.TestResult( + testName = "test1", + className = "test1.Test1", + suiteName = "suite2", + startAt = 0, + endsAt = 0, + status = JUnit.TestResult.Status.Skipped, + stack = emptyList() + ), + ) + + val actual = testCases.mapToTestSuites() + + println(xmlPrettyWriter.writeValueAsString(actual)) + + assertEquals(expected, actual) + } +} From 4a74122476514f3bcc9e0cfe8d26706afb6f80a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 2 Jun 2021 10:44:17 +0200 Subject: [PATCH 2/4] Update flank_corellium.md --- docs/flank_corellium.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/flank_corellium.md b/docs/flank_corellium.md index 1386ccaa02..76707db45e 100644 --- a/docs/flank_corellium.md +++ b/docs/flank_corellium.md @@ -160,6 +160,7 @@ The successful run should generate the following files: # Features * Calculating multi-module shards. +* Reusing test cases duration for sharding. * Creating or reusing instances (devices). * Installing APKs on remote devices. * Running android tests. @@ -170,7 +171,6 @@ The successful run should generate the following files: # Roadmap * Cleaning devices after test execution. -* Reusing test cases duration for sharding. * Flaky test detection. * Structural logging. * iOS support. From 9acc810c623309d354925c2ac2d127563d538f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 2 Jun 2021 11:57:44 +0200 Subject: [PATCH 3/4] Remove empty line --- .../flank/junit/internal/CalculateMedianDurationsKtTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt index b2fb50e1ef..5c432c27cb 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt @@ -45,4 +45,3 @@ private fun result( status = JUnit.TestResult.Status.Passed, suiteName = "" ) - From b498afb4db5d6f2fa009863d55177c37f83a66f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 2 Jun 2021 11:57:44 +0200 Subject: [PATCH 4/4] Remove empty line --- .../flank/junit/internal/CalculateMedianDurationsKtTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt index 5c432c27cb..d6001d5bfd 100644 --- a/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt +++ b/corellium/junit/src/test/kotlin/flank/junit/internal/CalculateMedianDurationsKtTest.kt @@ -35,7 +35,7 @@ class CalculateMedianDurationsKtTest { private fun result( name: String, - duration: Long + duration: Long, ) = JUnit.TestResult( className = name, testName = name, @@ -43,5 +43,5 @@ private fun result( endsAt = duration, stack = emptyList(), status = JUnit.TestResult.Status.Passed, - suiteName = "" + suiteName = "", )