Calculate flaky tests

corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt

@@ -150,6 +150,27 @@ class TestAndroidCommand :
         )
         @set:JsonProperty("num-flaky-test-attempts")
         var flakyTestAttempts: Int? by data
+
+        @set:JsonProperty("junit-report-config")
+        var junitReport: Map<String, Set<Args.Report.JUnit.Type>>? by data
+
+        @CommandLine.Option(
+            names = ["--junit-report-config"],
+            split = ";",
+            description = [
+                "A map of name suffixes related to set of result types required to include in custom junit report. " +
+                    "As results, this option will generate additional amount of junit reports named `JUnitReport-\$suffix.xml`." +
+                    "Available result types to include are: [Skipped, Passed, Failed, Flaky]." +
+                    "Default value is `--junit-report-config=failures=Failed,Flaky;`"
+            ]
+        )
+        fun setJUnitReport(map: Map<String, String>) {
+            junitReport = map.mapValues { (_, types) ->
+                types.split(",")
+                    .map { type -> type.lowercase().replaceFirstChar(Char::uppercaseChar) }
+                    .map(Args.Report.JUnit.Type::valueOf).toSet()
+            }
+        }
     }
 
     @CommandLine.Option(

corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt

@@ -18,5 +18,6 @@ internal val args = Args from setOf(Config) using context {
         gpuAcceleration = config.gpuAcceleration!!,
         scanPreviousDurations = config.scanPreviousDurations!!,
         flakyTestsAttempts = config.flakyTestAttempts!!,
+        junitReport = config.junitReport!!
     )
 }

corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt

@@ -30,6 +30,7 @@ private operator fun Config.plusAssign(args: TestAndroid.Args) {
     gpuAcceleration = args.gpuAcceleration
     scanPreviousDurations = args.scanPreviousDurations
     flakyTestAttempts = args.flakyTestsAttempts
+    junitReport = args.junitReport
 }
 
 internal fun yamlConfig(path: String?): Config =

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt

@@ -40,6 +40,7 @@ class ArgsKtTest {
                 gpuAcceleration = gpuAcceleration!!,
                 scanPreviousDurations = scanPreviousDurations!!,
                 flakyTestsAttempts = flakyTestAttempts!!,
+                junitReport = junitReport!!,
             )
         }
 

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt

@@ -37,7 +37,8 @@ class ConfigKtTest {
                 "--obfuscate=$obfuscate",
                 "--gpu-acceleration=$gpuAcceleration",
                 "--scan-previous-durations=$scanPreviousDurations",
-                "--num-flaky-test-attempts=$flakyTestAttempts"
+                "--num-flaky-test-attempts=$flakyTestAttempts",
+                "--junit-report-config=test1=Passed;test2=FAILED,flaky"
             )
         }
 
@@ -62,6 +63,9 @@ class ConfigKtTest {
             gpu-acceleration: $gpuAcceleration
             scan-previous-durations: $scanPreviousDurations
             num-flaky-test-attempts: $flakyTestAttempts
+            junit-report-config:
+              test1: [Passed]
+              test2: [FAILED, flaky]
             """.trimIndent()
         }
 

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt

@@ -2,6 +2,9 @@ package flank.corellium.cli.test.android.task
 
 import flank.corellium.cli.TestAndroidCommand
 import flank.corellium.domain.TestAndroid
+import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Failed
+import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Flaky
+import flank.corellium.domain.TestAndroid.Args.Report.JUnit.Type.Passed
 
 /**
  * Apply test values to config. Each value should be different than default.
@@ -34,4 +37,8 @@ fun TestAndroidCommand.Config.applyTestValues() = apply {
     gpuAcceleration = false
     scanPreviousDurations = 123
     flakyTestAttempts = Int.MAX_VALUE
+    junitReport = mapOf(
+        "test1" to setOf(Passed),
+        "test2" to setOf(Failed, Flaky),
+    )
 }

corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt

@@ -27,6 +27,7 @@ import flank.corellium.domain.test.android.task.loadPreviousDurations
 import flank.corellium.domain.test.android.task.parseApksInfo
 import flank.corellium.domain.test.android.task.parseTestCasesFromApks
 import flank.corellium.domain.test.android.task.prepareShards
+import flank.corellium.domain.test.android.task.processResults
 import flank.exection.parallel.Parallel
 import flank.exection.parallel.type
 import flank.instrument.log.Instrument
@@ -67,6 +68,7 @@ object TestAndroid {
         val gpuAcceleration: Boolean = true,
         val scanPreviousDurations: Int = 10,
         val flakyTestsAttempts: Int = 0,
+        val junitReport: JUnitReportConfig = Report.JUnit.Default,
     ) {
 
         companion object : Parallel.Type<Args> {
@@ -78,7 +80,7 @@ object TestAndroid {
         /**
          * Default output directory scheme.
          *
-         * @property new Directory name in format: `results/corellium/android/yyyy-MM-dd_HH-mm-ss-SSS`.
+         * @property new A directory name in format: `results/corellium/android/yyyy-MM-dd_HH-mm-ss-SSS`.
          */
         object DefaultOutputDir {
             internal const val ROOT = "results/corellium/android/"
@@ -111,6 +113,19 @@ object TestAndroid {
                 override val path: String
             ) : Apk()
         }
+
+        /**
+         * Report configuration file.
+         */
+        object Report {
+            object JUnit {
+                enum class Type { Skipped, Passed, Failed, Flaky }
+
+                val Default = mapOf(
+                    "failures" to setOf(Type.Failed, Type.Flaky)
+                )
+            }
+        }
     }
 
     // Context
@@ -132,7 +147,8 @@ object TestAndroid {
      * @property dispatch Channel for dispatching test shards to execute.
      * @property devices Channel for providing devices that are available and ready to use.
      * @property ids the ids of corellium device instances.
-     * @property testResult Execution results.
+     * @property rawResults Execution results.
+     * @property processResults Results processed according to [Args.junitReport] configuration.
      */
     internal class Context : Parallel.Context() {
         val api by !type<CorelliumApi>()
@@ -147,7 +163,8 @@ object TestAndroid {
         val dispatch: Channel<Dispatch.Data> by -Dispatch.Shards
         val devices: Channel<Device.Instance> by -AvailableDevices
         val ids: List<String> by -InvokeDevices
-        val testResult: List<Device.Result> by -ExecuteTests
+        val rawResults: List<Device.Result> by -ExecuteTests
+        val processedResults: Map<String, List<Device.Result>> by -ProcessedResults
     }
 
     internal val context = Parallel.Function(::Context)
@@ -226,6 +243,8 @@ object TestAndroid {
     }
 
     object ReleaseDevice : Parallel.Type<Unit>
+    object ProcessedResults : Parallel.Type<Map<String, List<Device.Result>>>
+
     object CleanUp : Parallel.Type<Unit>
     object GenerateReport : Parallel.Type<Unit>
     object CompleteTests : Parallel.Type<Unit>
@@ -292,6 +311,7 @@ object TestAndroid {
             val id: String,
             val data: Dispatch.Data,
             val value: List<Instrument>,
+            val flakes: Set<String> = emptySet(),
         )
 
         internal val execute by lazy {
@@ -307,24 +327,34 @@ object TestAndroid {
 
     // Evaluate lazy to avoid strange NullPointerException.
     val execute by lazy {
+        // Keep alphabetic order.
         setOf(
             context.validate,
             authorize,
+            availableDevices,
             createOutputDir,
+            dispatchFailedTests,
             dispatchShards,
             dispatchTests,
-            dispatchFailedTests,
             dumpShards,
             executeTestQueue,
             finish,
             generateReport,
             initResultsChannel,
-            availableDevices,
             invokeDevices,
             loadPreviousDurations,
             parseApksInfo,
             parseTestCasesFromApks,
             prepareShards,
+            processResults,
         )
     }
 }
+
+/**
+ * JUnit report configuration.
+ *
+ * key - suffix that will be added to [JUnit.REPORT_FILE_NAME] for creating custom JUnitReport file name.
+ * value - set of required results to include in report.
+ */
+typealias JUnitReportConfig = Map<String, Set<TestAndroid.Args.Report.JUnit.Type>>

corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt

@@ -24,12 +24,9 @@ internal val dispatchFailedTests = Dispatch.Failed from setOf(
     var index = 0
     results.consumeEach { result ->
         if (result.status is Instrument.Status) {
-            if (result.status.code in errorCodes) {
-                val shard = result.shard
-                    .reduceTo(result.status.details.fullTestName)
-                val attempt = runs
-                    .getOrPut(shard, counter)
-                    .getAndIncrement()
+            if (result.status.code in Instrument.Code.errors) {
+                val shard = result.shard.reduceTo(result.status.name)
+                val attempt = runs.getOrPut(shard, counter).getAndIncrement()
                 if (attempt < args.flakyTestsAttempts)
                     dispatch.send(
                         Dispatch.Data(
@@ -44,14 +41,6 @@ internal val dispatchFailedTests = Dispatch.Failed from setOf(
     runs.mapValues { (_, value) -> value.get() }
 }
 
-/**
- * Set of [Instrument] error codes.
- */
-private val errorCodes = setOf(
-    Instrument.Code.FAILED,
-    Instrument.Code.EXCEPTION,
-)
-
 /**
  * Creates new [InstanceShard] that contains only one test basing on the given [name].
  *
@@ -73,5 +62,3 @@ private fun InstanceShard.reduceTo(
             }.takeIf { it.isNotEmpty() }?.let { test.copy(cases = it) }
         }.takeIf { it.isNotEmpty() }?.let { app.copy(tests = it) }
     }
-
-private val Instrument.Status.Details.fullTestName get() = "$className#$testName"

corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt

@@ -6,11 +6,13 @@ import flank.corellium.domain.TestAndroid.Dispatch
 import flank.corellium.domain.TestAndroid.ExecuteTests
 import flank.corellium.domain.TestAndroid.GenerateReport
 import flank.corellium.domain.TestAndroid.OutputDir
+import flank.corellium.domain.TestAndroid.ProcessedResults
 import flank.corellium.domain.TestAndroid.context
 import flank.exection.parallel.from
 import flank.exection.parallel.using
 import flank.instrument.log.Instrument
 import flank.junit.JUnit
+import flank.junit.JUnit.REPORT_FILE_NAME
 import flank.junit.generateJUnitReport
 import flank.junit.writeAsXml
 import java.io.File
@@ -21,10 +23,24 @@ import java.io.File
  */
 internal val generateReport = GenerateReport from setOf(
     ExecuteTests,
-    OutputDir
+    ProcessedResults,
+    OutputDir,
 ) using context {
-    val file = File(args.outputDir, JUnit.REPORT_FILE_NAME)
-    testResult
+    // Generate default junit report from raw results.
+    generateReport(rawResults, REPORT_FILE_NAME)
+
+    // Generate junit reports from processed results.
+    processedResults.forEach { (suffix, results) ->
+        generateReport(results, REPORT_FILE_NAME.replace(".", "_$suffix."))
+    }
+}
+
+private fun TestAndroid.Context.generateReport(
+    results: List<Device.Result>,
+    fileName: String
+) {
+    val file = File(args.outputDir, fileName)
+    results
         .prepareInputForJUnit()
         .generateJUnitReport()
         .writeAsXml(file.bufferedWriter())
@@ -49,6 +65,7 @@ private fun List<Device.Result>.prepareInputForJUnit(): List<JUnit.TestResult> =
                 startAt = status.startTime,
                 endsAt = status.endTime,
                 stack = listOfNotNull(status.details.stack),
+                flaky = status.name in result.flakes,
                 status = when (status.code) {
                     Instrument.Code.PASSED -> JUnit.TestResult.Status.Passed
                     Instrument.Code.FAILED -> JUnit.TestResult.Status.Failed

corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ProcessResults.kt

@@ -0,0 +1,77 @@
+package flank.corellium.domain.test.android.task
+
+import flank.corellium.domain.TestAndroid
+import flank.corellium.domain.TestAndroid.Args.Report.JUnit
+import flank.corellium.domain.TestAndroid.Dispatch
+import flank.corellium.domain.TestAndroid.ExecuteTests
+import flank.corellium.domain.TestAndroid.ProcessedResults
+import flank.corellium.domain.TestAndroid.context
+import flank.exection.parallel.from
+import flank.exection.parallel.using
+import flank.instrument.log.Instrument
+import flank.instrument.log.Instrument.Code.PASSED
+import flank.instrument.log.Instrument.Code.SKIPPED
+import flank.instrument.log.Instrument.Code.errors
+
+/**
+ * Process raw test results according to [TestAndroid.Args.Report] requirements.
+ */
+val processResults = ProcessedResults from setOf(
+    ExecuteTests,
+) using context {
+    val codes = rawResults.mapTestCodes()
+    val shards = rawResults.filter { result -> result.data.type == Dispatch.Type.Shard }
+
+    args.junitReport.mapValues { (_, types) ->
+        shards.filter(codes.testNamesBy(types)).run {
+            if (JUnit.Type.Flaky !in types) this
+            else update(codes.testNamesBy(JUnit.Type.Flaky))
+        }
+    }
+}
+
+private typealias TestResults = List<TestAndroid.Device.Result>
+private typealias TestCodes = Map<String, List<Int>>
+
+private fun TestResults.mapTestCodes(): TestCodes = this
+    .flatMap { result ->
+        result.value
+            .filterIsInstance<Instrument.Status>()
+            .map { status -> status.run { name to code } }
+    }
+    .groupBy(
+        keySelector = { (name, _) -> name },
+        valueTransform = { (_, codes) -> codes }
+    )
+
+private fun TestCodes.testNamesBy(types: Set<JUnit.Type>): Set<String> =
+    types.flatMap { testNamesBy(it) }.toSet()
+
+private fun TestCodes.testNamesBy(type: JUnit.Type): Set<String> {
+    fun <K, V> Map<K, V>.keysByValues(predicate: V.() -> Boolean): Set<K> = filterValues(predicate).keys
+    return when (type) {
+        JUnit.Type.Skipped -> keysByValues { all { code -> code == SKIPPED } }
+        JUnit.Type.Passed -> keysByValues { all { code -> code == PASSED } }
+        JUnit.Type.Failed -> keysByValues { all { code -> code in errors } }
+        JUnit.Type.Flaky -> keysByValues { groupBy { code -> code in errors }.size == 2 }
+    }
+}
+
+private fun TestResults.filter(testNames: Set<String>): TestResults = map { result ->
+    result.copy(
+        value = result.value
+            .filterIsInstance<Instrument.Status>()
+            .filter { status -> status.name in testNames }
+    )
+}
+
+private fun TestResults.update(flakyTests: Set<String>): TestResults =
+    map { result ->
+        result.copy(
+            flakes = result.value
+                .filterIsInstance<Instrument.Status>()
+                .map(Instrument.Status::name)
+                .filter(flakyTests::contains)
+                .toSet()
+        )
+    }

test_configs/flank-corellium-many.yml

@@ -11,4 +11,9 @@ apks:
 max-test-shards: 6
 gpu-acceleration: false
 scan-previous-durations: 3
-num-flaky-test-attempts: 3
+num-flaky-test-attempts: 6
+
+junit-report-config:
+  skipped: [Skipped]
+  passed: [Passed]
+  failures: [Failed, Flaky]

test_configs/flank-corellium.yml

@@ -11,3 +11,4 @@ apks:
 max-test-shards: 3
 gpu-acceleration: false
 scan-previous-durations: 3
+num-flaky-test-attempts: 6