Rerun failed tests

corellium/cli/src/main/kotlin/flank/corellium/cli/TestAndroidCommand.kt

@@ -17,6 +17,7 @@ import flank.corellium.domain.TestAndroid.execute
 import flank.exection.parallel.Parallel
 import flank.exection.parallel.ParallelState
 import flank.exection.parallel.invoke
+import flank.exection.parallel.plus
 import flank.exection.parallel.verify
 import flank.log.output
 import kotlinx.coroutines.flow.last
@@ -139,6 +140,16 @@ class TestAndroidCommand :
         )
         @set:JsonProperty("scan-previous-durations")
         var scanPreviousDurations: Int? by data
+
+        @set:CommandLine.Option(
+            names = ["--num-flaky-test-attempts"],
+            description = [
+                "The number of times a test case should be re-attempted if fail for any reason. " +
+                    "Default value is 0, which implies no reruns."
+            ]
+        )
+        @set:JsonProperty("num-flaky-test-attempts")
+        var flakyTestAttempts: Int? by data
     }
 
     @CommandLine.Option(
@@ -159,8 +170,8 @@ class TestAndroidCommand :
 
     override fun run() {
         val seed: ParallelState = mapOf(
-            TestAndroidCommand to this,
-            Parallel.Logger to format.output,
+            TestAndroidCommand + this,
+            Parallel.Logger + format.output,
         )
         runBlocking {
             resolve(seed).last().verify().let { args ->

corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/Format.kt

@@ -23,7 +23,6 @@ internal val format = buildFormatter<String> {
     Event.Start(TestAndroid.ExecuteTests) { "* Executing tests" }
     Event.Start(TestAndroid.CompleteTests) { "* Finish" }
     Event.Start(TestAndroid.GenerateReport) { "* Generating report" }
-    Event.Start(TestAndroid.InstallApks) { "* Installing apks" }
     Event.Start(TestAndroid.InvokeDevices) { "* Invoking devices" }
     Event.Start(TestAndroid.LoadPreviousDurations) { "* Obtaining previous test cases durations" }
     Event.Start(TestAndroid.ParseApkInfo) { "* Parsing apk info" }
@@ -57,7 +56,7 @@ internal val format = buildFormatter<String> {
             "$id:\n" + commands.joinToString("\n") { " - $it" }
         }
     }
-    TestAndroid.ExecuteTests.Status::class {
+    TestAndroid.ExecuteTests.Result::class {
         when (val status = status) {
             is Instrument.Status -> "$id: " + status.details.run { "$className#$testName" } + " - " + status.code
             else -> null

corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Args.kt

@@ -17,5 +17,6 @@ internal val args = Args from setOf(Config) using context {
         obfuscateDumpShards = config.obfuscate!!,
         gpuAcceleration = config.gpuAcceleration!!,
         scanPreviousDurations = config.scanPreviousDurations!!,
+        flakyTestsAttempts = config.flakyTestAttempts!!,
     )
 }

corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/task/Config.kt

@@ -29,6 +29,7 @@ private operator fun Config.plusAssign(args: TestAndroid.Args) {
     obfuscate = args.obfuscateDumpShards
     gpuAcceleration = args.gpuAcceleration
     scanPreviousDurations = args.scanPreviousDurations
+    flakyTestAttempts = args.flakyTestsAttempts
 }
 
 internal fun yamlConfig(path: String?): Config =

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/FormatKtTest.kt

@@ -30,7 +30,6 @@ class FormatKtTest {
             TestAndroid.ExecuteTests event Event.Start,
             TestAndroid.CompleteTests event Event.Start,
             TestAndroid.GenerateReport event Event.Start,
-            TestAndroid.InstallApks event Event.Start,
             TestAndroid.InvokeDevices event Event.Start,
             TestAndroid.LoadPreviousDurations event Event.Start,
             TestAndroid.ParseApkInfo event Event.Start,
@@ -49,14 +48,15 @@ class FormatKtTest {
             Unit event TestAndroid.InvokeDevices.Status(AndroidInstance.Event.Creating(7)),
             Unit event TestAndroid.InvokeDevices.Status(AndroidInstance.Event.Waiting),
             Unit event TestAndroid.InvokeDevices.Status(AndroidInstance.Event.Ready("123456")),
-            Unit event TestAndroid.ExecuteTests.Status(
+            Unit event TestAndroid.ExecuteTests.Result(
                 id = "123456",
                 status = Instrument.Status(
                     code = 0,
                     startTime = 1,
                     endTime = 2,
                     details = Instrument.Status.Details(emptyMap(), "Class", "Test", null)
-                )
+                ),
+                shard = emptyList()
             ),
             Unit event TestAndroid.ExecuteTests.Error("1", Exception(), "path/to/log/1", 5..10),
             Unit event TestAndroid.Created(File("path/to/apk.apk")),

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ArgsKtTest.kt

@@ -5,6 +5,7 @@ import flank.corellium.cli.TestAndroidCommand
 import flank.corellium.domain.TestAndroid.Args
 import flank.exection.parallel.invoke
 import flank.exection.parallel.select
+import flank.exection.parallel.verify
 import kotlinx.coroutines.flow.last
 import kotlinx.coroutines.runBlocking
 import org.junit.Assert
@@ -38,14 +39,15 @@ class ArgsKtTest {
                 obfuscateDumpShards = obfuscate!!,
                 gpuAcceleration = gpuAcceleration!!,
                 scanPreviousDurations = scanPreviousDurations!!,
+                flakyTestsAttempts = flakyTestAttempts!!,
             )
         }
 
         val yamlAuth = expected.credentials.run {
             """
-host: $host
-username: $username
-password: $password
+            host: $host
+            username: $username
+            password: $password
             """.trimIndent()
         }
 
@@ -58,7 +60,7 @@ password: $password
         // ======================== WHEN ========================
 
         val actual = runBlocking {
-            setOf(args)(TestAndroidCommand.Config to testConfig).last().select(Args)
+            setOf(args)(TestAndroidCommand.Config to testConfig).last().verify().select(Args)
         }
 
         // ======================== THEN ========================

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/ConfigKtTest.kt

@@ -22,7 +22,7 @@ class ConfigKtTest {
         // ======================== GIVEN ========================
 
         // Create expected config for assertions
-        // Make sure to set values different then produced by defaultConfig() function,
+        // Make sure to set values different from produced by defaultConfig() function,
         // otherwise the test will fail
         val expectedConfig = Config().applyTestValues()
 
@@ -37,29 +37,31 @@ class ConfigKtTest {
                 "--obfuscate=$obfuscate",
                 "--gpu-acceleration=$gpuAcceleration",
                 "--scan-previous-durations=$scanPreviousDurations",
+                "--num-flaky-test-attempts=$flakyTestAttempts"
             )
         }
 
         val yamlConfig = expectedConfig.run {
             """
-auth: $auth
-project: $project
-apks:
-- path: "app1.apk"
-  tests:
-  - path: "app1-test1.apk"
-- path: "app2.apk"
-  tests:
-  - path: "app2-test1.apk"
-  - path: "app2-test2.apk"
-test-targets:
-  - class foo.Foo
-  - package bar
-max-test-shards: $maxTestShards
-local-result-dir: $localResultsDir
-obfuscate: $obfuscate
-gpu-acceleration: $gpuAcceleration
-scan-previous-durations: $scanPreviousDurations
+            auth: $auth
+            project: $project
+            apks:
+            - path: "app1.apk"
+              tests:
+              - path: "app1-test1.apk"
+            - path: "app2.apk"
+              tests:
+              - path: "app2-test1.apk"
+              - path: "app2-test2.apk"
+            test-targets:
+              - class foo.Foo
+              - package bar
+            max-test-shards: $maxTestShards
+            local-result-dir: $localResultsDir
+            obfuscate: $obfuscate
+            gpu-acceleration: $gpuAcceleration
+            scan-previous-durations: $scanPreviousDurations
+            num-flaky-test-attempts: $flakyTestAttempts
             """.trimIndent()
         }
 

corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/task/Util.kt

@@ -33,4 +33,5 @@ fun TestAndroidCommand.Config.applyTestValues() = apply {
     obfuscate = true
     gpuAcceleration = false
     scanPreviousDurations = 123
+    flakyTestAttempts = Int.MAX_VALUE
 }

corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt

@@ -11,12 +11,14 @@ import flank.corellium.domain.TestAndroid.Args.DefaultOutputDir.new
 import flank.corellium.domain.test.android.task.authorize
 import flank.corellium.domain.test.android.task.availableDevices
 import flank.corellium.domain.test.android.task.createOutputDir
+import flank.corellium.domain.test.android.task.dispatchFailedTests
 import flank.corellium.domain.test.android.task.dispatchShards
 import flank.corellium.domain.test.android.task.dispatchTests
 import flank.corellium.domain.test.android.task.dumpShards
-import flank.corellium.domain.test.android.task.executeTests
+import flank.corellium.domain.test.android.task.executeTestsV2
 import flank.corellium.domain.test.android.task.finish
 import flank.corellium.domain.test.android.task.generateReport
+import flank.corellium.domain.test.android.task.initResultsChannel
 import flank.corellium.domain.test.android.task.installApks
 import flank.corellium.domain.test.android.task.invokeDevices
 import flank.corellium.domain.test.android.task.loadPreviousDurations
@@ -58,9 +60,10 @@ object TestAndroid {
         val testTargets: List<String> = emptyList(),
         val maxShardsCount: Int = 1,
         val obfuscateDumpShards: Boolean = false,
-        val outputDir: String = DefaultOutputDir.new,
+        val outputDir: String = new,
         val gpuAcceleration: Boolean = true,
         val scanPreviousDurations: Int = 10,
+        val flakyTestsAttempts: Int = 0,
     ) {
 
         companion object : Parallel.Type<Args> {
@@ -132,15 +135,16 @@ object TestAndroid {
         val junit by !type<JUnit.Api>()
         val args by !Args
 
-        val dispatch by -DispatchShards
+        val dispatch by -Dispatch.Shards
         val devices by -AvailableDevices
+        val results by -ExecuteTests.Results
         val testCases: Map<String, List<String>> by -ParseTestCases
         val previousDurations: Map<String, Long> by -LoadPreviousDurations
         val shards: List<List<Shard.App>> by -PrepareShards
         val ids: List<String> by -InvokeDevices
         val packageNames by ParseApkInfo { packageNames }
         val testRunners by ParseApkInfo { testRunners }
-        val testResult: List<List<Instrument>> by -ExecuteTests
+        val testResult by -ExecuteTests
     }
 
     internal val context = Parallel.Function(::Context)
@@ -159,34 +163,61 @@ object TestAndroid {
     object ParseApkInfo : Parallel.Type<Info>
     object OutputDir : Parallel.Type<Unit>
     object DumpShards : Parallel.Type<Unit>
-    object DispatchShards : Parallel.Type<Channel<InstanceShard>>
     object Authorize : Parallel.Type<Unit>
     object AvailableDevices : Parallel.Type<Channel<Device.Instance>>
     object InvokeDevices : Parallel.Type<List<String>> {
         object Status : Event.Type<AndroidInstance.Event>
     }
 
-    object InstallApks : Parallel.Type<Unit> {
+    object InstallApks : Parallel.Type<List<String>> {
         object Status : Event.Type<AndroidApps.Event>
     }
 
-    object ExecuteTests : Parallel.Type<List<List<Instrument>>> {
+    object Dispatch {
+        object Shards : Parallel.Type<Channel<Data>>
+        object Tests : Parallel.Type<List<Device.Result>>
+        object Failed : Parallel.Type<Map<InstanceShard, Int>>
+
+        data class Data(
+            val index: Int,
+            val shard: InstanceShard,
+            val source: Parallel.Type<*>,
+        ) {
+            companion object : Parallel.Type<Data>
+        }
+    }
+
+    object ExecuteTests : Parallel.Type<List<Device.Result>> {
         const val ADB_LOG = "adb_log"
 
+        object Results : Parallel.Type<Channel<Result>>
+
         object Plan : Event.Type<AndroidTestPlan.Config>
-        data class Status(val id: String, val status: Instrument) : Event.Data
+
+        data class Dispatch(
+            val id: String,
+            val data: TestAndroid.Dispatch.Data
+        ) : Event.Data
+
+        data class Result(
+            val id: String,
+            val status: Instrument,
+            val shard: InstanceShard,
+        ) : Event.Data
+
         data class Error(
             val id: String,
             val cause: Throwable,
             val logFile: String,
             val lines: IntRange
         ) : Event.Data
-    }
 
-    object HandleResults : Parallel.Type<Unit> {
-        object Init : Parallel.Type<Channel<ExecuteTests.Status>>
+        data class Finish(
+            val id: String
+        ) : Event.Data
     }
 
+    object ReleaseDevice : Parallel.Type<Unit>
     object CleanUp : Parallel.Type<Unit>
     object GenerateReport : Parallel.Type<Unit>
     object CompleteTests : Parallel.Type<Unit>
@@ -214,8 +245,13 @@ object TestAndroid {
             val api by !type<CorelliumApi>()
             val args by !Args
             val info by !ParseApkInfo
-            val shard by !Shard
-            val instance by !Instance
+            val results by !ExecuteTests.Results
+            val data by !Dispatch.Data
+            val shard get() = data.shard
+            val device by !Instance
+            val release by !AvailableDevices
+
+            val installedApks by -InstallApks
         }
 
         internal val context = Parallel.Function(::Context)
@@ -229,47 +265,38 @@ object TestAndroid {
             companion object : Parallel.Type<Instance>
         }
 
+        data class Result(
+            val id: String,
+            val data: Dispatch.Data,
+            val value: List<Instrument>,
+        )
+
         internal val execute by lazy {
             setOf(
                 flank.corellium.domain.test.android.device.task.installApks,
-                flank.corellium.domain.test.android.device.task.executeTests,
+                flank.corellium.domain.test.android.device.task.executeShard,
+                flank.corellium.domain.test.android.device.task.releaseDevice,
             )
         }
     }
 
     // Execution tasks
 
     // Evaluate lazy to avoid strange NullPointerException.
-    val execute get() = executeV2
-
-    private val executeV1 by lazy {
-        setOf(
-            context.validate,
-            authorize,
-            createOutputDir,
-            dumpShards,
-            executeTests,
-            finish,
-            generateReport,
-            installApks,
-            invokeDevices,
-            loadPreviousDurations,
-            parseApksInfo,
-            parseTestCasesFromApks,
-            prepareShards,
-        )
-    }
-
-    private val executeV2 by lazy {
+    val execute by lazy {
         setOf(
             context.validate,
             authorize,
             createOutputDir,
             dispatchShards,
             dispatchTests,
             dumpShards,
+            // executeTests,
+            executeTestsV2,
             finish,
             generateReport,
+            dispatchFailedTests,
+            initResultsChannel,
             installApks,
             dispatchTests,
             availableDevices,