diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt index 19a36728c0..adae440227 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt @@ -9,7 +9,10 @@ import flank.corellium.api.CorelliumApi import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir import flank.corellium.domain.RunTestCorelliumAndroid.Args.DefaultOutputDir.new import flank.corellium.domain.run.test.android.step.authorize +import flank.corellium.domain.run.test.android.step.availableDevices import flank.corellium.domain.run.test.android.step.createOutputDir +import flank.corellium.domain.run.test.android.step.dispatchShards +import flank.corellium.domain.run.test.android.step.dispatchTests import flank.corellium.domain.run.test.android.step.dumpShards import flank.corellium.domain.run.test.android.step.executeTests import flank.corellium.domain.run.test.android.step.finish @@ -25,7 +28,9 @@ import flank.exection.parallel.type import flank.instrument.log.Instrument import flank.junit.JUnit import flank.log.Event +import flank.shard.InstanceShard import flank.shard.Shard +import kotlinx.coroutines.channels.Channel import java.io.File import java.lang.System.currentTimeMillis import java.text.SimpleDateFormat @@ -127,6 +132,8 @@ object RunTestCorelliumAndroid { val junit by !type() val args by !Args + val dispatch by -DispatchShards + val devices by -AvailableDevices val testCases: Map> by -ParseTestCases val previousDurations: Map by -LoadPreviousDurations val shards: List> by -PrepareShards @@ -152,7 +159,9 @@ object RunTestCorelliumAndroid { object ParseApkInfo : Parallel.Type object OutputDir : Parallel.Type object DumpShards : Parallel.Type + object DispatchShards : Parallel.Type> object Authorize : Parallel.Type + object AvailableDevices : Parallel.Type> object InvokeDevices : Parallel.Type> { object Status : Event.Type } @@ -174,6 +183,10 @@ object RunTestCorelliumAndroid { ) : Event.Data } + object HandleResults : Parallel.Type { + object Init : Parallel.Type> + } + object CleanUp : Parallel.Type object GenerateReport : Parallel.Type object CompleteTests : Parallel.Type @@ -190,10 +203,46 @@ object RunTestCorelliumAndroid { object Created : Event.Type object AlreadyExist : Event.Type + // Nested + + /** + * Nested scope that represents shard execution on single device + */ + object Device { + + internal class Context : Parallel.Context() { + val api by !type() + val args by !Args + val info by !ParseApkInfo + val shard by !Shard + val instance by !Instance + } + + internal val context = Parallel.Function(::Context) + + object Shard : Parallel.Type + + data class Instance( + val id: String, + val apks: Set = emptySet() + ) { + companion object : Parallel.Type + } + + internal val execute by lazy { + setOf( + flank.corellium.domain.run.test.android.device.task.installApks, + flank.corellium.domain.run.test.android.device.task.executeTests, + ) + } + } + // Execution tasks // Evaluate lazy to avoid strange NullPointerException. - val execute by lazy { + val execute get() = executeV2 + + private val executeV1 by lazy { setOf( context.validate, authorize, @@ -210,4 +259,25 @@ object RunTestCorelliumAndroid { prepareShards, ) } + + private val executeV2 by lazy { + setOf( + context.validate, + authorize, + createOutputDir, + dispatchShards, + dispatchTests, + dumpShards, + finish, + generateReport, + installApks, + dispatchTests, + availableDevices, + invokeDevices, + loadPreviousDurations, + parseApksInfo, + parseTestCasesFromApks, + prepareShards, + ) + } } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/ExecuteTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/ExecuteTests.kt new file mode 100644 index 0000000000..e2ea1e6f69 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/ExecuteTests.kt @@ -0,0 +1,95 @@ +package flank.corellium.domain.run.test.android.device.task + +import flank.corellium.api.AndroidTestPlan +import flank.corellium.domain.RunTestCorelliumAndroid +import flank.corellium.domain.RunTestCorelliumAndroid.Authorize +import flank.corellium.domain.RunTestCorelliumAndroid.Device +import flank.corellium.domain.RunTestCorelliumAndroid.ExecuteTests +import flank.corellium.domain.RunTestCorelliumAndroid.InstallApks +import flank.corellium.domain.RunTestCorelliumAndroid.ParseApkInfo +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.instrument.command.formatAmInstrumentCommand +import flank.instrument.log.Instrument +import flank.instrument.log.parseAdbInstrumentLog +import flank.log.Output +import flank.shard.Shard +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.take +import java.io.File + +/** + * The step is executing tests on previously invoked devices, and returning the test results. + * + * The side effect is console logs from `am instrument` saved inside [RunTestCorelliumAndroid.ExecuteTests.ADB_LOG] output subdirectory. + * + * The optional parsing errors are sent through [RunTestCorelliumAndroid.Context.out]. + */ +internal val executeTests = ExecuteTests from setOf( + ParseApkInfo, + Authorize, + InstallApks, +) using Device.context { + val outputDir = File(args.outputDir, ExecuteTests.ADB_LOG).apply { mkdir() } + val testPlan: AndroidTestPlan.Config = prepareTestPlan() + coroutineScope { + ExecuteTests.Plan(testPlan).out() + api.executeTest(testPlan).map { (id, flow) -> + async { flow.collectResults(id, shard.size, outputDir, out) } + }.awaitAll() + } +} + +/** + * Prepare [AndroidTestPlan.Config] for test execution. + * It is just mapping and formatting the data collected in state. + */ +private fun Device.Context.prepareTestPlan(): AndroidTestPlan.Config = + AndroidTestPlan.Config( + mapOf( + instance.id to shard.flatMap { app: Shard.App -> + app.tests.map { test -> + formatAmInstrumentCommand( + packageName = info.packageNames.getValue(test.name), + testRunner = info.testRunners.getValue(test.name), + testCases = test.cases.map { case -> "class " + case.name } + ) + } + } + ) + ) + +private suspend fun Flow.collectResults( + id: String, + expectedResults: Int, + dir: File, + out: Output, +): List { + var read = 0 + var parsed = 0 + val file = dir.resolve(id) + val results = mutableListOf() + this.onEach { file.appendText(it + "\n") } + .buffer(100) + .flowOn(Dispatchers.IO) + .onEach { ++read } + .parseAdbInstrumentLog() + .onEach { parsed = read } + .onEach { result -> results += result } + .onEach { result -> ExecuteTests.Status(id, result).out() } + .catch { cause -> ExecuteTests.Error(id, cause, file.path, ++parsed..read).out() } + .filterIsInstance() + .take(expectedResults) + .collect() + return results +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/InstallApks.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/InstallApks.kt new file mode 100644 index 0000000000..ede85016b3 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/device/task/InstallApks.kt @@ -0,0 +1,30 @@ +package flank.corellium.domain.run.test.android.device.task + +import flank.corellium.api.AndroidApps +import flank.corellium.domain.RunTestCorelliumAndroid.Authorize +import flank.corellium.domain.RunTestCorelliumAndroid.Device +import flank.corellium.domain.RunTestCorelliumAndroid.InstallApks +import flank.exection.parallel.from +import flank.exection.parallel.using +import kotlinx.coroutines.flow.collect + +/** + * The step is installing required software on android instances. + */ +internal val installApks = InstallApks from setOf( + Authorize +) using Device.context { + val apks = prepareApkToInstall() + api.installAndroidApps(listOf(apks)).collect { event -> + InstallApks.Status(event).out() + } + // If tests will be executed too fast just after the + // app installed, the instrumentation will fail + // delay(4_000) TODO: for verification +} + +private fun Device.Context.prepareApkToInstall() = + AndroidApps( + instanceId = instance.id, + paths = shard.flatMap { app -> app.tests.map { test -> test.name } + app.name } + ) diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/AvailableDevices.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/AvailableDevices.kt new file mode 100644 index 0000000000..5493f35014 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/AvailableDevices.kt @@ -0,0 +1,16 @@ +package flank.corellium.domain.run.test.android.step + +import flank.corellium.domain.RunTestCorelliumAndroid +import flank.corellium.domain.RunTestCorelliumAndroid.AvailableDevices +import flank.exection.parallel.from +import flank.exection.parallel.using +import kotlinx.coroutines.channels.Channel + +/** + * Prepare channel for providing devices ready to use. + */ +val availableDevices = AvailableDevices from setOf( + RunTestCorelliumAndroid.PrepareShards +) using RunTestCorelliumAndroid.context { + Channel(shards.size) +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchShards.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchShards.kt new file mode 100644 index 0000000000..432a0ff9d1 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchShards.kt @@ -0,0 +1,19 @@ +package flank.corellium.domain.run.test.android.step + +import flank.corellium.domain.RunTestCorelliumAndroid +import flank.corellium.domain.RunTestCorelliumAndroid.DispatchShards +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.shard.InstanceShard +import kotlinx.coroutines.channels.Channel + +/** + * Prepare channel for dispatching shards and fill the buffer with initial elements. + */ +val dispatchShards = DispatchShards from setOf( + RunTestCorelliumAndroid.PrepareShards +) using RunTestCorelliumAndroid.context { + Channel(shards.size).apply { + shards.forEach { shard -> send(shard) } + } +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchTests.kt new file mode 100644 index 0000000000..270747a44a --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/DispatchTests.kt @@ -0,0 +1,75 @@ +package flank.corellium.domain.run.test.android.step + +import flank.corellium.domain.RunTestCorelliumAndroid.Authorize +import flank.corellium.domain.RunTestCorelliumAndroid.AvailableDevices +import flank.corellium.domain.RunTestCorelliumAndroid.Device +import flank.corellium.domain.RunTestCorelliumAndroid.DispatchShards +import flank.corellium.domain.RunTestCorelliumAndroid.ExecuteTests +import flank.corellium.domain.RunTestCorelliumAndroid.ParseApkInfo +import flank.corellium.domain.RunTestCorelliumAndroid.context +import flank.exection.parallel.ParallelState +import flank.exection.parallel.from +import flank.exection.parallel.invoke +import flank.exection.parallel.select +import flank.exection.parallel.using +import flank.exection.parallel.verify +import flank.shard.InstanceShard +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch + +/** + * Dispatch each test shard execution into first best matching device. + */ +val dispatchTests = ExecuteTests from setOf( + ParseApkInfo, + Authorize, + AvailableDevices, + DispatchShards, +) using context { state -> + coroutineScope { + var count = 0 + channelFlow { + for (shard in dispatch) { + val index = ++count + val instance = devices.maxBy { instance -> + (shard.apks intersect instance.apks).size + } + val seed = mapOf( + Device.Shard to shard, + Device.Instance to instance, + ) + launch { + val result = Device + .execute(state + seed) + .last() + .verify() + .select(ExecuteTests) + send(result) + devices.send(instance) + if (index == count) + dispatch.close() + } + } + }.toList().reduce { accumulator, value -> + accumulator + value + } + } +} + +/** + * Flatten into the list of apk. + */ +private val InstanceShard.apks: List + get() = flatMap { app -> app.tests.map { test -> test.name } + app.name } + +private suspend fun Channel.maxBy(select: (T) -> Int): T = + mutableListOf().apply { + add(receive()) + while (!isEmpty) add(receive()) + sortByDescending(select) + drop(1).forEach { send(it) } + }.first() diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt index 0eeaf4d027..3fccd9e0e6 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/GenerateReport.kt @@ -3,6 +3,7 @@ package flank.corellium.domain.run.test.android.step import flank.corellium.domain.RunTestCorelliumAndroid import flank.corellium.domain.RunTestCorelliumAndroid.ExecuteTests import flank.corellium.domain.RunTestCorelliumAndroid.GenerateReport +import flank.corellium.domain.RunTestCorelliumAndroid.InvokeDevices import flank.corellium.domain.RunTestCorelliumAndroid.OutputDir import flank.corellium.domain.RunTestCorelliumAndroid.context import flank.exection.parallel.from @@ -18,6 +19,7 @@ import java.io.File * Generated JUnit report is saved as formatted xml file */ internal val generateReport = GenerateReport from setOf( + InvokeDevices, ExecuteTests, OutputDir ) using context { diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/HandleResults.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/HandleResults.kt new file mode 100644 index 0000000000..c4e4369945 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/HandleResults.kt @@ -0,0 +1,11 @@ +package flank.corellium.domain.run.test.android.step + +import flank.corellium.domain.RunTestCorelliumAndroid.HandleResults +import flank.exection.parallel.from +import flank.exection.parallel.using + +val handleResults = HandleResults from setOf( + HandleResults.Init +) using { + TODO() +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/InvokeDevices.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/InvokeDevices.kt index 9189870a0a..fe6dbf0dcb 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/InvokeDevices.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/InvokeDevices.kt @@ -2,6 +2,8 @@ package flank.corellium.domain.run.test.android.step import flank.corellium.api.AndroidInstance import flank.corellium.domain.RunTestCorelliumAndroid.Authorize +import flank.corellium.domain.RunTestCorelliumAndroid.AvailableDevices +import flank.corellium.domain.RunTestCorelliumAndroid.Device import flank.corellium.domain.RunTestCorelliumAndroid.InvokeDevices import flank.corellium.domain.RunTestCorelliumAndroid.PrepareShards import flank.corellium.domain.RunTestCorelliumAndroid.context @@ -18,7 +20,8 @@ import kotlinx.coroutines.flow.toList */ internal val invokeDevices = InvokeDevices from setOf( Authorize, - PrepareShards + PrepareShards, + AvailableDevices, ) using context { val config = AndroidInstance.Config( amount = shards.size, @@ -28,5 +31,6 @@ internal val invokeDevices = InvokeDevices from setOf( .onEach { event -> InvokeDevices.Status(event).out() } .filterIsInstance() .map { instance -> instance.id } + .onEach { id -> devices.send(Device.Instance(id)) } .toList() } diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt index 84fd4e0cac..206ad51441 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestMockApiAndroid.kt @@ -10,6 +10,7 @@ import flank.exection.parallel.Parallel import flank.exection.parallel.invoke import flank.exection.parallel.type import flank.exection.parallel.validate +import flank.exection.parallel.verify import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.collect @@ -70,12 +71,10 @@ class RunTestAndroidCorelliumTestMockApiAndroid { }, installAndroidApps = { list -> println(list) - assertEquals(expectedShardsCount, list.size) emptyFlow() }, executeTest = { (instances) -> println(instances) - assertEquals(expectedShardsCount, instances.size) emptyList() }, ), @@ -114,7 +113,7 @@ class RunTestAndroidCorelliumTestMockApiAndroid { @Test fun test() { - runBlocking { execute(CompleteTests)(initial).collect() } + runBlocking { execute(CompleteTests)(initial).collect { it.verify()} } } @After diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt index c52257c4e0..ab162b2b3a 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/RunTestAndroidCorelliumTestParsingAndroid.kt @@ -9,6 +9,7 @@ import flank.exection.parallel.Parallel import flank.exection.parallel.invoke import flank.exection.parallel.type import flank.exection.parallel.validate +import flank.exection.parallel.verify import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.collect @@ -56,7 +57,6 @@ class RunTestAndroidCorelliumTestParsingAndroid { println(path) } } - assertEquals(expectedShardsCount, apps.size) emptyFlow() }, executeTest = { (instances) -> @@ -66,7 +66,6 @@ class RunTestAndroidCorelliumTestParsingAndroid { println(shard) } } - assertEquals(expectedShardsCount, instances.size) emptyList() }, ), @@ -85,7 +84,7 @@ class RunTestAndroidCorelliumTestParsingAndroid { @Test fun test() { - runBlocking { execute(CompleteTests)(initial).collect() } + runBlocking { execute(CompleteTests)(initial).collect { it.verify() } } } @After diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Select.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Select.kt index 5436ea6c98..4e2dcf99d3 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Select.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Select.kt @@ -4,4 +4,4 @@ package flank.exection.parallel * Select value by type. */ @Suppress("UNCHECKED_CAST") -fun ParallelState.select(type: Parallel.Type) = get(type) as T +infix fun ParallelState.select(type: Parallel.Type) = get(type) as T diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt index b622c554ed..1abcc9a3e1 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt @@ -2,6 +2,7 @@ package flank.exection.parallel.internal import flank.exection.parallel.ExecuteTask import flank.exection.parallel.Parallel +import flank.exection.parallel.ParallelState import flank.exection.parallel.validator /** @@ -10,8 +11,8 @@ import flank.exection.parallel.validator abstract class ContextProvider internal constructor() { protected abstract val context: () -> X - operator fun invoke(body: suspend X.() -> R): ExecuteTask = - { context().also { it.state = this }.body() } + operator fun invoke(body: suspend X.(ParallelState) -> R): ExecuteTask = + { context().also { it.state = this }.body(this) } val validate by lazy { validator(context) } } diff --git a/tool/log/src/main/kotlin/flank/log/Event.kt b/tool/log/src/main/kotlin/flank/log/Event.kt index 0e8cc00e64..5b19c2791e 100644 --- a/tool/log/src/main/kotlin/flank/log/Event.kt +++ b/tool/log/src/main/kotlin/flank/log/Event.kt @@ -41,6 +41,7 @@ data class Event internal constructor( * Creates [Event] from [this] context and given [any] type or value. */ infix fun Any.event(any: Any): Event = when (any) { + is Event<*> -> any is Pair<*, *> -> Event(this, any.first!!, any.second!!) is Event.Data -> Event(this, any::class.java, any) is Event.Type<*> -> Event(this, any, Unit)