From e44503b9505aee510288b0f12db77e5de7d65374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 23 Jul 2021 12:45:25 +0200 Subject: [PATCH 1/7] Rerun failed tests using dispatch channel. --- .../corellium/cli/test/android/Format.kt | 1 - .../cli/test/android/FormatKtTest.kt | 3 +- .../flank/corellium/domain/TestAndroid.kt | 143 +++++++++++++++--- .../task/ExecuteTestShard.kt} | 80 +++++----- .../test/android/device/task/InstallApks.kt | 33 ++++ .../test/android/device/task/ReleaseDevice.kt | 21 +++ .../test/android/task/AvailableDevices.kt | 16 ++ .../test/android/task/DispatchFailed.kt | 78 ++++++++++ .../test/android/task/DispatchShards.kt | 47 ++++++ .../domain/test/android/task/DispatchTests.kt | 84 ++++++++++ .../test/android/task/ExecuteTestQueue.kt | 19 +++ .../test/android/task/GenerateReport.kt | 22 ++- .../test/android/task/InitResultsChannel.kt | 24 +++ .../domain/test/android/task/InstallApks.kt | 37 ----- .../domain/test/android/task/InvokeDevices.kt | 6 +- .../domain/TestAndroidMockApiTest.kt | 5 +- .../domain/TestAndroidParsingTest.kt | 111 +++++++++++--- ...stsKtTest.kt => ExecuteTestQueueKtTest.kt} | 39 +++-- 18 files changed, 644 insertions(+), 125 deletions(-) rename corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/{task/ExecuteTests.kt => device/task/ExecuteTestShard.kt} (53%) create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/InstallApks.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/AvailableDevices.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchShards.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueue.kt create mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InitResultsChannel.kt delete mode 100644 corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InstallApks.kt rename corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/{ExecuteTestsKtTest.kt => ExecuteTestQueueKtTest.kt} (80%) diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/Format.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/Format.kt index 1d7e5ca663..99095b3bbe 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/Format.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/test/android/Format.kt @@ -23,7 +23,6 @@ internal val format = buildFormatter { Event.Start(TestAndroid.ExecuteTests) { "* Executing tests" } Event.Start(TestAndroid.CompleteTests) { "* Finish" } Event.Start(TestAndroid.GenerateReport) { "* Generating report" } - Event.Start(TestAndroid.InstallApks) { "* Installing apks" } Event.Start(TestAndroid.InvokeDevices) { "* Invoking devices" } Event.Start(TestAndroid.LoadPreviousDurations) { "* Obtaining previous test cases durations" } Event.Start(TestAndroid.ParseApkInfo) { "* Parsing apk info" } diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/FormatKtTest.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/FormatKtTest.kt index 01dac689e8..708e6f2dd3 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/FormatKtTest.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/test/android/FormatKtTest.kt @@ -55,7 +55,8 @@ class FormatKtTest { startTime = 1, endTime = 2, details = Instrument.Status.Details(emptyMap(), "Class", "Test", null) - ) + ), + shard = emptyList() ), Unit event TestAndroid.ExecuteTests.Error("1", Exception(), "path/to/log/1", 5..10), Unit event TestAndroid.Created(File("path/to/apk.apk")), diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt index 761cde635c..a2175c4d99 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt @@ -8,13 +8,20 @@ import flank.corellium.api.Authorization import flank.corellium.api.CorelliumApi import flank.corellium.domain.TestAndroid.Args.DefaultOutputDir import flank.corellium.domain.TestAndroid.Args.DefaultOutputDir.new +import flank.corellium.domain.test.android.device.task.executeTestShard +import flank.corellium.domain.test.android.device.task.installApks +import flank.corellium.domain.test.android.device.task.releaseDevice import flank.corellium.domain.test.android.task.authorize +import flank.corellium.domain.test.android.task.availableDevices import flank.corellium.domain.test.android.task.createOutputDir +import flank.corellium.domain.test.android.task.dispatchFailedTests +import flank.corellium.domain.test.android.task.dispatchShards +import flank.corellium.domain.test.android.task.dispatchTests import flank.corellium.domain.test.android.task.dumpShards -import flank.corellium.domain.test.android.task.executeTests +import flank.corellium.domain.test.android.task.executeTestQueue import flank.corellium.domain.test.android.task.finish import flank.corellium.domain.test.android.task.generateReport -import flank.corellium.domain.test.android.task.installApks +import flank.corellium.domain.test.android.task.initResultsChannel import flank.corellium.domain.test.android.task.invokeDevices import flank.corellium.domain.test.android.task.loadPreviousDurations import flank.corellium.domain.test.android.task.parseApksInfo @@ -25,7 +32,10 @@ import flank.exection.parallel.type import flank.instrument.log.Instrument import flank.junit.JUnit import flank.log.Event +import flank.shard.InstanceShard import flank.shard.Shard +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.SendChannel import java.io.File import java.lang.System.currentTimeMillis import java.text.SimpleDateFormat @@ -53,7 +63,7 @@ object TestAndroid { val testTargets: List = emptyList(), val maxShardsCount: Int = 1, val obfuscateDumpShards: Boolean = false, - val outputDir: String = DefaultOutputDir.new, + val outputDir: String = new, val gpuAcceleration: Boolean = true, val scanPreviousDurations: Int = 10, val flakyTestsAttempts: Int = 0, @@ -110,17 +120,19 @@ object TestAndroid { * Is providing access to initial arguments and data collected during the execution process. * For convenience the properties are sorted in order equal to its initialization. * - * @property api - Corellium API functions. - * @property apk - APK parsing functions. - * @property junit - JUnit parsing functions. - * @property args - User arguments for execution. + * @property api Corellium API functions. + * @property apk APK parsing functions. + * @property junit JUnit parsing functions. + * @property args User arguments for execution. * * @property testCases key - path to the test apk, value - list of test method names. * @property previousDurations key - test case name, value - calculated previous duration. * @property shards each item is representing list of apps to run on another device instance. + * @property results Channel for providing result from each executed test case. + * @property dispatch Channel for dispatching test shards to execute. + * @property devices Channel for providing devices that are available and ready to use. * @property ids the ids of corellium device instances. - * @property packageNames key - path to the test apk, value - package name. - * @property testRunners key - path to the test apk, value - fully qualified test runner name. + * @property testResult Execution results. */ internal class Context : Parallel.Context() { val api by !type() @@ -128,13 +140,14 @@ object TestAndroid { val junit by !type() val args by !Args + val shards: List> by -PrepareShards val testCases: Map> by -ParseTestCases val previousDurations: Map by -LoadPreviousDurations - val shards: List> by -PrepareShards + val results: Channel by -ExecuteTests.Results + val dispatch: Channel by -Dispatch.Shards + val devices: Channel by -AvailableDevices val ids: List by -InvokeDevices - val packageNames by ParseApkInfo { packageNames } - val testRunners by ParseApkInfo { testRunners } - val testResult: List> by -ExecuteTests + val testResult: List by -ExecuteTests } internal val context = Parallel.Function(::Context) @@ -154,22 +167,48 @@ object TestAndroid { object OutputDir : Parallel.Type object DumpShards : Parallel.Type object Authorize : Parallel.Type + object AvailableDevices : Parallel.Type> object InvokeDevices : Parallel.Type> { object Status : Event.Type } - object InstallApks : Parallel.Type { + object InstallApks : Parallel.Type> { object Status : Event.Type } - object ExecuteTests : Parallel.Type>> { + object Dispatch { + + object Shards : Parallel.Type> + object Tests : Parallel.Type> + object Failed : Parallel.Type> + + enum class Type { Shard, Rerun } + + data class Data( + val index: Int, + val shard: InstanceShard, + val type: Type, + ) { + companion object : Parallel.Type + } + } + + object ExecuteTests : Parallel.Type> { const val ADB_LOG = "adb_log" + object Results : Parallel.Type> + object Plan : Event.Type + data class Dispatch( + val id: String, + val data: TestAndroid.Dispatch.Data + ) : Event.Data + data class Result( val id: String, val status: Instrument, + val shard: InstanceShard, ) : Event.Data data class Error( @@ -178,8 +217,13 @@ object TestAndroid { val logFile: String, val lines: IntRange ) : Event.Data + + data class Finish( + val id: String + ) : Event.Data } + object ReleaseDevice : Parallel.Type object CleanUp : Parallel.Type object GenerateReport : Parallel.Type object CompleteTests : Parallel.Type @@ -196,6 +240,67 @@ object TestAndroid { object Created : Event.Type object AlreadyExist : Event.Type + // Nested + + /** + * Nested scope that represents shard execution on single device + */ + object Device { + + /** + * The context of android test execution on single device. + * + * @property api Corellium API functions. + * @property args User arguments for execution. + * @property packageNames key - path to the test apk, value - package name. + * @property testRunners key - path to the test apk, value - fully qualified test runner name. + * @property shard Tests dispatched to run on device. + * @property device Device instance specified to execute test shard. + * @property results Channel for providing result from each executed test case. + * @property release Channel for releasing instance device after shard execution. + * + * @property installedApks List of apk names that was installed on the device during the current execution. + */ + internal class Context : Parallel.Context() { + val api by !type() + val args by !Args + val packageNames: Map by ParseApkInfo { packageNames } + val testRunners: Map by ParseApkInfo { testRunners } + val data: Dispatch.Data by !Dispatch.Data + val shard get() = data.shard + val device: Instance by !Instance + val results: SendChannel by !ExecuteTests.Results + val release: SendChannel by !AvailableDevices + + val installedApks by -InstallApks + } + + internal val context = Parallel.Function(::Context) + + object Shard : Parallel.Type + + data class Instance( + val id: String, + val apks: Set = emptySet() + ) { + companion object : Parallel.Type + } + + data class Result( + val id: String, + val data: Dispatch.Data, + val value: List, + ) + + internal val execute by lazy { + setOf( + installApks, + executeTestShard, + releaseDevice, + ) + } + } + // Execution tasks // Evaluate lazy to avoid strange NullPointerException. @@ -204,11 +309,15 @@ object TestAndroid { context.validate, authorize, createOutputDir, + dispatchShards, + dispatchTests, + dispatchFailedTests, dumpShards, - executeTests, + executeTestQueue, finish, generateReport, - installApks, + initResultsChannel, + availableDevices, invokeDevices, loadPreviousDurations, parseApksInfo, diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt similarity index 53% rename from corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTests.kt rename to corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt index a97518bd37..3fccc292e3 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTests.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt @@ -1,14 +1,14 @@ -package flank.corellium.domain.test.android.task +package flank.corellium.domain.test.android.device.task import flank.corellium.api.AndroidTestPlan import flank.corellium.domain.TestAndroid import flank.corellium.domain.TestAndroid.Authorize +import flank.corellium.domain.TestAndroid.Device import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.ExecuteTests.Error +import flank.corellium.domain.TestAndroid.ExecuteTests.Result import flank.corellium.domain.TestAndroid.InstallApks -import flank.corellium.domain.TestAndroid.InvokeDevices import flank.corellium.domain.TestAndroid.ParseApkInfo -import flank.corellium.domain.TestAndroid.PrepareShards -import flank.corellium.domain.TestAndroid.context import flank.exection.parallel.from import flank.exection.parallel.using import flank.instrument.command.formatAmInstrumentCommand @@ -19,6 +19,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.collect @@ -29,44 +30,33 @@ import kotlinx.coroutines.flow.take import java.io.File /** - * Executes given tests on previously invoked devices, and returns the test results. + * Executes given test shard on device, and returns the test results. * * The side effect is console logs from `am instrument` saved inside [TestAndroid.ExecuteTests.ADB_LOG] output subdirectory. * * The optional parsing errors are sent through [TestAndroid.Context.out]. */ -internal val executeTests = ExecuteTests from setOf( - PrepareShards, +internal val executeTestShard = ExecuteTests from setOf( ParseApkInfo, Authorize, - InvokeDevices, InstallApks, -) using context { + ExecuteTests.Results, +) using Device.context { val outputDir = File(args.outputDir, ExecuteTests.ADB_LOG).apply { mkdir() } val testPlan: AndroidTestPlan.Config = prepareTestPlan() coroutineScope { ExecuteTests.Plan(testPlan).out() api.executeTest(testPlan).map { (id, flow) -> async { - var read = 0 - var parsed = 0 - val file = outputDir.resolve(id) - val results = mutableListOf() - flow.onEach { file.appendText(it + "\n") } - .buffer(100) - .flowOn(Dispatchers.IO) - .onEach { ++read } - .parseAdbInstrumentLog() - .onEach { parsed = read } - .onEach { result -> results += result } - .onEach { result -> ExecuteTests.Result(id, result).out() } - .catch { cause -> ExecuteTests.Error(id, cause, file.path, ++parsed..read).out() } - .filterIsInstance() - .take(expectedResultsCountFor(id)) - .collect() - results + Device.Result( + id = id, + data = data, + value = collectResults(flow, id, outputDir) + ) } - }.awaitAll() + }.awaitAll().also { + ExecuteTests.Finish(device.id).out() + } } } @@ -74,11 +64,11 @@ internal val executeTests = ExecuteTests from setOf( * Prepare [AndroidTestPlan.Config] for test execution. * It is just mapping and formatting the data collected in state. */ -private fun TestAndroid.Context.prepareTestPlan(): AndroidTestPlan.Config = +private fun Device.Context.prepareTestPlan(): AndroidTestPlan.Config = AndroidTestPlan.Config( - shards.mapIndexed { index, shards -> - ids[index] to shards.flatMap { shard: Shard.App -> - shard.tests.map { test -> + mapOf( + device.id to shard.flatMap { app: Shard.App -> + app.tests.map { test -> formatAmInstrumentCommand( packageName = packageNames.getValue(test.name), testRunner = testRunners.getValue(test.name), @@ -86,8 +76,30 @@ private fun TestAndroid.Context.prepareTestPlan(): AndroidTestPlan.Config = ) } } - }.toMap() + ) ) -private fun TestAndroid.Context.expectedResultsCountFor(id: String): Int = - shards[ids.indexOf(id)].size +private suspend fun Device.Context.collectResults( + flow: Flow, + id: String, + dir: File, +): List { + suspend fun Result.send() = also { results.send(it) } + val file = dir.resolve(id) + var parsed = file.run { if (exists()) useLines { it.count() } else 0 } + var read = parsed + val results = mutableListOf() + flow.onEach { file.appendText(it + "\n") } + .buffer(100) + .flowOn(Dispatchers.IO) + .onEach { ++read } + .parseAdbInstrumentLog() + .onEach { parsed = read } + .onEach { result -> results += result } + .onEach { result -> Result(id, result, shard).send().out() } + .catch { cause -> Error(id, cause, file.path, ++parsed..read).out() } + .filterIsInstance() + .take(shard.size) + .collect() + return results +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/InstallApks.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/InstallApks.kt new file mode 100644 index 0000000000..b1fdfa64d7 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/InstallApks.kt @@ -0,0 +1,33 @@ +package flank.corellium.domain.test.android.device.task + +import flank.corellium.api.AndroidApps +import flank.corellium.domain.TestAndroid.Authorize +import flank.corellium.domain.TestAndroid.Device +import flank.corellium.domain.TestAndroid.InstallApks +import flank.exection.parallel.from +import flank.exection.parallel.using +import kotlinx.coroutines.flow.collect + +/** + * Installs the required software on android instance. + */ +internal val installApks = InstallApks from setOf( + Authorize, +) using Device.context { + val apks = requiredApks() - device.apks + if (apks.isNotEmpty()) { + val config = listOf(AndroidApps(device.id, apks)) + api.installAndroidApps(config).collect { event -> + InstallApks.Status(event).out() + } + } + apks + // If tests will be executed too fast just after the + // app installed, the instrumentation will fail + // delay(4_000) TODO: for verification +} + +private fun Device.Context.requiredApks(): List = + shard.flatMap { app -> + app.tests.map { test -> test.name } + app.name + } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt new file mode 100644 index 0000000000..51dcc53fb9 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt @@ -0,0 +1,21 @@ +package flank.corellium.domain.test.android.device.task + +import flank.corellium.domain.TestAndroid.Device +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.InstallApks +import flank.corellium.domain.TestAndroid.ReleaseDevice +import flank.exection.parallel.from +import flank.exection.parallel.using + +/** + * Makes device available again by adding it device channel. + */ +internal val releaseDevice = ReleaseDevice from setOf( + InstallApks, + ExecuteTests +) using Device.context { + release.send(device + installedApks) +} + +private operator fun Device.Instance.plus(apks: Iterable) = + copy(apks = this.apks + apks) diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/AvailableDevices.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/AvailableDevices.kt new file mode 100644 index 0000000000..238a61a12e --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/AvailableDevices.kt @@ -0,0 +1,16 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.AvailableDevices +import flank.exection.parallel.from +import flank.exection.parallel.using +import kotlinx.coroutines.channels.Channel + +/** + * Prepares channel for providing devices ready to use. + */ +val availableDevices = AvailableDevices from setOf( + TestAndroid.PrepareShards +) using TestAndroid.context { + Channel(shards.size) +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt new file mode 100644 index 0000000000..c6dbbdb074 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt @@ -0,0 +1,78 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.PrepareShards +import flank.corellium.domain.TestAndroid.context +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.instrument.log.Instrument +import flank.shard.InstanceShard +import kotlinx.coroutines.channels.consumeEach +import java.util.concurrent.atomic.AtomicInteger + +/** + * Collects each [ExecuteTests.Results] and checks its status. + * Each failed is dispatched again at most as many times as specified in flakyTestsAttempts argument. + * Rerunning failed tests help detect flakiness. + */ +internal val dispatchFailedTests = Dispatch.Failed from setOf( + PrepareShards, + Dispatch.Shards, + ExecuteTests.Results, +) using context { + val counter = { AtomicInteger(0) } + val runs = mutableMapOf() + var index = 0 + results.consumeEach { result -> + if (result.status is Instrument.Status) { + if (result.status.code in errorCodes) { + val shard = result.shard + .reduceTo(result.status.details.fullTestName) + val attempt = runs + .getOrPut(shard, counter) + .getAndIncrement() + if (attempt < args.flakyTestsAttempts) + dispatch.send( + Dispatch.Data( + index = index++, + shard = shard, + type = Dispatch.Type.Rerun, + ) + ) + } + } + } + runs.mapValues { (_, value) -> value.get() } +} + +/** + * Set of [Instrument] error codes. + */ +private val errorCodes = setOf( + Instrument.Code.FAILED, + Instrument.Code.EXCEPTION, +) + +/** + * Create new [InstanceShard] that contains only one test with basing on given [name]. + * + * If given [name] refers parameterized test method, algorithm will return [InstanceShard] with test case class instead of test case method. + */ +private fun InstanceShard.reduceTo( + name: String, +): InstanceShard = + mapNotNull { app -> + app.tests.mapNotNull { test -> + test.cases.mapNotNull { case -> + when { + name == case.name -> case + !name.startsWith(case.name) -> null + name.removePrefix(case.name).firstOrNull() == '#' -> case + else -> null + } + }.takeIf { it.isNotEmpty() }?.let { test.copy(cases = it) } + }.takeIf { it.isNotEmpty() }?.let { app.copy(tests = it) } + } + +private val Instrument.Status.Details.fullTestName get() = "$className#$testName" diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchShards.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchShards.kt new file mode 100644 index 0000000000..cbec7f2cb1 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchShards.kt @@ -0,0 +1,47 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.PrepareShards +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.shard.Shard +import kotlinx.coroutines.channels.Channel + +/** + * Prepares channel for dispatching shards to run and fills the buffer with initial elements. + */ +val dispatchShards = Dispatch.Shards from setOf( + PrepareShards +) using TestAndroid.context { + Channel(bufferSize).apply { + shards.forEachIndexed { index, shard -> + send( + Dispatch.Data( + index = index, + shard = shard, + type = Dispatch.Type.Shard, + ) + ) + } + } +} + +/** + * Calculates buffer size for [Dispatch.Data]. + * Proper size is crucial for avoiding deadlock on dispatch channel. + * The safe size should be large enough to keep all possible events at single time. + */ +private val TestAndroid.Context.bufferSize: Int + get() = shards.flatten().flatMap(Shard.App::tests) + .size * args.flakyTestsAttempts + + shards.size * EVENT_COUNT_FOR_SHARD + +/** + * The expected amount of events that can be emitted per one shard. + */ +private val EVENT_COUNT_FOR_SHARD = setOf( + ExecuteTests.Dispatch::class, + ExecuteTests.Finish::class, +).size diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt new file mode 100644 index 0000000000..b5318bfee6 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt @@ -0,0 +1,84 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid.Authorize +import flank.corellium.domain.TestAndroid.AvailableDevices +import flank.corellium.domain.TestAndroid.Device +import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.ParseApkInfo +import flank.corellium.domain.TestAndroid.PrepareShards +import flank.corellium.domain.TestAndroid.context +import flank.exection.parallel.from +import flank.exection.parallel.invoke +import flank.exection.parallel.plus +import flank.exection.parallel.select +import flank.exection.parallel.using +import flank.exection.parallel.verify +import flank.shard.InstanceShard +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.consumeEach +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.launch + +/** + * Dispatches each test shard execution into the first best matching device, and collects results. + */ +val dispatchTests = Dispatch.Tests from setOf( + ParseApkInfo, + Authorize, + PrepareShards, + AvailableDevices, + Dispatch.Shards, + ExecuteTests.Results, +) using context { state -> + channelFlow { + var running = 0 + dispatch.consumeEach { data: Dispatch.Data -> + ++running + val instance = devices.maxBy { instance -> + (data.shard.apks intersect instance.apks).size + } + val seed = mapOf( + Dispatch.Data + data, + Device.Instance + instance + ) + ExecuteTests.Dispatch(instance.id, data).out() + + launch { + Device + .execute(state + seed) + .last() + .verify() + .select(ExecuteTests) + .let { send(it) } + + if (--running == 0) { + results.close() + dispatch.close() + } + } + } + }.toList().reduce { accumulator, value -> + accumulator + value + } +} + +/** + * Flatten [InstanceShard] to the apk names. + */ +private val InstanceShard.apks: List + get() = flatMap { app -> app.tests.map { test -> test.name } + app.name } + +/** + * Receives all pending elements and returns one represented by the biggest number returned from [select] function. + * The rest of the received values will be added back to the channel. + */ +private suspend fun Channel.maxBy(select: (T) -> Int): T = + mutableListOf().apply { + add(receive()) + while (!isEmpty) add(receive()) + sortByDescending(select) + drop(1).forEach { send(it) } + }.first() diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueue.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueue.kt new file mode 100644 index 0000000000..280b55cba2 --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueue.kt @@ -0,0 +1,19 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.InvokeDevices +import flank.exection.parallel.from +import flank.exection.parallel.select +import flank.exection.parallel.using + +/** + * Specifies all tasks required to fulfill execution based on dispatch queue. + */ +internal val executeTestQueue = ExecuteTests from setOf( + InvokeDevices, + Dispatch.Tests, + Dispatch.Failed, +) using { + select(Dispatch.Tests) +} diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt index 297722da95..87712c49be 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/GenerateReport.kt @@ -1,6 +1,8 @@ package flank.corellium.domain.test.android.task import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.Device +import flank.corellium.domain.TestAndroid.Dispatch import flank.corellium.domain.TestAndroid.ExecuteTests import flank.corellium.domain.TestAndroid.GenerateReport import flank.corellium.domain.TestAndroid.OutputDir @@ -36,11 +38,12 @@ internal val generateReport = GenerateReport from setOf( * @receiver Instrument results of each instance execution * @return prepared input for generating JUnitReport */ -private fun List>.prepareInputForJUnit(): List = - flatMapIndexed { index: Int, list: List -> - list.filterIsInstance().map { status -> +private fun List.prepareInputForJUnit(): List = + flatMap { result -> + val suiteName = result.suiteName + result.value.filterIsInstance().map { status -> JUnit.TestResult( - suiteName = "shard_$index", + suiteName = suiteName, testName = status.details.testName, className = status.details.className, startAt = status.startTime, @@ -56,3 +59,14 @@ private fun List>.prepareInputForJUnit(): List "rerun" + Dispatch.Type.Shard -> "shard" + }.let { prefix -> + "${prefix}_${data.index}_$id" + } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InitResultsChannel.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InitResultsChannel.kt new file mode 100644 index 0000000000..686c08119c --- /dev/null +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InitResultsChannel.kt @@ -0,0 +1,24 @@ +package flank.corellium.domain.test.android.task + +import flank.corellium.domain.TestAndroid +import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.PrepareShards +import flank.corellium.domain.TestAndroid.context +import flank.exection.parallel.from +import flank.exection.parallel.using +import flank.shard.Shard +import kotlinx.coroutines.channels.Channel + +/** + * Creates channel for dispatching and receiving [TestAndroid.Device.Result]. + * + * Required for flaky tests detection. + */ +internal val initResultsChannel = ExecuteTests.Results from setOf( + PrepareShards +) using context { + Channel(bufferSize) +} + +private val TestAndroid.Context.bufferSize: Int + get() = shards.flatten().flatMap(Shard.App::tests).size diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InstallApks.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InstallApks.kt deleted file mode 100644 index 7f58f1a760..0000000000 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InstallApks.kt +++ /dev/null @@ -1,37 +0,0 @@ -package flank.corellium.domain.test.android.task - -import flank.corellium.api.AndroidApps -import flank.corellium.domain.TestAndroid -import flank.corellium.domain.TestAndroid.Authorize -import flank.corellium.domain.TestAndroid.InstallApks -import flank.corellium.domain.TestAndroid.InvokeDevices -import flank.corellium.domain.TestAndroid.PrepareShards -import flank.corellium.domain.TestAndroid.context -import flank.exection.parallel.from -import flank.exection.parallel.using -import flank.shard.Shard -import kotlinx.coroutines.flow.collect - -/** - * Installs the required software on android instances. - */ -internal val installApks = InstallApks from setOf( - Authorize, - PrepareShards, - InvokeDevices, -) using context { - require(shards.size <= ids.size) { "Not enough instances, required ${shards.size} but was $ids.size" } - val apks = prepareApkToInstall() - api.installAndroidApps(apks).collect { event -> InstallApks.Status(event).out() } - // If tests will be executed too fast just after the - // app installed, the instrumentation will fail - // delay(4_000) TODO: for verification -} - -private fun TestAndroid.Context.prepareApkToInstall(): List = - shards.mapIndexed { index, list: List -> - AndroidApps( - instanceId = ids[index], - paths = list.flatMap { app -> app.tests.map { test -> test.name } + app.name } - ) - } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InvokeDevices.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InvokeDevices.kt index 78c3131988..add6bf2358 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InvokeDevices.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/InvokeDevices.kt @@ -2,6 +2,8 @@ package flank.corellium.domain.test.android.task import flank.corellium.api.AndroidInstance import flank.corellium.domain.TestAndroid.Authorize +import flank.corellium.domain.TestAndroid.AvailableDevices +import flank.corellium.domain.TestAndroid.Device import flank.corellium.domain.TestAndroid.InvokeDevices import flank.corellium.domain.TestAndroid.PrepareShards import flank.corellium.domain.TestAndroid.context @@ -19,7 +21,8 @@ import kotlinx.coroutines.flow.toList */ internal val invokeDevices = InvokeDevices from setOf( Authorize, - PrepareShards + PrepareShards, + AvailableDevices, ) using context { val config = AndroidInstance.Config( amount = shards.size, @@ -29,5 +32,6 @@ internal val invokeDevices = InvokeDevices from setOf( .onEach { event -> InvokeDevices.Status(event).out() } .filterIsInstance() .map { instance -> instance.id } + .onEach { id -> devices.send(Device.Instance(id)) } .toList() } diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidMockApiTest.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidMockApiTest.kt index 44c852276a..a470afe6bc 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidMockApiTest.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidMockApiTest.kt @@ -10,6 +10,7 @@ import flank.exection.parallel.Parallel import flank.exection.parallel.invoke import flank.exection.parallel.type import flank.exection.parallel.validate +import flank.exection.parallel.verify import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.collect @@ -70,12 +71,10 @@ class TestAndroidMockApiTest { }, installAndroidApps = { list -> println(list) - assertEquals(expectedShardsCount, list.size) emptyFlow() }, executeTest = { (instances) -> println(instances) - assertEquals(expectedShardsCount, instances.size) emptyList() }, ), @@ -114,7 +113,7 @@ class TestAndroidMockApiTest { @Test fun test() { - runBlocking { execute(CompleteTests)(initial).collect() } + runBlocking { execute(CompleteTests)(initial).collect { it.verify() } } } @After diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidParsingTest.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidParsingTest.kt index 2a9dfbabe7..52baf5ccd7 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidParsingTest.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/TestAndroidParsingTest.kt @@ -1,18 +1,25 @@ package flank.corellium.domain import flank.apk.Apk +import flank.corellium.api.AmInstrumentCommand import flank.corellium.api.AndroidInstance import flank.corellium.api.CorelliumApi +import flank.corellium.api.InstanceId import flank.corellium.domain.TestAndroid.CompleteTests import flank.corellium.domain.TestAndroid.execute import flank.exection.parallel.Parallel import flank.exection.parallel.invoke +import flank.exection.parallel.plus import flank.exection.parallel.type import flank.exection.parallel.validate +import flank.exection.parallel.verify +import flank.instrument.log.Instrument import flank.junit.JUnit import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.runBlocking import org.junit.After @@ -34,9 +41,9 @@ class TestAndroidParsingTest { } private val initial = mapOf( - TestAndroid.Args to args, + TestAndroid.Args + args, - type() to CorelliumApi( + type() + CorelliumApi( authorize = { credentials -> println(credentials) assertEquals(args.credentials, credentials) @@ -51,31 +58,49 @@ class TestAndroidParsingTest { }, installAndroidApps = { apps -> apps.forEach { (key, value) -> - println("$key:") - value.forEach { path -> - println(path) - } + println("$key:\n" + value.joinToString("") { "$it\n" }) } - assertEquals(expectedShardsCount, apps.size) emptyFlow() }, - executeTest = { (instances) -> + executeTest = { (instances: Map>) -> instances.forEach { (key, value) -> - println("$key:") - value.forEach { shard -> - println(shard) - } + println("$key:\n" + value.joinToString("") { "$it\n" }) + } + instances.map { (id, commands) -> + id to flow { + commands.forEach { command -> + val testNames = command.split(" ") + .run { drop(indexOf("-e") + 2).first() } + .split(",") + + testNames.forEachIndexed { index, name -> + val (className, testName) = name.split("#").run { + first() to getOrNull(1) + } + produceInstrumentLog( + current = index + 1, + numTests = testNames.size, + className = className, + testName = testName ?: "test[0]", + code = randomCode(), + ).lineSequence().forEach { emit(it) } + } + + produceInstrumentResult( + time = 2.888f, + numTests = testNames.size + ).lineSequence().forEach { emit(it) } + } + }.buffer(Int.MAX_VALUE) } - assertEquals(expectedShardsCount, instances.size) - emptyList() }, ), - Parallel.Logger to fun Any.() = println(this), + Parallel.Logger + { println(this) }, - type() to JUnit.Api(), + type() + JUnit.Api(), - type() to Apk.Api(), + type() + Apk.Api(), ) @Test @@ -85,7 +110,7 @@ class TestAndroidParsingTest { @Test fun test() { - runBlocking { execute(CompleteTests)(initial).collect() } + runBlocking { execute(CompleteTests)(initial).collect { state -> state.verify() } } } @After @@ -93,3 +118,53 @@ class TestAndroidParsingTest { File(args.outputDir).deleteRecursively() } } + +private fun randomCode(): Int = setOf( + Instrument.Code.PASSED, + Instrument.Code.FAILED, + Instrument.Code.SKIPPED, +).random() + +private fun produceInstrumentLog( + current: Int, + numTests: Int, + testName: String, + className: String, + code: Int, + stack: Throwable? = null, +) = """ +INSTRUMENTATION_STATUS: class=$className +INSTRUMENTATION_STATUS: current=$current +INSTRUMENTATION_STATUS: id=AndroidJUnitRunner +INSTRUMENTATION_STATUS: numtests=$numTests +INSTRUMENTATION_STATUS: stream= +$className: +INSTRUMENTATION_STATUS: test=$testName +INSTRUMENTATION_STATUS_CODE: 1 +INSTRUMENTATION_STATUS: class=$className +INSTRUMENTATION_STATUS: current=$current +INSTRUMENTATION_STATUS: id=AndroidJUnitRunner +INSTRUMENTATION_STATUS: numtests=$numTests +${stack.logStacktrace()}INSTRUMENTATION_STATUS: stream= +$className: +INSTRUMENTATION_STATUS: test=$testName +INSTRUMENTATION_STATUS_CODE: $code +""".trimIndent() + +fun Throwable?.logStacktrace(): String = if (this == null) "" else + "INSTRUMENTATION_STATUS: stack=${stackTraceToString()}\n" + +private fun produceInstrumentResult( + time: Float, + numTests: Int, + code: Int = -1, +) = """ +INSTRUMENTATION_RESULT: stream= + +Time: $time + +OK ($numTests tests) + + +INSTRUMENTATION_CODE: $code +""".trimIndent() diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestsKtTest.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueueKtTest.kt similarity index 80% rename from corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestsKtTest.kt rename to corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueueKtTest.kt index e6a5345b87..e960fe3d32 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestsKtTest.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/test/android/task/ExecuteTestQueueKtTest.kt @@ -1,13 +1,12 @@ package flank.corellium.domain.test.android.task +import flank.corellium.api.AndroidInstance import flank.corellium.api.CorelliumApi import flank.corellium.domain.TestAndroid import flank.corellium.domain.TestAndroid.Args import flank.corellium.domain.TestAndroid.Authorize import flank.corellium.domain.TestAndroid.ExecuteTests import flank.corellium.domain.TestAndroid.ExecuteTests.ADB_LOG -import flank.corellium.domain.TestAndroid.InstallApks -import flank.corellium.domain.TestAndroid.InvokeDevices import flank.corellium.domain.TestAndroid.ParseApkInfo import flank.corellium.domain.TestAndroid.PrepareShards import flank.corellium.domain.invalidLog @@ -18,14 +17,18 @@ import flank.exection.parallel.ParallelState import flank.exection.parallel.invoke import flank.exection.parallel.select import flank.exection.parallel.type +import flank.exection.parallel.validate +import flank.exection.parallel.verify import flank.log.Event import flank.log.Output import flank.shard.Shard import kotlinx.coroutines.delay import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.flattenConcat import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.runBlocking import org.junit.After @@ -35,7 +38,7 @@ import org.junit.Before import org.junit.Test import java.io.File -class ExecuteTestsKtTest { +class ExecuteTestQueueKtTest { private val dir = File(Args.DefaultOutputDir.new) private val instanceId = "1" @@ -50,15 +53,28 @@ class ExecuteTestsKtTest { PrepareShards to listOf((0..2).map { Shard.App("$it", emptyList()) }), ParseApkInfo to TestAndroid.Info(), Authorize to Unit, - InvokeDevices to listOf(instanceId), - InstallApks to Unit, ) - private val execute = setOf(executeTests) + private val dependencies = setOf( + initResultsChannel, + availableDevices, + invokeDevices, + dispatchShards, + dispatchTests, + dispatchFailedTests, + ) + + private val execute = dependencies + executeTestQueue // simulate additional unneeded input that will be omitted. private val additionalInput = (0..1000).map(Int::toString).asFlow().onStart { delay(500) } private fun corelliumApi(log: String) = CorelliumApi( + invokeAndroidDevices = { config -> + (0 until config.amount).asFlow().map { id -> + AndroidInstance.Event.Ready("$id") + } + }, + installAndroidApps = { emptyFlow() }, executeTest = { listOf(instanceId to flowOf(log.lines().asFlow(), additionalInput).flattenConcat()) } ) @@ -72,6 +88,11 @@ class ExecuteTestsKtTest { dir.deleteRecursively() } + @Test + fun validate() { + execute.validate(initial) + } + /** * Valid console output should be completely saved in file, parsed and returned as testResult. */ @@ -83,10 +104,10 @@ class ExecuteTestsKtTest { ) // when - val testResult = runBlocking { execute(args).last() } + val testResult = runBlocking { execute(args).last() }.verify() // then - assertEquals(9, testResult.select(ExecuteTests).first().size) + assertEquals(9, testResult.select(ExecuteTests).first().value.size) // Right after reading the required results count from validLog the stream is closing. // Saved log is same as validLog without unneeded additionalInput. @@ -115,7 +136,7 @@ class ExecuteTestsKtTest { val testResult = runBlocking { execute(args).last().select(ExecuteTests) } // then - assertTrue(testResult.first().isNotEmpty()) // Valid lines parsed before error will be returned + assertTrue(testResult.first().value.isNotEmpty()) // Valid lines parsed before error will be returned val error = events.mapNotNull { it.value as? ExecuteTests.Error }.first() // Obtain error assertEquals(instanceId, error.id) // Error should contain correct instanceId From fd4dfc61ac3f80b1c4be9fdac7a1f51ccb7ad93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 29 Jul 2021 10:56:46 +0200 Subject: [PATCH 2/7] Update execution diagram --- corellium/domain/README.md | 2 +- corellium/domain/TestAndroid-execute.puml | 30 ++++++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/corellium/domain/README.md b/corellium/domain/README.md index eccdc62274..65359ee88b 100644 --- a/corellium/domain/README.md +++ b/corellium/domain/README.md @@ -20,4 +20,4 @@ Execution can be represented as a graph of tasks relations without cycles. ### New version draft: -![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_Add_module_tool-execution-parallel-plantuml/corellium/domain/TestAndroid-execute.puml) +![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid-execute.puml) diff --git a/corellium/domain/TestAndroid-execute.puml b/corellium/domain/TestAndroid-execute.puml index 872c132e14..715fa1b978 100644 --- a/corellium/domain/TestAndroid-execute.puml +++ b/corellium/domain/TestAndroid-execute.puml @@ -14,29 +14,41 @@ end note [ParseTestCases] #fbfbfb [LoadPreviousDurations] #dfdfdf [PrepareShards] #c3c3c3 +[Dispatch.Shards] #a7a7a7 [DumpShards] #a7a7a7 -[InvokeDevices] #a7a7a7 -[InstallApks] #8b8b8b +[ExecuteTests.Results] #a7a7a7 +[AvailableDevices] #a7a7a7 +[Dispatch.Tests] #8b8b8b +[Dispatch.Failed] #8b8b8b +[InvokeDevices] #8b8b8b [ExecuteTests] #6f6f6f [GenerateReport] #535353 [CompleteTests] #373737 +[Dispatch.Shards] --> [PrepareShards] +[Dispatch.Tests] --> [ParseApkInfo] +[Dispatch.Tests] --> [Authorize] +[Dispatch.Tests] --> [PrepareShards] +[Dispatch.Tests] --> [AvailableDevices] +[Dispatch.Tests] --> [Dispatch.Shards] +[Dispatch.Tests] --> [ExecuteTests.Results] +[Dispatch.Failed] --> [PrepareShards] +[Dispatch.Failed] --> [Dispatch.Shards] +[Dispatch.Failed] --> [ExecuteTests.Results] [DumpShards] --> [PrepareShards] [DumpShards] --> [OutputDir] -[ExecuteTests] --> [PrepareShards] -[ExecuteTests] --> [ParseApkInfo] -[ExecuteTests] --> [Authorize] [ExecuteTests] --> [InvokeDevices] -[ExecuteTests] --> [InstallApks] +[ExecuteTests] --> [Dispatch.Tests] +[ExecuteTests] --> [Dispatch.Failed] [CompleteTests] --> [GenerateReport] [CompleteTests] --> [DumpShards] [GenerateReport] --> [ExecuteTests] [GenerateReport] --> [OutputDir] -[InstallApks] --> [Authorize] -[InstallApks] --> [PrepareShards] -[InstallApks] --> [InvokeDevices] +[ExecuteTests.Results] --> [PrepareShards] +[AvailableDevices] --> [PrepareShards] [InvokeDevices] --> [Authorize] [InvokeDevices] --> [PrepareShards] +[InvokeDevices] --> [AvailableDevices] [LoadPreviousDurations] --> [ParseTestCases] [PrepareShards] --> [ParseTestCases] [PrepareShards] --> [LoadPreviousDurations] From 37cdcbc9ab92050c1b20f919a1c30e163eef04f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 29 Jul 2021 12:25:24 +0200 Subject: [PATCH 3/7] Generate additional diagram for sub-execution on device. --- corellium/domain/README.md | 17 +++++++++-- corellium/domain/TestAndroid-execute.puml | 14 ++++++---- .../domain/TestAndroid_Device-execute.puml | 28 +++++++++++++++++++ .../flank/corellium/domain/TestAndroid.kt | 2 ++ .../android/device/task/ExecuteTestShard.kt | 3 +- .../test/android/device/task/ReleaseDevice.kt | 6 ++-- .../domain/test/android/task/DispatchTests.kt | 3 +- .../corellium/domain/AndroidTestDiagram.kt | 10 +++++++ .../internal/GeneratePlantUmlString.kt | 18 +++++++----- 9 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 corellium/domain/TestAndroid_Device-execute.puml diff --git a/corellium/domain/README.md b/corellium/domain/README.md index 65359ee88b..f0eceb8ecf 100644 --- a/corellium/domain/README.md +++ b/corellium/domain/README.md @@ -13,11 +13,22 @@ This module is specifying public API and internal implementation of Flank-Corell Execution can be represented as a graph of tasks relations without cycles. - #### Version from master branch: -![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid-execute.puml) +Core execution + +![TestAndroid.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid-execute.puml) + +Device execution + +![TestAndroid.Device.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid_Device-execute.puml) ### New version draft: -![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid-execute.puml) +Core execution + +![TestAndroid.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid-execute.puml) + +Device execution + +![TestAndroid.Device.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid_Device-execute.puml) diff --git a/corellium/domain/TestAndroid-execute.puml b/corellium/domain/TestAndroid-execute.puml index 715fa1b978..1c591c0463 100644 --- a/corellium/domain/TestAndroid-execute.puml +++ b/corellium/domain/TestAndroid-execute.puml @@ -2,11 +2,15 @@ skinparam componentStyle rectangle -note as N #ffffff -* Brighter tasks are required by the darker tasks. -* The brightness means how fast the task will start. -* White tasks are starting first. -end note +legend left + |= Color |= Description | + |<#373737>| The final task that completes the whole execution | + |<#a7a7a7>| Brighter tasks are required by the darker tasks | + |<#fbfbfb>| Tasks that are starting first | + |<#LightYellow>| Explicitly declared dependencies that needs be delivered from outside of execution | + * The brightness means how fast the task will start. + * Explicitly declaring initial dependencies for tasks is optional, so they may not be included in diagram. +end legend [Authorize] #fbfbfb [OutputDir] #fbfbfb diff --git a/corellium/domain/TestAndroid_Device-execute.puml b/corellium/domain/TestAndroid_Device-execute.puml new file mode 100644 index 0000000000..cf44bc2759 --- /dev/null +++ b/corellium/domain/TestAndroid_Device-execute.puml @@ -0,0 +1,28 @@ +@startuml + +skinparam componentStyle rectangle + +legend left + |= Color |= Description | + |<#373737>| The final task that completes the whole execution | + |<#9b9b9b>| Brighter tasks are required by the darker tasks | + |<#ffffff>| Tasks that are starting first | + |<#LightYellow>| Explicitly declared dependencies that need to be delivered from the outside of execution. | + * The brightness means how fast the task will start. + * Explicitly declaring initial dependencies for tasks is optional, so they may not be included in diagram. +end legend + +[InstallApks] #ffffff +[ExecuteTestShard] #9b9b9b +[ReleaseDevice] #373737 + +[InstallApks] --> [Authorize] +[ExecuteTestShard] --> [ParseApkInfo] +[ExecuteTestShard] --> [Authorize] +[ExecuteTestShard] --> [InstallApks] +[ExecuteTestShard] --> [ExecuteTests.Results] +[ReleaseDevice] --> [InstallApks] +[ReleaseDevice] --> [ExecuteTestShard] +[ReleaseDevice] --> [AvailableDevices] + +@enduml diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt index a2175c4d99..b27137d7d0 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/TestAndroid.kt @@ -193,6 +193,8 @@ object TestAndroid { } } + object ExecuteTestShard : Parallel.Type> + object ExecuteTests : Parallel.Type> { const val ADB_LOG = "adb_log" diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt index 3fccc292e3..d450af36f7 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ExecuteTestShard.kt @@ -4,6 +4,7 @@ import flank.corellium.api.AndroidTestPlan import flank.corellium.domain.TestAndroid import flank.corellium.domain.TestAndroid.Authorize import flank.corellium.domain.TestAndroid.Device +import flank.corellium.domain.TestAndroid.ExecuteTestShard import flank.corellium.domain.TestAndroid.ExecuteTests import flank.corellium.domain.TestAndroid.ExecuteTests.Error import flank.corellium.domain.TestAndroid.ExecuteTests.Result @@ -36,7 +37,7 @@ import java.io.File * * The optional parsing errors are sent through [TestAndroid.Context.out]. */ -internal val executeTestShard = ExecuteTests from setOf( +internal val executeTestShard = ExecuteTestShard from setOf( ParseApkInfo, Authorize, InstallApks, diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt index 51dcc53fb9..70e58d09ad 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/device/task/ReleaseDevice.kt @@ -1,7 +1,8 @@ package flank.corellium.domain.test.android.device.task +import flank.corellium.domain.TestAndroid.AvailableDevices import flank.corellium.domain.TestAndroid.Device -import flank.corellium.domain.TestAndroid.ExecuteTests +import flank.corellium.domain.TestAndroid.ExecuteTestShard import flank.corellium.domain.TestAndroid.InstallApks import flank.corellium.domain.TestAndroid.ReleaseDevice import flank.exection.parallel.from @@ -12,7 +13,8 @@ import flank.exection.parallel.using */ internal val releaseDevice = ReleaseDevice from setOf( InstallApks, - ExecuteTests + ExecuteTestShard, + AvailableDevices, ) using Device.context { release.send(device + installedApks) } diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt index b5318bfee6..4fcee8f08d 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchTests.kt @@ -4,6 +4,7 @@ import flank.corellium.domain.TestAndroid.Authorize import flank.corellium.domain.TestAndroid.AvailableDevices import flank.corellium.domain.TestAndroid.Device import flank.corellium.domain.TestAndroid.Dispatch +import flank.corellium.domain.TestAndroid.ExecuteTestShard import flank.corellium.domain.TestAndroid.ExecuteTests import flank.corellium.domain.TestAndroid.ParseApkInfo import flank.corellium.domain.TestAndroid.PrepareShards @@ -51,7 +52,7 @@ val dispatchTests = Dispatch.Tests from setOf( .execute(state + seed) .last() .verify() - .select(ExecuteTests) + .select(ExecuteTestShard) .let { send(it) } if (--running == 0) { diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt index 8d610317d2..87e4234547 100644 --- a/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt @@ -2,6 +2,7 @@ package flank.corellium.domain import flank.exection.parallel.plantuml.generatePlanUml import org.junit.Test +import java.io.File class AndroidTestDiagram { @@ -9,4 +10,13 @@ class AndroidTestDiagram { fun generate() { TestAndroid.run { generatePlanUml(execute - context.validate) } } + + @Test + fun generateDevice() { + generatePlanUml( + tasks = TestAndroid.Device.execute, + path = File("TestAndroid_Device").absolutePath + "-execute.puml", + prefixToRemove = TestAndroid.javaClass.name + ) + } } diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt index 0a932ce5b7..37f8f51405 100644 --- a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt @@ -21,11 +21,15 @@ internal fun generatePlantUmlString( skinparam componentStyle rectangle -note as N #ffffff -* Brighter tasks are required by the darker tasks. -* The brightness means how fast the task will start. -* White tasks are starting first. -end note +legend left + |= Color |= Description | + |<${calculateColor(maxDepth, maxDepth)}>| The final task that completes the whole execution | + |<${calculateColor(maxDepth, maxDepth / 2)}>| Brighter tasks are required by the darker tasks | + |<${calculateColor(maxDepth, 0)}>| Tasks that are starting first | + |<#LightYellow>| Explicitly declared dependencies that needs be delivered from outside of execution | + * The brightness means how fast the task will start. + * Explicitly declaring initial dependencies for tasks is optional, so they may not be included in diagram. +end legend ${colors.printColors(name)} @@ -44,7 +48,7 @@ private typealias Colors = Map private fun calculateColor(maxDepth: Int, value: Int): String { val c = (COLOR_MAX / maxDepth) * (maxDepth - value) + COLOR_OFFSET - return Integer.toHexString(Color(c, c, c).rgb).drop(2) // drop alpha + return "#" + Integer.toHexString(Color(c, c, c).rgb).drop(2) // drop alpha } private const val COLOR_MAX = 200 @@ -71,7 +75,7 @@ private fun Map>.calculateDepth(): Map { private fun Colors.printColors( name: Node.() -> String ) = toList().joinToString("\n") { (node, value) -> - "${node.name()} #$value" + "${node.name()} $value" } private fun Graph.printRelations( From 40ac3f0a27e2a9d0207b2575c9adb259b754dd55 Mon Sep 17 00:00:00 2001 From: jan-gogo Date: Thu, 29 Jul 2021 10:31:22 +0000 Subject: [PATCH 4/7] [Auto-commit] Added files fixed by Detekt --- corellium/domain/TestAndroid_Device-execute.puml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corellium/domain/TestAndroid_Device-execute.puml b/corellium/domain/TestAndroid_Device-execute.puml index cf44bc2759..988186f156 100644 --- a/corellium/domain/TestAndroid_Device-execute.puml +++ b/corellium/domain/TestAndroid_Device-execute.puml @@ -7,7 +7,7 @@ legend left |<#373737>| The final task that completes the whole execution | |<#9b9b9b>| Brighter tasks are required by the darker tasks | |<#ffffff>| Tasks that are starting first | - |<#LightYellow>| Explicitly declared dependencies that need to be delivered from the outside of execution. | + |<#LightYellow>| Explicitly declared dependencies that needs be delivered from outside of execution | * The brightness means how fast the task will start. * Explicitly declaring initial dependencies for tasks is optional, so they may not be included in diagram. end legend From 043b8fa5166a2de56f4c61db08a0d9f3d29855af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 29 Jul 2021 12:33:31 +0200 Subject: [PATCH 5/7] Fix README.md --- corellium/domain/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corellium/domain/README.md b/corellium/domain/README.md index f0eceb8ecf..ec26f8b48b 100644 --- a/corellium/domain/README.md +++ b/corellium/domain/README.md @@ -15,20 +15,20 @@ Execution can be represented as a graph of tasks relations without cycles. #### Version from master branch: -Core execution +Core execution. ![TestAndroid.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid-execute.puml) -Device execution +Device sub-execution triggered for each shard or rerun by the `Device.Tests` task. ![TestAndroid.Device.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid_Device-execute.puml) ### New version draft: -Core execution +Core execution. ![TestAndroid.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid-execute.puml) -Device execution +Device sub-execution triggered for each shard or rerun by the `Device.Tests` task. ![TestAndroid.Device.execute](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_test_dispatch_flow/corellium/domain/TestAndroid_Device-execute.puml) From d60f8a1b829408c6018fbb59beb45848b6dc8b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 29 Jul 2021 12:53:44 +0200 Subject: [PATCH 6/7] Correct the kdoc --- .../corellium/domain/test/android/task/DispatchFailed.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt index c6dbbdb074..2b322bdaf1 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt @@ -13,7 +13,7 @@ import java.util.concurrent.atomic.AtomicInteger /** * Collects each [ExecuteTests.Results] and checks its status. - * Each failed is dispatched again at most as many times as specified in flakyTestsAttempts argument. + * Each failed test is dispatched again at most as many times as specified in flakyTestsAttempts argument. * Rerunning failed tests help detect flakiness. */ internal val dispatchFailedTests = Dispatch.Failed from setOf( @@ -55,9 +55,10 @@ private val errorCodes = setOf( ) /** - * Create new [InstanceShard] that contains only one test with basing on given [name]. + * Creates new [InstanceShard] that contains only one test basing on the given [name]. * - * If given [name] refers parameterized test method, algorithm will return [InstanceShard] with test case class instead of test case method. + * If the given [name] refers parameterized test method, the algorithm will return [InstanceShard] with a test case for the whole class instead of the test case method. + * This behaviour is required because parameterized test methods are produced in runtime based on annotations and cannot be run separately due to limitations of android instrumentation. */ private fun InstanceShard.reduceTo( name: String, From 2cff49f9f787b6fd2da07f1bb632294558e3ec2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 29 Jul 2021 12:59:40 +0200 Subject: [PATCH 7/7] Fix --- corellium/domain/TestAndroid-execute.puml | 1 - .../flank/corellium/domain/test/android/task/DispatchFailed.kt | 2 -- 2 files changed, 3 deletions(-) diff --git a/corellium/domain/TestAndroid-execute.puml b/corellium/domain/TestAndroid-execute.puml index 1c591c0463..6d4a5be5fb 100644 --- a/corellium/domain/TestAndroid-execute.puml +++ b/corellium/domain/TestAndroid-execute.puml @@ -36,7 +36,6 @@ end legend [Dispatch.Tests] --> [AvailableDevices] [Dispatch.Tests] --> [Dispatch.Shards] [Dispatch.Tests] --> [ExecuteTests.Results] -[Dispatch.Failed] --> [PrepareShards] [Dispatch.Failed] --> [Dispatch.Shards] [Dispatch.Failed] --> [ExecuteTests.Results] [DumpShards] --> [PrepareShards] diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt index 2b322bdaf1..098a0c88b0 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/test/android/task/DispatchFailed.kt @@ -2,7 +2,6 @@ package flank.corellium.domain.test.android.task import flank.corellium.domain.TestAndroid.Dispatch import flank.corellium.domain.TestAndroid.ExecuteTests -import flank.corellium.domain.TestAndroid.PrepareShards import flank.corellium.domain.TestAndroid.context import flank.exection.parallel.from import flank.exection.parallel.using @@ -17,7 +16,6 @@ import java.util.concurrent.atomic.AtomicInteger * Rerunning failed tests help detect flakiness. */ internal val dispatchFailedTests = Dispatch.Failed from setOf( - PrepareShards, Dispatch.Shards, ExecuteTests.Results, ) using context {