From e85039a0740dfdd55e271e5d008d5a39af6da20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Wed, 30 Jun 2021 13:20:15 +0200 Subject: [PATCH] feat: Save `am instrument` logs to file & handle parsing error. (#2049) --- corellium/adapter/build.gradle.kts | 1 + .../src/main/kotlin/flank/corellium/Api.kt | 3 +- .../corellium/adapter/ClientReference.kt | 4 + .../adapter/ExecuteAndroidTestPlan.kt | 11 +- .../adapter/ExecuteAndroidTestPlanKtTest.kt | 72 ++++++++ .../flank/corellium/api/CorelliumApi.kt | 8 +- .../cli/RunTestCorelliumAndroidCommand.kt | 7 + .../cli/RunTestCorelliumAndroidCommandTest.kt | 1 + .../domain/RunTestCorelliumAndroid.kt | 8 + .../run/test/android/step/ExecuteTests.kt | 48 ++++-- .../kotlin/flank/corellium/domain/AdbLog.kt | 160 ++++++++++++++++++ .../test/android/step/ExecuteTestsKtTest.kt | 116 +++++++++++++ docs/flank_corellium.md | 32 +++- 13 files changed, 447 insertions(+), 24 deletions(-) create mode 100644 corellium/adapter/src/test/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlanKtTest.kt create mode 100644 corellium/domain/src/test/kotlin/flank/corellium/domain/AdbLog.kt create mode 100644 corellium/domain/src/test/kotlin/flank/corellium/domain/run/test/android/step/ExecuteTestsKtTest.kt diff --git a/corellium/adapter/build.gradle.kts b/corellium/adapter/build.gradle.kts index c1588f02b3..c6c350d8a5 100644 --- a/corellium/adapter/build.gradle.kts +++ b/corellium/adapter/build.gradle.kts @@ -15,5 +15,6 @@ dependencies { implementation(project(":corellium:client")) implementation(Dependencies.KOTLIN_COROUTINES_CORE) testImplementation(Dependencies.JUNIT) + testImplementation(Dependencies.MOCKK) } diff --git a/corellium/adapter/src/main/kotlin/flank/corellium/Api.kt b/corellium/adapter/src/main/kotlin/flank/corellium/Api.kt index d21f515da8..962f440b03 100644 --- a/corellium/adapter/src/main/kotlin/flank/corellium/Api.kt +++ b/corellium/adapter/src/main/kotlin/flank/corellium/Api.kt @@ -1,6 +1,7 @@ package flank.corellium import flank.corellium.adapter.executeAndroidTestPlan +import flank.corellium.adapter.getCorellium import flank.corellium.adapter.installAndroidApps import flank.corellium.adapter.invokeAndroidDevices import flank.corellium.adapter.requestAuthorization @@ -16,5 +17,5 @@ fun corelliumApi( invokeAndroidDevices = invokeAndroidDevices( projectName = projectName ), - executeTest = executeAndroidTestPlan, + executeTest = executeAndroidTestPlan(getCorellium), ) diff --git a/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ClientReference.kt b/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ClientReference.kt index 47438f8a89..43a8fd6b65 100644 --- a/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ClientReference.kt +++ b/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ClientReference.kt @@ -9,3 +9,7 @@ internal val corellium: Corellium // It's totally ok to keep corellium as singleton since we don't need handle more than one connection for single run. internal var corelliumRef: Corellium? = null + +internal typealias GetCorellium = () -> Corellium + +internal val getCorellium: GetCorellium = { corellium } diff --git a/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlan.kt b/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlan.kt index a635bfe022..a354ad4d5a 100644 --- a/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlan.kt +++ b/corellium/adapter/src/main/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlan.kt @@ -2,21 +2,26 @@ package flank.corellium.adapter import flank.corellium.api.AndroidTestPlan import flank.corellium.client.console.clear +import flank.corellium.client.console.close import flank.corellium.client.console.flowLogs import flank.corellium.client.console.sendCommand import flank.corellium.client.console.waitForIdle import flank.corellium.client.core.connectConsole +import kotlinx.coroutines.channels.trySendBlocking import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.collect import kotlinx.coroutines.launch -val executeAndroidTestPlan = AndroidTestPlan.Execute { config -> +fun executeAndroidTestPlan( + corellium: GetCorellium +) = AndroidTestPlan.Execute { config -> config.instances.map { (instanceId, commands: List) -> instanceId to channelFlow { - corellium.connectConsole(instanceId).apply { + corellium().connectConsole(instanceId).apply { + invokeOnClose { launch { close() } } clear() launch { commands.forEach { string -> sendCommand(string) } } - launch { flowLogs().collect(channel::send) } + launch { flowLogs().collect(channel::trySendBlocking) } waitForIdle(10_000) } } diff --git a/corellium/adapter/src/test/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlanKtTest.kt b/corellium/adapter/src/test/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlanKtTest.kt new file mode 100644 index 0000000000..cfb78bdaa3 --- /dev/null +++ b/corellium/adapter/src/test/kotlin/flank/corellium/adapter/ExecuteAndroidTestPlanKtTest.kt @@ -0,0 +1,72 @@ +package flank.corellium.adapter + +import flank.corellium.api.AndroidTestPlan +import flank.corellium.client.console.Console +import flank.corellium.client.console.close +import flank.corellium.client.console.flowLogs +import flank.corellium.client.console.sendCommand +import flank.corellium.client.core.Corellium +import flank.corellium.client.core.connectConsole +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import org.junit.After +import org.junit.Assert +import org.junit.Test + +class ExecuteAndroidTestPlanKtTest { + + /** + * Connection to [Console] should be closed once subscription detach. + */ + @Test + fun closeConsole() { + // given + mockkStatic( + "flank.corellium.client.console.ApiKt", + "flank.corellium.client.core.ApiKt", + ) + + val instanceId = "1" + val command = "command" + val emitted = mutableListOf() + val range = ('a'..'z') + val linesFlow: Flow = channelFlow { range.forEach { send("$it"); emitted += it; delay(10) } } + val config = AndroidTestPlan.Config(mapOf(instanceId to listOf(command))) + + val console: Console = mockk(relaxed = true) { + val context = Job() + every { flowLogs() } returns linesFlow + every { coroutineContext } returns context + } + + val corellium: Corellium = mockk(relaxed = true) { + coEvery { connectConsole(instanceId) } returns console + } + + val flow: Flow = executeAndroidTestPlan { corellium } + .invoke(config).first().second + + // when + runBlocking { flow.first { it == "c" } } // Detach the subscription after "c" element + + // then + coVerify(exactly = 1) { console.sendCommand(command) } // Verify sendCommand called. + coVerify(exactly = 1) { console.close() } // Verify console close. + Assert.assertNotEquals(range.last, emitted.last()) // Check the emission not reach last element. + } + + @After + fun tearDown() { + unmockkAll() + } +} diff --git a/corellium/api/src/main/kotlin/flank/corellium/api/CorelliumApi.kt b/corellium/api/src/main/kotlin/flank/corellium/api/CorelliumApi.kt index 7678bcb44c..3b1a3cd7ef 100644 --- a/corellium/api/src/main/kotlin/flank/corellium/api/CorelliumApi.kt +++ b/corellium/api/src/main/kotlin/flank/corellium/api/CorelliumApi.kt @@ -4,8 +4,8 @@ package flank.corellium.api * Corellium API functions. */ class CorelliumApi( - val authorize: Authorization.Request, - val invokeAndroidDevices: AndroidInstance.Invoke, - val installAndroidApps: AndroidApps.Install, - val executeTest: AndroidTestPlan.Execute, + val authorize: Authorization.Request = Authorization.Request { throw NotImplementedError() }, + val invokeAndroidDevices: AndroidInstance.Invoke = AndroidInstance.Invoke { throw NotImplementedError() }, + val installAndroidApps: AndroidApps.Install = AndroidApps.Install { throw NotImplementedError() }, + val executeTest: AndroidTestPlan.Execute = AndroidTestPlan.Execute { throw NotImplementedError() }, ) diff --git a/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt b/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt index 71ff9131f4..9d51c2aa92 100644 --- a/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt +++ b/corellium/cli/src/main/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommand.kt @@ -232,6 +232,13 @@ internal val format = buildFormatter { else -> null } } + ExecuteTests.Error::class { + """ + Error while parsing results from instance $id. + For details check $logFile lines $lines. + + """.trimIndent() + cause.stackTraceToString() + } RunTestCorelliumAndroid.Created { "Created $path" } RunTestCorelliumAndroid.AlreadyExist { "Already exist $path" } diff --git a/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt b/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt index 0def003b92..b177198e54 100644 --- a/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt +++ b/corellium/cli/src/test/kotlin/flank/corellium/cli/RunTestCorelliumAndroidCommandTest.kt @@ -239,6 +239,7 @@ password: $password details = Instrument.Status.Details(emptyMap(), "Class", "Test", null) ) ), + Unit event ExecuteTests.Error("1", Exception(), "path/to/log/1", 5..10), Unit event RunTestCorelliumAndroid.Created(File("path/to/apk.apk")), Unit event RunTestCorelliumAndroid.AlreadyExist(File("path/to/apk.apk")), ) diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt index 603c1a3f49..b4a6847656 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/RunTestCorelliumAndroid.kt @@ -143,7 +143,15 @@ object RunTestCorelliumAndroid { object OutputDir object DumpShards object ExecuteTests { + const val ADB_LOG = "adb_log" + data class Status(val id: String, val status: Instrument) : Event.Data + data class Error( + val id: String, + val cause: Throwable, + val logFile: String, + val lines: IntRange + ) : Event.Data } object CompleteTests diff --git a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/ExecuteTests.kt b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/ExecuteTests.kt index 96b39ed7b9..c7cbdea992 100644 --- a/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/ExecuteTests.kt +++ b/corellium/domain/src/main/kotlin/flank/corellium/domain/run/test/android/step/ExecuteTests.kt @@ -5,18 +5,28 @@ import flank.corellium.domain.RunTestCorelliumAndroid import flank.corellium.domain.RunTestCorelliumAndroid.ExecuteTests import flank.corellium.domain.step import flank.instrument.command.formatAmInstrumentCommand +import flank.instrument.log.Instrument import flank.instrument.log.parseAdbInstrumentLog +import flank.shard.Shard import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.dropWhile +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.catch +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.flow.take +import java.io.File /** - * The step is executing tests on previously invoked devices. + * The step is executing tests on previously invoked devices, and returning the test results. + * + * The side effect is console logs from `am instrument` saved inside [RunTestCorelliumAndroid.ExecuteTests.ADB_LOG] output subdirectory. + * + * The optional parsing errors are sent through [RunTestCorelliumAndroid.Context.out]. * * require: * * [RunTestCorelliumAndroid.Context.authorize] @@ -29,15 +39,28 @@ import kotlinx.coroutines.flow.toList * * [RunTestCorelliumAndroid.State.shards] */ internal fun RunTestCorelliumAndroid.Context.executeTests() = step(ExecuteTests) { out -> + val outputDir = File(args.outputDir, ExecuteTests.ADB_LOG).apply { mkdir() } val testPlan: AndroidTestPlan.Config = prepareTestPlan() val list = coroutineScope { api.executeTest(testPlan).map { (id, flow) -> async { - flow.flowOn(Dispatchers.IO) - .dropWhile { line -> !line.startsWith("INSTRUMENTATION_STATUS") } + var read = 0 + var parsed = 0 + val file = outputDir.resolve(id) + val results = mutableListOf() + flow.onEach { file.appendText(it + "\n") } + .buffer(100) + .flowOn(Dispatchers.IO) + .onEach { ++read } .parseAdbInstrumentLog() - .onEach { status -> ExecuteTests.Status(id, status).out() } - .toList() + .onEach { parsed = read } + .onEach { result -> results += result } + .onEach { result -> ExecuteTests.Status(id, result).out() } + .catch { cause -> ExecuteTests.Error(id, cause, file.path, ++parsed..read).out() } + .filterIsInstance() + .take(expectedResultsCountFor(id)) + .collect() + results } }.awaitAll() } @@ -46,21 +69,22 @@ internal fun RunTestCorelliumAndroid.Context.executeTests() = step(ExecuteTests) /** * Prepare [AndroidTestPlan.Config] for test execution. - * It just mapping and formatting the data collected in state. + * It is just mapping and formatting the data collected in state. */ private fun RunTestCorelliumAndroid.State.prepareTestPlan(): AndroidTestPlan.Config = AndroidTestPlan.Config( shards.mapIndexed { index, shards -> - ids[index] to shards.flatMap { shard -> + ids[index] to shards.flatMap { shard: Shard.App -> shard.tests.map { test -> formatAmInstrumentCommand( packageName = packageNames.getValue(test.name), testRunner = testRunners.getValue(test.name), - testCases = test.cases.map { case -> - "class " + case.name - } + testCases = test.cases.map { case -> "class " + case.name } ) } } }.toMap() ) + +private fun RunTestCorelliumAndroid.State.expectedResultsCountFor(id: String): Int = + shards[ids.indexOf(id)].size diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/AdbLog.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/AdbLog.kt new file mode 100644 index 0000000000..79dd2ae00d --- /dev/null +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/AdbLog.kt @@ -0,0 +1,160 @@ +package flank.corellium.domain + +val validLog = """ +>() + out = { events += this as Event<*> } + + // when + val testResult = runBlocking { executeTests()(state).testResult } + + // then + assertTrue(testResult.first().isNotEmpty()) // Valid lines parsed before error will be returned + + val error = events.mapNotNull { it.value as? ExecuteTests.Error }.first() // Obtain error + assertEquals(instanceId, error.id) // Error should contain correct instanceId + + val lines = dir.resolve(ADB_LOG).resolve(instanceId).readLines() // Read log saved in file + assertTrue(lines.size > error.lines.last) // Task can save more output lines than was marked in error which is expected behaviour + + val invalid = lines.indexOfFirst { it.endsWith("INVALID LINE") } + 1 // Obtain invalid line number + assertTrue(invalid in error.lines) // Error should reference affected lines + } +} diff --git a/docs/flank_corellium.md b/docs/flank_corellium.md index 634ba78253..fe62a52c5d 100644 --- a/docs/flank_corellium.md +++ b/docs/flank_corellium.md @@ -160,6 +160,34 @@ The successful run should generate the following files: * JUnitReport.xml * android_shards.json +* adb_log + * Directory that contains dumped log from `am instrument` commands. + * Each dump name is related instance id. + +### Errors + +The execution or its part can fail due to exceptions occur. +Typically, most of the errors can be sourced in incorrect initial arguments or network issues. + +List of known possible errors: + +#### Test result parsing + +Flank is using `am instrument` command to execute tests on Corellium devices. +The console output of the device is collected and parsed until all expected tests return their results. +Due to an invalid apk file, the console can print unexpected output that cannot be parsed by Flank. +In this case, Flank will print an error message similar to the following: + +``` +Error while parsing results from instance { instance id }. +For details check "results/corellium/android/{ report_dir }/adb_log/{ instance id }" lines { from..to }. +java.lang.Exception + at flank.corellium.cli.RunTestCorelliumAndroidCommandTest.outputTest(RunTestCorelliumAndroidCommandTest.kt:242) + { ... } +``` + +The visible exception is related directly to the parsing issue. +To see the source of the problem check the log file referenced in the error message. The file should contain a direct dump from `am instrument command`. # Features @@ -179,7 +207,3 @@ The successful run should generate the following files: * Structural logging. * iOS support. * and much more... - -# Known bugs - -* Missing `` tag for skipped test cases in JUnit report