From ffe46ca6bd78e837dd190ff50001fb592eee6580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Tue, 27 Apr 2021 14:29:42 +0200 Subject: [PATCH] Refactor corellium client to functional style Add some diagrams --- .../kotlin/flank/corellium/client/Agent.kt | 153 ---------------- .../kotlin/flank/corellium/client/Console.kt | 52 ------ .../flank/corellium/client/Corellium.kt | 168 ------------------ .../main/kotlin/flank/corellium/client/VPN.kt | 5 - .../flank/corellium/client/agent/Agent.kt | 16 ++ .../flank/corellium/client/agent/Connect.kt | 101 +++++++++++ .../corellium/client/agent/Disconnect.kt | 7 + .../corellium/client/agent/UploadFile.kt | 37 ++++ .../flank/corellium/client/agent/Util.kt | 23 +++ .../flank/corellium/client/console/Connect.kt | 20 +++ .../flank/corellium/client/console/Console.kt | 10 ++ .../corellium/client/console/Functions.kt | 38 ++++ .../kotlin/flank/corellium/client/core/Api.kt | 146 +++++++++++++++ .../flank/corellium/client/core/Connect.kt | 52 ++++++ .../flank/corellium/client/core/Corellium.kt | 9 + .../flank/corellium/client/data/DTOs.kt | 74 ++++---- .../corellium/client/logging/LoggingLevel.kt | 12 -- .../flank/corellium/client/util/Retry.kt | 2 + corellium/sandbox/build.gradle.kts | 4 +- .../sandbox/android/AndroidExample.kt | 21 ++- .../sandbox/android/AndroidExampleNoVPN.kt | 54 ++++-- .../sandbox/ios/ExampleCorelliumRun.kt | 21 ++- .../android-cli-domain-sequence.puml | 34 ++++ docs/corellium/android-domain-class.puml | 33 ++++ docs/corellium/client-api-class.puml | 50 ++++++ docs/corellium/modules.puml | 14 ++ 26 files changed, 693 insertions(+), 463 deletions(-) delete mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/Agent.kt delete mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/Console.kt delete mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/Corellium.kt delete mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/VPN.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/agent/Agent.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/agent/Connect.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/agent/Disconnect.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/agent/UploadFile.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/agent/Util.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/console/Connect.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/console/Console.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/console/Functions.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/core/Api.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/core/Connect.kt create mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/core/Corellium.kt delete mode 100644 corellium/client/src/main/kotlin/flank/corellium/client/logging/LoggingLevel.kt create mode 100644 docs/corellium/android-cli-domain-sequence.puml create mode 100644 docs/corellium/android-domain-class.puml create mode 100644 docs/corellium/client-api-class.puml create mode 100644 docs/corellium/modules.puml diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/Agent.kt b/corellium/client/src/main/kotlin/flank/corellium/client/Agent.kt deleted file mode 100644 index 1220361665..0000000000 --- a/corellium/client/src/main/kotlin/flank/corellium/client/Agent.kt +++ /dev/null @@ -1,153 +0,0 @@ -package flank.corellium.client - -import flank.corellium.client.data.AgentOperation -import flank.corellium.client.data.CommandResult -import flank.corellium.client.logging.LoggingLevel -import flank.corellium.client.util.withProgress -import io.ktor.client.HttpClient -import io.ktor.client.features.logging.Logging -import io.ktor.client.features.websocket.ClientWebSocketSession -import io.ktor.client.features.websocket.WebSockets -import io.ktor.client.features.websocket.webSocketSession -import io.ktor.client.request.header -import io.ktor.client.request.url -import io.ktor.http.cio.websocket.Frame -import io.ktor.http.cio.websocket.close -import io.ktor.http.cio.websocket.readText -import kotlinx.coroutines.CompletableJob -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.withTimeout -import kotlinx.coroutines.withTimeoutOrNull -import kotlinx.serialization.decodeFromString -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger - -class Agent( - private val agentUrl: String, - private val logging: LoggingLevel, - private val token: String -) { - private lateinit var connection: ClientWebSocketSession - - private val counter = AtomicInteger(1) - private val format = Json {} - private val tasks = ConcurrentHashMap Unit>() - private var uploading = false - - private val wsClient = HttpClient { - install(WebSockets) - install(Logging) { - level = logging.level - } - } - - private suspend fun connect() { - connection = wsClient.webSocketSession { - url(agentUrl) - header("Authorization", token) - }.apply { - launch { - for (frame in incoming) { - when (frame) { - is Frame.Text -> textFrameHandler(frame) - is Frame.Ping -> println("got ping") - is Frame.Pong -> println("got pong") - else -> println(frame.data.decodeToString()) - } - } - } - } - } - - private fun textFrameHandler(frame: Frame.Text) { - println("\nReceived: ${frame.readText()}") - val result = format.decodeFromString(frame.readText()) - tasks[result.id]?.let { it(result) } - } - - private suspend fun isReady() { - val task = Job() - val id = counter.getAndIncrement() - connection.sendCommand( - AgentOperation( - type = "app", - op = "ready", - id = id - ) - ) - tasks[id] = getCommonHandler(task) - withTimeout(20_000) { - task.join() - } - } - - suspend fun waitForAgentReady() = withProgress { - var booting: Boolean - do { - delay(20_000) - booting = try { - connect() - isReady() - false - } catch (ex: Exception) { - true - } - } while (booting) - } - - suspend fun uploadFile(path: String, bytes: ByteArray) { - val id = counter.getAndIncrement() - connection.sendCommand( - AgentOperation( - type = "file", - op = "upload", - id = id, - path = path - ) - ) - - val idBytes = ByteBuffer.allocate(8) - .order(ByteOrder.LITTLE_ENDIAN) - .put(id.toByte()) - .array() - val payload = ByteBuffer.wrap(idBytes + bytes).order(ByteOrder.LITTLE_ENDIAN) - - connection.send(Frame.Binary(true, payload)) - connection.send(Frame.Binary(true, idBytes)) - - val task = Job() - tasks[id] = getUploadHandler(task) - withTimeoutOrNull(10_000) { - task.join() - } - } - - suspend fun close() = connection.close() - - private suspend fun ClientWebSocketSession.sendCommand(command: AgentOperation) = - send(Frame.Text(format.encodeToString(command))) - - private fun getCommonHandler(task: CompletableJob): (CommandResult) -> Unit = { - if (it.success) task.complete() - else { - println("Task ${it.id} failed") - println(it) - } - } - - private fun getUploadHandler(task: CompletableJob): (CommandResult) -> Unit = { - if (it.success) { - task.complete() - uploading = false - } else { - println("Task ${it.id} failed") - println(it) - } - } -} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/Console.kt b/corellium/client/src/main/kotlin/flank/corellium/client/Console.kt deleted file mode 100644 index 257297b3df..0000000000 --- a/corellium/client/src/main/kotlin/flank/corellium/client/Console.kt +++ /dev/null @@ -1,52 +0,0 @@ -package flank.corellium.client - -import io.ktor.client.HttpClient -import io.ktor.client.features.websocket.WebSockets -import io.ktor.client.features.websocket.webSocketSession -import io.ktor.client.request.header -import io.ktor.client.request.url -import io.ktor.http.cio.websocket.Frame -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking - -class Console( - url: String, - token: String -) { - private var open = true - private val socket = runBlocking { - HttpClient { - install(WebSockets) - }.webSocketSession { - url(url) - header("Authorization", token) - }.apply { - launch { - var closingJob: Job? = null - for (frame in incoming) { - if (frame is Frame.Binary) { - closingJob?.let { - it.cancel() - - // we want to skip first message since corellium sends old console logs - print(frame.data.decodeToString()) - } - closingJob = launch { - delay(5_000) - open = false - } - } - } - } - } - } - - suspend fun sendCommand(command: String) = - socket.send(Frame.Binary(true, (command + "\n").encodeToByteArray())) - - suspend fun waitUntilFinished() { - while (open) delay(5_000) - } -} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/Corellium.kt b/corellium/client/src/main/kotlin/flank/corellium/client/Corellium.kt deleted file mode 100644 index c7fc8d8343..0000000000 --- a/corellium/client/src/main/kotlin/flank/corellium/client/Corellium.kt +++ /dev/null @@ -1,168 +0,0 @@ -package flank.corellium.client - -import flank.corellium.client.data.ConsoleSocket -import flank.corellium.client.data.Credentials -import flank.corellium.client.data.Id -import flank.corellium.client.data.Instance -import flank.corellium.client.data.LoginResponse -import flank.corellium.client.data.Project -import flank.corellium.client.logging.LoggingLevel -import flank.corellium.client.util.withProgress -import flank.corellium.client.util.withRetry -import io.ktor.client.HttpClient -import io.ktor.client.engine.cio.CIO -import io.ktor.client.features.json.JsonFeature -import io.ktor.client.features.json.serializer.KotlinxSerializer -import io.ktor.client.features.logging.Logging -import io.ktor.client.request.delete -import io.ktor.client.request.get -import io.ktor.client.request.header -import io.ktor.client.request.post -import io.ktor.client.request.url -import io.ktor.client.statement.HttpResponse -import io.ktor.http.ContentType -import io.ktor.http.contentType -import io.ktor.util.cio.writeChannel -import io.ktor.utils.io.copyAndClose -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.delay -import kotlinx.serialization.json.Json -import java.io.File -import java.util.UUID - -@Suppress("unused", "MemberVisibilityCanBePrivate") -class Corellium( - api: String, - private val username: String, - private val password: String, - private val tokenFallback: String = "", - private val logging: LoggingLevel = LoggingLevel.None -) { - private val client = HttpClient(CIO) { - install(JsonFeature) { - serializer = KotlinxSerializer( - Json { - ignoreUnknownKeys = true - encodeDefaults = false - } - ) - } - install(Logging) { - level = logging.level - } - } - - private val urlBase = "https://$api/api/v1" - - private var _token: String? = null - - private val token: String - get() = _token ?: tokenFallback - - suspend fun logIn(): String { - _token = withRetry { - client.post { - url("$urlBase/tokens") - contentType(ContentType.Application.Json) - body = Credentials(username, password) - }.token - } - return token - } - - suspend fun getProjectIdList(): List = withRetry { - client.get>( - block = { - url("$urlBase/projects?ids_only=1") - contentType(ContentType.Application.Json) - header("Authorization", token) - } - ).map { it.id } - } - - suspend fun getAllProjects(): List = getProjectIdList() - .map { - withRetry { - async { - client.get { - url("$urlBase/projects/$it") - contentType(ContentType.Application.Json) - header("Authorization", token) - } - } - } - } - .awaitAll() - - suspend fun getProjectInstancesList(projectId: String): List = withRetry { - client.get { - url("$urlBase/projects/$projectId/instances") - contentType(ContentType.Application.Json) - header("Authorization", token) - } - } - - suspend fun createNewInstance(newInstance: Instance): String = withRetry { - client.post( - block = { - url("$urlBase/instances") - contentType(ContentType.Application.Json) - header("Authorization", token) - body = newInstance - } - ).id - } - - suspend fun deleteInstance(instanceId: String): Unit = withRetry { - client.delete { - url("$urlBase/instances/$instanceId") - contentType(ContentType.Application.Json) - header("Authorization", token) - } - } - - suspend fun getInstanceInfo(instanceId: String): Instance = withRetry { - client.get { - url("$urlBase/instances/$instanceId") - contentType(ContentType.Application.Json) - header("Authorization", token) - } - } - - suspend fun waitUntilInstanceIsReady(instanceId: String) = withProgress { - while (true) { - if (getInstanceInfo(instanceId).state == "on") { - println() - break - } - // it really takes loooong time - delay(20_000) - } - } - - fun createAgent(agentInfo: String): Agent = Agent( - agentUrl = "${urlBase.replace("https", "wss")}/agent/$agentInfo", - logging = logging, - token = token - ) - - suspend fun getVPNConfig(projectId: String, type: VPN, id: String = UUID.randomUUID().toString()) { - val response: HttpResponse = withRetry { - client.get { - url("$urlBase/projects/$projectId/vpn-configs/$id.${type.name.toLowerCase()}") - header("Authorization", token) - } - } - val fileName = if (type == VPN.TBLK) "tblk.zip" else "config.ovpn" - response.content.copyAndClose(File(fileName).writeChannel()) - } - - suspend fun getInstanceConsole(instanceId: String) = withRetry { - val url = client.get { - url("$urlBase/instances/$instanceId/console") - header("Authorization", token) - }.url - Console(url, token) - } -} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/VPN.kt b/corellium/client/src/main/kotlin/flank/corellium/client/VPN.kt deleted file mode 100644 index 1f2eb56088..0000000000 --- a/corellium/client/src/main/kotlin/flank/corellium/client/VPN.kt +++ /dev/null @@ -1,5 +0,0 @@ -package flank.corellium.client - -enum class VPN { - OVPN, TBLK -} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/agent/Agent.kt b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Agent.kt new file mode 100644 index 0000000000..c43bd01e71 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Agent.kt @@ -0,0 +1,16 @@ +package flank.corellium.client.agent + +import flank.corellium.client.data.CommandResult +import io.ktor.client.features.websocket.ClientWebSocketSession +import kotlinx.serialization.json.Json +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +class Agent internal constructor( + internal val session: ClientWebSocketSession, + internal val tasks: TasksMap = TasksMap(), + internal val counter: AtomicInteger = AtomicInteger(1), + internal val format: Json = Json {}, +) + +typealias TasksMap = ConcurrentHashMap Unit> diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/agent/Connect.kt b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Connect.kt new file mode 100644 index 0000000000..a97743bedc --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Connect.kt @@ -0,0 +1,101 @@ +package flank.corellium.client.agent + +import flank.corellium.client.data.AgentOperation +import flank.corellium.client.data.CommandResult +import flank.corellium.client.util.withProgress +import io.ktor.client.HttpClient +import io.ktor.client.features.logging.LogLevel +import io.ktor.client.features.logging.Logging +import io.ktor.client.features.websocket.DefaultClientWebSocketSession +import io.ktor.client.features.websocket.WebSockets +import io.ktor.client.features.websocket.webSocketSession +import io.ktor.client.request.header +import io.ktor.client.request.url +import io.ktor.http.cio.websocket.Frame +import io.ktor.http.cio.websocket.readText +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeout +import kotlinx.serialization.decodeFromString + +suspend fun connectAgent( + agentUrl: String, + token: String, + logLevel: LogLevel = LogLevel.NONE +): Agent = withProgress { + val client = HttpClient { + install(WebSockets) + install(Logging) { + level = logLevel + } + } + + var connection: Agent? = null + var retryCount = 10 + do { + require(retryCount-- > 0) { "Max retry count reached" } + try { + connection = client + .createSession(agentUrl, token) + .let(::Agent) + .apply { + handleIncomingFrames() + waitForReady() + } + } catch (ex: Exception) { + ex.printStackTrace() + delay(20_000) + } + } while (connection == null) + + connection +} + +private suspend fun HttpClient.createSession( + agentUrl: String, + token: String +): DefaultClientWebSocketSession = + webSocketSession { + url(agentUrl) + header("Authorization", token) + } + +private suspend fun Agent.waitForReady() { + val task = Job() + val id = counter.getAndIncrement() + sendCommand( + AgentOperation( + type = "app", + op = "ready", + id = id + ) + ) + tasks[id] = { result -> + if (result.success) task.complete() else { + println("Task ${result.id} failed") + println(result) + } + } + withTimeout(20_000) { + task.join() + } +} + +private fun Agent.handleIncomingFrames() = + session.launch { + for (frame in session.incoming) { + when (frame) { + is Frame.Text -> handleTestFrame(frame) + is Frame.Ping -> println("got ping") + is Frame.Pong -> println("got pong") + else -> println(frame.data.decodeToString()) + } + } + } + +private fun Agent.handleTestFrame(frame: Frame.Text) { + println("\nReceived: ${frame.readText()}") + val result = format.decodeFromString(frame.readText()) + tasks[result.id]?.invoke(result) +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/agent/Disconnect.kt b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Disconnect.kt new file mode 100644 index 0000000000..513958adc1 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Disconnect.kt @@ -0,0 +1,7 @@ +package flank.corellium.client.agent + +import io.ktor.http.cio.websocket.close + +suspend fun Agent.disconnect() { + session.close() +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/agent/UploadFile.kt b/corellium/client/src/main/kotlin/flank/corellium/client/agent/UploadFile.kt new file mode 100644 index 0000000000..7994f45474 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/agent/UploadFile.kt @@ -0,0 +1,37 @@ +package flank.corellium.client.agent + +import flank.corellium.client.data.AgentOperation +import io.ktor.http.cio.websocket.Frame +import kotlinx.coroutines.Job +import kotlinx.coroutines.withTimeoutOrNull +import java.nio.ByteBuffer +import java.nio.ByteOrder + +suspend fun Agent.uploadFile(path: String, bytes: ByteArray) { + val id = counter.getAndIncrement() + + sendCommand( + AgentOperation( + type = "file", + op = "upload", + id = id, + path = path + ) + ) + + val idBytes = ByteBuffer.allocate(8) + .order(ByteOrder.LITTLE_ENDIAN) + .put(id.toByte()) + .array() + + val payload = ByteBuffer.wrap(idBytes + bytes).order(ByteOrder.LITTLE_ENDIAN) + + session.send(Frame.Binary(true, payload)) + session.send(Frame.Binary(true, idBytes)) + + val task = Job() + tasks[id] = defaultResultHandler(task) + withTimeoutOrNull(10_000) { + task.join() + } +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/agent/Util.kt b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Util.kt new file mode 100644 index 0000000000..f1756890d7 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/agent/Util.kt @@ -0,0 +1,23 @@ +package flank.corellium.client.agent + +import flank.corellium.client.data.AgentOperation +import flank.corellium.client.data.CommandResult +import io.ktor.http.cio.websocket.Frame +import kotlinx.coroutines.CompletableJob +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json + +internal fun defaultResultHandler(task: CompletableJob): (CommandResult) -> Unit = { result -> + if (result.success) task.complete() else { + println("Task ${result.id} failed") + println(format.encodeToString(result)) + } +} + +internal suspend fun Agent.sendCommand(command: AgentOperation) = + session.send(Frame.Text(format.encodeToString(command))) + +private val format = Json { + ignoreUnknownKeys = true + encodeDefaults = false +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/console/Connect.kt b/corellium/client/src/main/kotlin/flank/corellium/client/console/Connect.kt new file mode 100644 index 0000000000..fb85ab259e --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/console/Connect.kt @@ -0,0 +1,20 @@ +package flank.corellium.client.console + +import io.ktor.client.HttpClient +import io.ktor.client.features.websocket.WebSockets +import io.ktor.client.features.websocket.webSocketSession +import io.ktor.client.request.header +import io.ktor.client.request.url + +suspend fun connectConsole( + url: String, + token: String +): Console = + HttpClient { + install(WebSockets) + }.webSocketSession { + url(url) + header("Authorization", token) + }.let { session -> + Console(session) + } diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/console/Console.kt b/corellium/client/src/main/kotlin/flank/corellium/client/console/Console.kt new file mode 100644 index 0000000000..84ab174415 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/console/Console.kt @@ -0,0 +1,10 @@ +package flank.corellium.client.console + +import io.ktor.client.features.websocket.ClientWebSocketSession +import kotlinx.coroutines.CoroutineScope + +class Console internal constructor( + internal val session: ClientWebSocketSession +) : CoroutineScope by session { + internal var lastResponseTime = System.currentTimeMillis() +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/console/Functions.kt b/corellium/client/src/main/kotlin/flank/corellium/client/console/Functions.kt new file mode 100644 index 0000000000..5b461dddaf --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/console/Functions.kt @@ -0,0 +1,38 @@ +package flank.corellium.client.console + +import io.ktor.http.cio.websocket.Frame +import io.ktor.http.cio.websocket.close +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.launch + +suspend fun Console.sendCommand(command: String) = + session.send(Frame.Binary(true, (command + "\n").encodeToByteArray())) + +suspend fun Console.waitForIdle(timeToWait: Long) { + delay(10_000) + while (System.currentTimeMillis() - lastResponseTime > timeToWait) delay(100) +} + +suspend fun Console.close() = session.close() + +fun Console.launchOutputPrinter() = session.launch { + // drop console bash history which is received as first frame + session.incoming.receive() + + for (frame in session.incoming) { + lastResponseTime = System.currentTimeMillis() + print(frame.data.decodeToString()) + } +} + +suspend fun Console.clear() { + session.incoming.receive() +} + +fun Console.flowLogs() = session.incoming + .receiveAsFlow() + .onEach { lastResponseTime = System.currentTimeMillis() } + .map { it.data.decodeToString() } diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/core/Api.kt b/corellium/client/src/main/kotlin/flank/corellium/client/core/Api.kt new file mode 100644 index 0000000000..8aa37ace5a --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/core/Api.kt @@ -0,0 +1,146 @@ +package flank.corellium.client.core + +import flank.corellium.client.agent.Agent +import flank.corellium.client.agent.connectAgent +import flank.corellium.client.console.Console +import flank.corellium.client.console.connectConsole +import flank.corellium.client.data.ConsoleSocket +import flank.corellium.client.data.Id +import flank.corellium.client.data.Instance +import flank.corellium.client.data.Project +import flank.corellium.client.util.withProgress +import flank.corellium.client.util.withRetry +import io.ktor.client.features.logging.LogLevel +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.url +import io.ktor.client.statement.HttpResponse +import io.ktor.http.ContentType +import io.ktor.http.contentType +import io.ktor.util.cio.writeChannel +import io.ktor.utils.io.copyAndClose +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.delay +import java.io.File +import java.util.UUID + +suspend fun Corellium.getAllProjects(): List = + getProjectIdList().map { + withRetry { + async { + client.get { + url("$urlBase/projects/$it") + contentType(ContentType.Application.Json) + header("Authorization", token) + } + } + } + }.awaitAll() + +suspend fun Corellium.getProjectIdList(): List = + withRetry { + client.get> { + url("$urlBase/projects?ids_only=1") + contentType(ContentType.Application.Json) + header("Authorization", token) + }.map { it.id } + } + +suspend fun Corellium.getProjectInstancesList( + projectId: String +): List = + withRetry { + client.get { + url("$urlBase/projects/$projectId/instances") + contentType(ContentType.Application.Json) + header("Authorization", token) + } + } + +suspend fun Corellium.createNewInstance( + newInstance: Instance +): String = + withRetry { + client.post { + url("$urlBase/instances") + contentType(ContentType.Application.Json) + header("Authorization", token) + body = newInstance + }.id + } + +suspend fun Corellium.deleteInstance( + instanceId: String +): Unit = + withRetry { + client.delete { + url("$urlBase/instances/$instanceId") + contentType(ContentType.Application.Json) + header("Authorization", token) + } + } + +suspend fun Corellium.getInstanceInfo( + instanceId: String +): Instance = + withRetry { + client.get { + url("$urlBase/instances/$instanceId") + contentType(ContentType.Application.Json) + header("Authorization", token) + } + } + +suspend fun Corellium.waitUntilInstanceIsReady( + instanceId: String +): Unit = + withProgress { + while (true) { + if (getInstanceInfo(instanceId).state == "on") { + println() + break + } + // it really takes loooong time + delay(20_000) + } + } + +suspend fun Corellium.connectAgent( + agentInfo: String +): Agent = + connectAgent( + agentUrl = "${urlBase.replace("https", "wss")}/agent/$agentInfo", + token = token, + logLevel = LogLevel.NONE, + ) + +suspend fun Corellium.getVPNConfig( + projectId: String, + type: VPN, + id: String = UUID.randomUUID().toString() +) { + val response: HttpResponse = withRetry { + client.get { + url("$urlBase/projects/$projectId/vpn-configs/$id.${type.name.toLowerCase()}") + header("Authorization", token) + } + } + val fileName = if (type == VPN.TBLK) "tblk.zip" else "config.ovpn" + response.content.copyAndClose(File(fileName).writeChannel()) +} + +enum class VPN { OVPN, TBLK } + +suspend fun Corellium.connectConsole( + instanceId: String +): Console = + withRetry { + val url = client.get { + url("$urlBase/instances/$instanceId/console") + header("Authorization", token) + }.url + connectConsole(url, token) + } diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/core/Connect.kt b/corellium/client/src/main/kotlin/flank/corellium/client/core/Connect.kt new file mode 100644 index 0000000000..0081974a49 --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/core/Connect.kt @@ -0,0 +1,52 @@ +package flank.corellium.client.core + +import flank.corellium.client.data.Credentials +import flank.corellium.client.data.LoginResponse +import flank.corellium.client.util.withRetry +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.features.json.JsonFeature +import io.ktor.client.features.json.serializer.KotlinxSerializer +import io.ktor.client.features.logging.LogLevel +import io.ktor.client.features.logging.Logging +import io.ktor.client.request.post +import io.ktor.client.request.url +import io.ktor.http.ContentType +import io.ktor.http.contentType +import kotlinx.serialization.json.Json + +suspend fun connectCorellium( + api: String, + username: String, + password: String +): Corellium { + val client = HttpClient(CIO) { + install(JsonFeature) { + serializer = KotlinxSerializer( + Json { + ignoreUnknownKeys = true + encodeDefaults = false + } + ) + } + install(Logging) { + level = LogLevel.NONE + } + } + + val urlBase = "https://$api/api/v1" + + val token = withRetry { + client.post { + url("$urlBase/tokens") + contentType(ContentType.Application.Json) + body = Credentials(username, password) + }.token + } + + return Corellium( + client = client, + urlBase = urlBase, + token = token + ) +} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/core/Corellium.kt b/corellium/client/src/main/kotlin/flank/corellium/client/core/Corellium.kt new file mode 100644 index 0000000000..5cd05c34db --- /dev/null +++ b/corellium/client/src/main/kotlin/flank/corellium/client/core/Corellium.kt @@ -0,0 +1,9 @@ +package flank.corellium.client.core + +import io.ktor.client.HttpClient + +class Corellium internal constructor( + internal val client: HttpClient, + internal val urlBase: String, + internal val token: String +) diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/data/DTOs.kt b/corellium/client/src/main/kotlin/flank/corellium/client/data/DTOs.kt index e8a7109875..12c91f76bb 100644 --- a/corellium/client/src/main/kotlin/flank/corellium/client/data/DTOs.kt +++ b/corellium/client/src/main/kotlin/flank/corellium/client/data/DTOs.kt @@ -2,13 +2,6 @@ package flank.corellium.client.data import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json - -private val format = Json { - ignoreUnknownKeys = true - encodeDefaults = false -} @Serializable data class Instance( @@ -23,26 +16,26 @@ data class Instance( val patches: List = emptyList(), val os: String = "", val osbuild: String = "", - val agent: InstanceAgent? = InstanceAgent(), + val agent: Agent? = Agent(), val serviceIp: String = "", @SerialName("port-adb") val portAdb: String = "" -) - -@Serializable -data class InstanceAgent( - val hash: String = "", - val info: String = "" -) +) { + @Serializable + data class Agent( + val hash: String = "", + val info: String = "" + ) -@Serializable -data class BootOptions( - val bootArgs: String = "", - val restoreBootArgs: String = "", - val udid: String = "", - val ecid: String = "", - val screen: String = "" -) + @Serializable + data class BootOptions( + val bootArgs: String = "", + val restoreBootArgs: String = "", + val udid: String = "", + val ecid: String = "", + val screen: String = "" + ) +} @Serializable data class Project( @@ -50,17 +43,18 @@ data class Project( val name: String, val quotas: Quotas, val quotasUsed: Quotas -) +) { -@Serializable -data class Quotas( - val cores: Int, - val instances: Int, - val ram: Int, - val cpus: Int, - val gpus: Int? = null, - val instance: Int? = null -) + @Serializable + data class Quotas( + val cores: Int, + val instances: Int, + val ram: Int, + val cpus: Int, + val gpus: Int? = null, + val instance: Int? = null + ) +} @Serializable data class Id( @@ -90,17 +84,15 @@ data class AgentOperation( data class CommandResult( val id: Int, val success: Boolean, - val error: CommandError? = null + val error: Error? = null ) { - override fun toString() = format.encodeToString(this) + @Serializable + data class Error( + val name: String = "", + val message: String = "", + ) } -@Serializable -data class CommandError( - val name: String = "", - val message: String = "", -) - @Serializable data class ConsoleSocket( val url: String diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/logging/LoggingLevel.kt b/corellium/client/src/main/kotlin/flank/corellium/client/logging/LoggingLevel.kt deleted file mode 100644 index ddcb31210d..0000000000 --- a/corellium/client/src/main/kotlin/flank/corellium/client/logging/LoggingLevel.kt +++ /dev/null @@ -1,12 +0,0 @@ -@file:Suppress("unused") -package flank.corellium.client.logging - -import io.ktor.client.features.logging.LogLevel - -sealed class LoggingLevel(val level: LogLevel) { - object All : LoggingLevel(LogLevel.ALL) - object None : LoggingLevel(LogLevel.NONE) - object Body : LoggingLevel(LogLevel.BODY) - object Headers : LoggingLevel(LogLevel.HEADERS) - object Info : LoggingLevel(LogLevel.INFO) -} diff --git a/corellium/client/src/main/kotlin/flank/corellium/client/util/Retry.kt b/corellium/client/src/main/kotlin/flank/corellium/client/util/Retry.kt index 98babf6f48..a82f3d2e68 100644 --- a/corellium/client/src/main/kotlin/flank/corellium/client/util/Retry.kt +++ b/corellium/client/src/main/kotlin/flank/corellium/client/util/Retry.kt @@ -7,6 +7,8 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.launch +// TODO convert print lines to structural logging + suspend inline fun withRetry(crossinline block: suspend CoroutineScope.() -> T) = coroutineScope { var currentDelay = 500L repeat(6) { diff --git a/corellium/sandbox/build.gradle.kts b/corellium/sandbox/build.gradle.kts index 6673cf871d..844423ac8c 100644 --- a/corellium/sandbox/build.gradle.kts +++ b/corellium/sandbox/build.gradle.kts @@ -62,13 +62,13 @@ val androidExampleNoVPNJar by tasks.registering(Jar::class) { val runAndroidExample by tasks.registering(Exec::class) { dependsOn(androidExampleJar) workingDir = project.rootDir - commandLine("java", "-jar", "./corellium/corellium-sandbox/build/libs/android-example-all.jar") + commandLine("java", "-jar", "./corellium/sandbox/build/libs/android-example-all.jar") } val runAndroidExampleNoVPN by tasks.registering(Exec::class) { dependsOn(androidExampleNoVPNJar) workingDir = project.rootDir - commandLine("java", "-jar", "./corellium/corellium-sandbox/build/libs/android-example-no-vpn-all.jar") + commandLine("java", "-jar", "./corellium/sandbox/build/libs/android-example-no-vpn-all.jar") } file("./src/main/resources/corellium-config.properties").also { propFile -> diff --git a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExample.kt b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExample.kt index b7dfab576c..27df707838 100644 --- a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExample.kt +++ b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExample.kt @@ -1,9 +1,15 @@ @file:JvmName("AndroidExample") package flank.corellium.sandbox.android -import flank.corellium.client.Corellium -import flank.corellium.client.data.BootOptions +import flank.corellium.client.core.connectAgent +import flank.corellium.client.core.connectCorellium +import flank.corellium.client.core.createNewInstance +import flank.corellium.client.core.getAllProjects +import flank.corellium.client.core.getInstanceInfo +import flank.corellium.client.core.getProjectInstancesList +import flank.corellium.client.core.waitUntilInstanceIsReady import flank.corellium.client.data.Instance +import flank.corellium.client.data.Instance.BootOptions import flank.corellium.sandbox.config.Config import kotlinx.coroutines.runBlocking @@ -12,19 +18,17 @@ private const val flavor = "ranchu" private const val os = "11.0.0" private const val screen = "720x1280:280" private const val projectName = "Default Project" -private const val apkPath = "./corellium/corellium-sandbox/src/main/resources/android/app-debug.apk" +private const val apkPath = "./corellium/sandbox/src/main/resources/android/app-debug.apk" private const val testApkPath = - "./corellium/corellium-sandbox/src/main/resources/android/app-multiple-success-debug-androidTest.apk" + "./corellium/sandbox/src/main/resources/android/app-multiple-success-debug-androidTest.apk" fun main(): Unit = runBlocking { - val client = Corellium( + val client = connectCorellium( api = Config.api, username = Config.username, password = Config.password ) - client.logIn() - val projectId = client.getAllProjects().first { it.name == projectName }.id println("Looking for $instanceName instance") @@ -51,9 +55,8 @@ fun main(): Unit = runBlocking { val instance = client.getInstanceInfo(instanceId) println("Creating agent") - val agent = client.createAgent(instance.agent?.info ?: error("Agent info is not present")) println("Await agent is connected and ready to use") - agent.waitForAgentReady() + client.connectAgent(instance.agent?.info ?: error("Agent info is not present")) println("Agent ready") runProcess("adb", "connect", "${instance.serviceIp}:${instance.portAdb}") diff --git a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExampleNoVPN.kt b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExampleNoVPN.kt index f920af10e2..321bc51fa5 100644 --- a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExampleNoVPN.kt +++ b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/android/AndroidExampleNoVPN.kt @@ -2,32 +2,43 @@ package flank.corellium.sandbox.android -import flank.corellium.client.Corellium -import flank.corellium.client.data.BootOptions +import flank.corellium.client.agent.disconnect +import flank.corellium.client.agent.uploadFile +import flank.corellium.client.console.close +import flank.corellium.client.console.launchOutputPrinter +import flank.corellium.client.console.sendCommand +import flank.corellium.client.console.waitForIdle +import flank.corellium.client.core.connectAgent +import flank.corellium.client.core.connectConsole +import flank.corellium.client.core.connectCorellium +import flank.corellium.client.core.createNewInstance +import flank.corellium.client.core.getAllProjects +import flank.corellium.client.core.getInstanceInfo +import flank.corellium.client.core.getProjectInstancesList +import flank.corellium.client.core.waitUntilInstanceIsReady import flank.corellium.client.data.Instance +import flank.corellium.client.data.Instance.BootOptions import flank.corellium.sandbox.config.Config import kotlinx.coroutines.runBlocking import java.io.File -private const val instanceName = "corellium-android" +private const val instanceName = "corellium-android-2" private const val flavor = "ranchu" private const val os = "11.0.0" private const val screen = "720x1280:280" private const val projectName = "Default Project" -private const val apkPath = "./corellium/corellium-sandbox/src/main/resources/android/app-debug.apk" +private const val apkPath = "./corellium/sandbox/src/main/resources/android/app-debug.apk" private const val testApkPath = - "./corellium/corellium-sandbox/src/main/resources/android/app-multiple-success-debug-androidTest.apk" + "./corellium/sandbox/src/main/resources/android/app-multiple-success-debug-androidTest.apk" private const val pathToUpload = "/sdcard" fun main(): Unit = runBlocking { - val client = Corellium( + val client = connectCorellium( api = Config.api, username = Config.username, password = Config.password ) - client.logIn() - val projectId = client.getAllProjects().first { it.name == projectName }.id println("Looking for $instanceName instance") @@ -54,26 +65,43 @@ fun main(): Unit = runBlocking { val instance = client.getInstanceInfo(instanceId) println("Creating agent") - val agent = client.createAgent(instance.agent?.info ?: error("Agent info is not present")) println("Await agent is connected and ready to use") - agent.waitForAgentReady() + val agent = client.connectAgent(instance.agent?.info ?: error("Agent info is not present")) println("Agent ready") agent.uploadFile( path = "$pathToUpload/app-debug.apk", bytes = File(apkPath).readBytes() ) + println("App apk uploaded") agent.uploadFile( path = "$pathToUpload/app-multiple-success-debug-androidTest.apk", bytes = File(testApkPath).readBytes() ) + println("Test apk uploaded") + + val console = client.connectConsole(instanceId) + println("Console connected") - val console = client.getInstanceConsole(instanceId) + // prevent flooding the output by the system and kernel logging + console.sendCommand("su") + console.sendCommand("dmesg -n 1") + console.sendCommand("exit") + println("Installing apps...") console.sendCommand("pm install $pathToUpload/app-debug.apk") console.sendCommand("pm install -t $pathToUpload/app-multiple-success-debug-androidTest.apk") - console.sendCommand("am instrument -w com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner") - console.waitUntilFinished() + println("Running tests... ") + console.sendCommand("am instrument -r -w com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner") + console.launchOutputPrinter() + + console.waitForIdle(10_000) + println() + println("Console idle up 10s") + println("Disconnecting...") + console.close() + + agent.disconnect() } diff --git a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/ios/ExampleCorelliumRun.kt b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/ios/ExampleCorelliumRun.kt index 73cedefac4..1f3a78dba7 100644 --- a/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/ios/ExampleCorelliumRun.kt +++ b/corellium/sandbox/src/main/kotlin/flank/corellium/sandbox/ios/ExampleCorelliumRun.kt @@ -1,9 +1,17 @@ @file:JvmName("ExampleCorelliumRun") package flank.corellium.sandbox.ios -import flank.corellium.client.Corellium -import flank.corellium.client.data.BootOptions +import flank.corellium.client.agent.disconnect +import flank.corellium.client.agent.uploadFile +import flank.corellium.client.core.connectAgent +import flank.corellium.client.core.connectCorellium +import flank.corellium.client.core.createNewInstance +import flank.corellium.client.core.getAllProjects +import flank.corellium.client.core.getInstanceInfo +import flank.corellium.client.core.getProjectInstancesList +import flank.corellium.client.core.waitUntilInstanceIsReady import flank.corellium.client.data.Instance +import flank.corellium.client.data.Instance.BootOptions import flank.corellium.sandbox.config.Config import kotlinx.coroutines.runBlocking import java.io.File @@ -14,14 +22,12 @@ private val localPathString = Config.plistPath private val xctestrunPath = Config.xctestrunPath fun main() = runBlocking { - val client = Corellium( + val client = connectCorellium( api = Config.api, username = Config.username, password = Config.password ) - client.logIn() - println("Fetching [Amanda] project") val projectId = client.getAllProjects().first { it.name == "Amanda" }.id @@ -50,9 +56,8 @@ fun main() = runBlocking { val instance = client.getInstanceInfo(instanceId) println("Creating agent") - val agent = client.createAgent(instance.agent?.info ?: error("Agent info is not present")) println("Await agent is connected and ready to use") - agent.waitForAgentReady() + val agent = client.connectAgent(instance.agent?.info ?: error("Agent info is not present")) println("Agent ready") println("Uploading plist file") @@ -75,5 +80,5 @@ fun main() = runBlocking { .redirectError(ProcessBuilder.Redirect.INHERIT) .start().waitFor() - agent.close() + agent.disconnect() } diff --git a/docs/corellium/android-cli-domain-sequence.puml b/docs/corellium/android-cli-domain-sequence.puml new file mode 100644 index 0000000000..ded2b3ed99 --- /dev/null +++ b/docs/corellium/android-cli-domain-sequence.puml @@ -0,0 +1,34 @@ +@startuml +box "presentation - CLI" #lightBlue +participant AndroidCorelliumRunCommand +end box + +box "domain - top level" #lightGreen +participant RunTestAndroidCorellium +end box + +box "domain - low level" #aquamarine +participant calculateShards +participant runDevices +participant installApks +participant runTests +end box + + +AndroidCorelliumRunCommand -> RunTestAndroidCorellium +RunTestAndroidCorellium -> calculateShards +RunTestAndroidCorellium <-- calculateShards : Shards +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Shards +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Devices starting +RunTestAndroidCorellium -> runDevices +RunTestAndroidCorellium <-- runDevices : Unit | Exception +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Devices started | Exception +RunTestAndroidCorellium -> installApks +RunTestAndroidCorellium <-- installApks : Unit | Exception +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Devices started | Exception +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Starting tests | Exception +RunTestAndroidCorellium -> runTests +RunTestAndroidCorellium <-- runTests : Flow test status +AndroidCorelliumRunCommand <-- RunTestAndroidCorellium : Tests finished | Exception + +@enduml diff --git a/docs/corellium/android-domain-class.puml b/docs/corellium/android-domain-class.puml new file mode 100644 index 0000000000..87a1164867 --- /dev/null +++ b/docs/corellium/android-domain-class.puml @@ -0,0 +1,33 @@ +@startuml + +abstract class Execution +abstract class Device + +interface Shards >> + +entity TestContext { +shards: List +} + +entity Shard { +appApk: Reference +testApk: Reference +testCases: List +} + +entity TestCase { +name: String +duration: Long +} + +Execution "1" ..> "1" Shards : calculating +Execution "1" ..> "*" Device : invoking +Execution "1" ..> "*" TestContext : running + +TestContext "1" .> "1" Device : on +TestContext "1" o-- "*" Shard + +Shards "1" o-- "*" Shard +Shard "1" o-- "*" TestCase + +@enduml diff --git a/docs/corellium/client-api-class.puml b/docs/corellium/client-api-class.puml new file mode 100644 index 0000000000..558446139d --- /dev/null +++ b/docs/corellium/client-api-class.puml @@ -0,0 +1,50 @@ +@startuml + +left to right direction + +object connectCorellium + +class Corellium +object getAllProjects +object getProjectIdList +object getProjectInstancesList +object createNewInstance +object deleteInstance +object getInstanceInfo +object waitUntilInstanceIsReady +object getVPNConfig +object connectAgent +object connectConsole + +class Agent +object uploadFile + +class Console +object sendCommand +object waitForIdle +object close + + +connectCorellium ..> Corellium + +Corellium -- getAllProjects +Corellium -- getProjectIdList +Corellium -- getProjectInstancesList +Corellium -- createNewInstance +Corellium -- deleteInstance +Corellium -- getInstanceInfo +Corellium -- waitUntilInstanceIsReady +Corellium -- getVPNConfig +Corellium -- connectAgent +Corellium -- connectConsole + +connectAgent ..> Agent +connectConsole ..> Console + +Agent -- uploadFile + +Console -- sendCommand +Console -- waitForIdle +Console -- close + +@enduml diff --git a/docs/corellium/modules.puml b/docs/corellium/modules.puml new file mode 100644 index 0000000000..896045ca5e --- /dev/null +++ b/docs/corellium/modules.puml @@ -0,0 +1,14 @@ +@startuml + +left to right direction + +[cli] #snow +[domain] #snow + +[cli] --> [domain] +[domain] --> [api] +[domain] ...> [adapter] +[api] <-- [adapter] +[adapter] --> [client] + +@enduml