From e2696786c20c1f269ed6ba011c48ef7856dfdee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 24 Jul 2021 13:59:32 +0200 Subject: [PATCH 1/4] Generate tasks diagram --- corellium/domain/README.md | 19 ++-- corellium/domain/TestAndroid-execute.puml | 44 +++++++++ corellium/domain/build.gradle.kts | 1 + .../corellium/domain/AndroidTestDiagram.kt | 12 +++ settings.gradle.kts | 1 + .../parallel/plantuml/build.gradle.kts | 17 ++++ .../exection/parallel/plantuml/Plantuml.kt | 90 +++++++++++++++++++ 7 files changed, 171 insertions(+), 13 deletions(-) create mode 100644 corellium/domain/TestAndroid-execute.puml create mode 100644 corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt create mode 100644 tool/execution/parallel/plantuml/build.gradle.kts create mode 100644 tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt diff --git a/corellium/domain/README.md b/corellium/domain/README.md index 0fb5bbd49b..eccdc62274 100644 --- a/corellium/domain/README.md +++ b/corellium/domain/README.md @@ -9,22 +9,15 @@ This module is specifying public API and internal implementation of Flank-Corell * Public API - files inside [flank.corellium.domain](./src/main/kotlin/flank/corellium/domain) * Internal functions - nested packages inside [flank.corellium.domain](./src/main/kotlin/flank/corellium/domain) -## Design +## Execution -### Stateful execution +Execution can be represented as a graph of tasks relations without cycles. -Following specification is suitable for complicated long-running use-cases, when becomes convenient to split execution into smaller atomic chunks of work. -#### Definition: +#### Version from master branch: -* `Execution` is process which is creating initial `State`, and is running the set of `steps` on it in specific `Context`. -* `Execution` can pass `Context` to `Step` if needed. -* `Step` is a suspendable operation that is receiving `State` as the only argument. -* `Step` must return received or new `State`. -* `Step` can generate side effects. -* `State` is a structure used for sharing data between preceding and following `steps`. -* `Context` is providing arguments and functions for `step`. +![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/master/corellium/domain/TestAndroid-execute.puml) -#### Utility: +### New version draft: -* [`Transform.kt`](src/main/kotlin/flank/corellium/domain/util/Transform.kt) +![TestAndroid.execute graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2083_Add_module_tool-execution-parallel-plantuml/corellium/domain/TestAndroid-execute.puml) diff --git a/corellium/domain/TestAndroid-execute.puml b/corellium/domain/TestAndroid-execute.puml new file mode 100644 index 0000000000..318ea54220 --- /dev/null +++ b/corellium/domain/TestAndroid-execute.puml @@ -0,0 +1,44 @@ +@startuml + +skinparam componentStyle rectangle + +note as N #ffffff +* Brighter tasks are required by the darker tasks. +* The brightness means how fast the task will start. +* White tasks are starting first. +end note + +[Authorize] #fbfbfb +[OutputDir] #fbfbfb +[ParseApkInfo] #fbfbfb +[ParseTestCases] #fbfbfb +[LoadPreviousDurations] #dfdfdf +[PrepareShards] #c3c3c3 +[DumpShards] #a7a7a7 +[InvokeDevices] #a7a7a7 +[InstallApks] #8b8b8b +[ExecuteTests] #6f6f6f +[GenerateReport] #535353 +[CompleteTests] #373737 + +[DumpShards] --> [PrepareShards] +[DumpShards] --> [OutputDir] +[ExecuteTests] --> [PrepareShards] +[ExecuteTests] --> [ParseApkInfo] +[ExecuteTests] --> [Authorize] +[ExecuteTests] --> [InvokeDevices] +[ExecuteTests] --> [InstallApks] +[CompleteTests] --> [GenerateReport] +[CompleteTests] --> [DumpShards] +[GenerateReport] --> [ExecuteTests] +[GenerateReport] --> [OutputDir] +[InstallApks] --> [Authorize] +[InstallApks] --> [PrepareShards] +[InstallApks] --> [InvokeDevices] +[InvokeDevices] --> [Authorize] +[InvokeDevices] --> [PrepareShards] +[LoadPreviousDurations] --> [ParseTestCases] +[PrepareShards] --> [ParseTestCases] +[PrepareShards] --> [LoadPreviousDurations] + +@enduml \ No newline at end of file diff --git a/corellium/domain/build.gradle.kts b/corellium/domain/build.gradle.kts index e04c7f98b3..8e8bbd7d8b 100644 --- a/corellium/domain/build.gradle.kts +++ b/corellium/domain/build.gradle.kts @@ -29,6 +29,7 @@ dependencies { testImplementation(Dependencies.JUNIT) testImplementation(project(":corellium:adapter")) + testImplementation(project(":tool:execution:parallel:plantuml")) } tasks.test { diff --git a/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt b/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt new file mode 100644 index 0000000000..8d610317d2 --- /dev/null +++ b/corellium/domain/src/test/kotlin/flank/corellium/domain/AndroidTestDiagram.kt @@ -0,0 +1,12 @@ +package flank.corellium.domain + +import flank.exection.parallel.plantuml.generatePlanUml +import org.junit.Test + +class AndroidTestDiagram { + + @Test + fun generate() { + TestAndroid.run { generatePlanUml(execute - context.validate) } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 53ff47465d..2f7589d4a4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,7 @@ include( ":tool:log", ":tool:log:format", ":tool:execution:parallel", + ":tool:execution:parallel:plantuml", ":tool:execution:synchronized", ":tool:analytics", ":tool:analytics:mixpanel", diff --git a/tool/execution/parallel/plantuml/build.gradle.kts b/tool/execution/parallel/plantuml/build.gradle.kts new file mode 100644 index 0000000000..cd61feb4d4 --- /dev/null +++ b/tool/execution/parallel/plantuml/build.gradle.kts @@ -0,0 +1,17 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin(Plugins.Kotlin.PLUGIN_JVM) +} + +repositories { + mavenCentral() +} + +tasks.withType { kotlinOptions.jvmTarget = "1.8" } + +dependencies { + api(project(":tool:execution:parallel")) + implementation(Dependencies.KOTLIN_COROUTINES_CORE) + testImplementation(Dependencies.JUNIT) +} diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt new file mode 100644 index 0000000000..3d7559434e --- /dev/null +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt @@ -0,0 +1,90 @@ +package flank.exection.parallel.plantuml + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks +import java.awt.Color +import java.io.File + +infix fun Any.generatePlanUml( + tasks: Tasks, +) { + val name = javaClass.simpleName + val plant = generatePlantUml(tasks) + println(plant) + File("$name-execute.puml").writeText(plant) +} + +fun Any.generatePlantUml( + tasks: Tasks, +): String { + val source = javaClass.name + "$" + val graph: Graph = tasks.associate { task -> task.signature.type to task.signature.args } + val depth: Map = graph.calculateDepth() + val maxDepth: Int = depth.values.maxOrNull() ?: 0 + val colors: Colors = depth.mapValues { (_, value) -> calculateColor(maxDepth, value) } + val name = fun Any.() = javaClass.name.removePrefix(source).replace('$', '.').let { "[$it]" } + + return """ +@startuml + +skinparam componentStyle rectangle + +note as N #ffffff +* Brighter tasks are required by the darker tasks. +* The brightness means how fast the task will start. +* White tasks are starting first. +end note + +${colors.printColors(name)} + +${graph.printRelations(name)} + +@enduml + """.trimIndent() +} + +private typealias Graph = Map> +private typealias Node = Parallel.Type<*> +private typealias Colors = Map + +private fun calculateColor(maxDepth: Int, value: Int): String { + val c = (COLOR_MAX / maxDepth) * (maxDepth - value) + COLOR_OFFSET + return Integer.toHexString(Color(c, c, c).rgb).drop(2) // drop alpha +} + +private const val COLOR_MAX = 200 +private const val COLOR_OFFSET = 55 + +private fun Map>.calculateDepth(): Map { + var jump = 0 + val state = (values.flatten() - keys).toMutableList() + val remaining: MutableMap> = toMutableMap() + val depth = mutableMapOf() + + while (depth.size < size) { + val current = remaining + .filterValues { state.containsAll(it) }.keys + .onEach { depth[it] = jump } + state += current + remaining -= current + jump++ + } + + return depth +} + +private fun Colors.printColors( + name: Node.() -> String +) = toList().joinToString("\n") { (node, value) -> + "${node.name()} #$value" +} + +private fun Graph.printRelations( + name: Node.() -> String +) = filterValues { dependencies -> + dependencies.isNotEmpty() +}.toList().joinToString("\n") { (node, dependencies) -> + dependencies.joinToString("\n") { dep -> + node.name() + " --> " + dep.name() + } +} From 6300ce0cd814b9002221d8b2f71e3632771cd036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 24 Jul 2021 21:32:02 +0200 Subject: [PATCH 2/4] Separate public API from implementation --- .../exection/parallel/plantuml/PlantUml.kt | 66 +++++++++++++++++++ .../plantuml/internal/GeneratePlantUmlFile.kt | 14 ++++ .../GeneratePlantUmlString.kt} | 20 ++---- 3 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt create mode 100644 tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlFile.kt rename tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/{Plantuml.kt => internal/GeneratePlantUmlString.kt} (83%) diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt new file mode 100644 index 0000000000..041ff6c479 --- /dev/null +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt @@ -0,0 +1,66 @@ +package flank.exection.parallel.plantuml + +import flank.exection.parallel.Tasks +import flank.exection.parallel.plantuml.internal.generatePlanUmlFile +import flank.exection.parallel.plantuml.internal.generatePlantUmlString +import java.io.File + +/** + * Generates plant uml file and save on drive. + * + * @receiver Source object of execution. Used for generating filename and reducing tasks names. + * @param tasks Tasks relations graph required to generate diagram. + * @param dir optional path to directory for generated file. + */ +fun Any.generatePlanUml( + tasks: Tasks, + dir: String = "" +): File = generatePlanUmlFile( + tasks = tasks, + path = File(dir, javaClass.simpleName).absolutePath + "-execute.puml", + prefixToRemove = javaClass.name +) + +/** + * Generates plant uml file and save on drive. + * + * @param tasks Tasks relations graph required to generate diagram. + * @param path Path to generated file. + * @param prefixToRemove Optional prefix to remove from each task name. + */ +fun generatePlanUml( + tasks: Tasks, + path: String, + prefixToRemove: String = "", +): File = generatePlanUmlFile( + tasks = tasks, + path = path, + prefixToRemove = prefixToRemove +) + +/** + * Generates plant uml string. + * + * @receiver Source object of execution. Used for reducing tasks names. + * @param tasks Tasks relations graph required to generate diagram. + */ +fun Any.generatePlantUml( + tasks: Tasks +): String = generatePlantUmlString( + tasks = tasks, + prefixToRemove = javaClass.name +) + +/** + * Generates plant uml string. + * + * @param tasks Tasks relations graph required to generate diagram. + * @param prefixToRemove Optional prefix to remove from each task name. + */ +fun generatePlantUml( + tasks: Tasks, + prefixToRemove: String = "", +): String = generatePlantUmlString( + tasks = tasks, + prefixToRemove = prefixToRemove +) diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlFile.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlFile.kt new file mode 100644 index 0000000000..0d4d0eff62 --- /dev/null +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlFile.kt @@ -0,0 +1,14 @@ +package flank.exection.parallel.plantuml.internal + +import flank.exection.parallel.Tasks +import java.io.File + +internal fun generatePlanUmlFile( + tasks: Tasks, + path: String, + prefixToRemove: String, +): File { + val plant = generatePlantUmlString(tasks, prefixToRemove) + println(plant) + return File(path).apply { writeText(plant) } +} diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt similarity index 83% rename from tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt rename to tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt index 3d7559434e..18f63bb1ed 100644 --- a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/Plantuml.kt +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt @@ -1,28 +1,20 @@ -package flank.exection.parallel.plantuml +package flank.exection.parallel.plantuml.internal import flank.exection.parallel.Parallel import flank.exection.parallel.Tasks import java.awt.Color -import java.io.File -infix fun Any.generatePlanUml( - tasks: Tasks, -) { - val name = javaClass.simpleName - val plant = generatePlantUml(tasks) - println(plant) - File("$name-execute.puml").writeText(plant) -} +// =================== Internal API =================== -fun Any.generatePlantUml( +internal fun generatePlantUmlString( tasks: Tasks, + prefixToRemove: String = "", ): String { - val source = javaClass.name + "$" val graph: Graph = tasks.associate { task -> task.signature.type to task.signature.args } val depth: Map = graph.calculateDepth() val maxDepth: Int = depth.values.maxOrNull() ?: 0 val colors: Colors = depth.mapValues { (_, value) -> calculateColor(maxDepth, value) } - val name = fun Any.() = javaClass.name.removePrefix(source).replace('$', '.').let { "[$it]" } + val name = fun Any.() = javaClass.name.removePrefix("$prefixToRemove$").replace('$', '.').let { "[$it]" } return """ @startuml @@ -43,6 +35,8 @@ ${graph.printRelations(name)} """.trimIndent() } +// =================== Private implementation =================== + private typealias Graph = Map> private typealias Node = Parallel.Type<*> private typealias Colors = Map From c3b7e5069a65c7134bd16f8d841f005fcb431a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 24 Jul 2021 21:39:42 +0200 Subject: [PATCH 3/4] Add empty end-line to generated plantuml file --- corellium/domain/TestAndroid-execute.puml | 2 +- .../main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt | 2 +- .../parallel/plantuml/internal/GeneratePlantUmlString.kt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/corellium/domain/TestAndroid-execute.puml b/corellium/domain/TestAndroid-execute.puml index 318ea54220..872c132e14 100644 --- a/corellium/domain/TestAndroid-execute.puml +++ b/corellium/domain/TestAndroid-execute.puml @@ -41,4 +41,4 @@ end note [PrepareShards] --> [ParseTestCases] [PrepareShards] --> [LoadPreviousDurations] -@enduml \ No newline at end of file +@enduml diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt index 041ff6c479..f11233c9cb 100644 --- a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt @@ -17,7 +17,7 @@ fun Any.generatePlanUml( dir: String = "" ): File = generatePlanUmlFile( tasks = tasks, - path = File(dir, javaClass.simpleName).absolutePath + "-execute.puml", + path = File(dir).resolve(javaClass.simpleName).absolutePath + "-execute.puml", prefixToRemove = javaClass.name ) diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt index 18f63bb1ed..0a932ce5b7 100644 --- a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/internal/GeneratePlantUmlString.kt @@ -32,6 +32,7 @@ ${colors.printColors(name)} ${graph.printRelations(name)} @enduml + """.trimIndent() } From 2efede9d92cb3038e3ab01bbd460ca29240621cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 24 Jul 2021 21:41:46 +0200 Subject: [PATCH 4/4] Fix kdoc --- .../exection/parallel/plantuml/PlantUml.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt index f11233c9cb..561c79df14 100644 --- a/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt +++ b/tool/execution/parallel/plantuml/src/main/kotlin/flank/exection/parallel/plantuml/PlantUml.kt @@ -6,11 +6,11 @@ import flank.exection.parallel.plantuml.internal.generatePlantUmlString import java.io.File /** - * Generates plant uml file and save on drive. + * Generates plantuml file and saves on drive. * * @receiver Source object of execution. Used for generating filename and reducing tasks names. - * @param tasks Tasks relations graph required to generate diagram. - * @param dir optional path to directory for generated file. + * @param tasks Graph of tasks required to generate diagram. + * @param dir Optional path to directory for generated file. */ fun Any.generatePlanUml( tasks: Tasks, @@ -22,9 +22,9 @@ fun Any.generatePlanUml( ) /** - * Generates plant uml file and save on drive. + * Generates plantuml file and saves on drive. * - * @param tasks Tasks relations graph required to generate diagram. + * @param tasks Graph of tasks required to generate diagram. * @param path Path to generated file. * @param prefixToRemove Optional prefix to remove from each task name. */ @@ -39,10 +39,10 @@ fun generatePlanUml( ) /** - * Generates plant uml string. + * Generates plantuml string. * * @receiver Source object of execution. Used for reducing tasks names. - * @param tasks Tasks relations graph required to generate diagram. + * @param tasks Graph of tasks required to generate diagram. */ fun Any.generatePlantUml( tasks: Tasks @@ -52,9 +52,9 @@ fun Any.generatePlantUml( ) /** - * Generates plant uml string. + * Generates plantuml string. * - * @param tasks Tasks relations graph required to generate diagram. + * @param tasks Graph of tasks required to generate diagram. * @param prefixToRemove Optional prefix to remove from each task name. */ fun generatePlantUml(