feat: Add sharding implementation for corellium

corellium/shard/README.md

@@ -0,0 +1,100 @@
+# Sharding
+
+Depending on the provided test cases duration, the sharding algorithm will:
+
+* Split the test cases from one test apk to run on many devices.
+* Group the test cases from many test apks to run on a single device.
+* A mix both of the above options.
+
+## Diagram
+
+![sharding_class_diagram](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/1801_Multi-module_sharding_algorithm/docs/corellium/sharding-class.puml)
+
+## Example
+
+### Input
+
+```yaml
+bundle:
+  - app: app1
+    tests:
+      - test: app1-test1
+        cases:
+          - "class app1.test1.TestClass#test1" // 1s
+  - app: app2
+    tests:
+      - test: app2-test1
+        cases:
+          - "class app2.test1.TestClass#test2" // 2s
+          - "class app2.test1.TestClass#test3" // 3s
+      - test: app2-test2
+        cases:
+          - "class app2.test2.TestClass#test7" // 7s
+          - "class app2.test2.TestClass#test8" // 8s
+          - "class app2.test2.TestClass#test9" // 9s
+```
+
+### Output
+
+#### Max shards 3
+
+```yaml
+shards:
+  - shard1:
+      - app: app1
+        tests:
+          - test: app1-test1
+            cases:
+              - "class app1.test1.TestClass#test1" // 1s
+      - app: app2
+        tests:
+          - test: app2-test2
+            cases:
+              - "class app2.test2.TestClass#test9" // 9s
+  - shard2:
+      - app: app2
+        test: app2-test1
+        cases:
+          - "class app1.test2.TestClass#test2" // 2s
+      - app: app2
+        test: app2-test2
+        cases:
+          - "class app2.test2.TestClass#test8" // 8s
+  - shard3:
+      - app: app2
+        test: app2-test1
+        cases:
+          - "class app2.test1.TestClass#test3" // 3s
+      - app: app2
+        test: app2-test2
+        cases:
+          - "class app2.test2.TestClass#test7" // 7s
+```
+
+#### Max shards 2
+
+```yaml
+shards:
+  - shard1:
+      - app: app1
+        tests:
+          - test: app1-test1
+            cases:
+              - "class app1.test1.TestClass#test1" // 1sz
+      - app: app2
+        tests:
+          - test: app2-test1
+            cases:
+              - "class app2.test1.TestClass#test2" // 2s
+              - "class app2.test2.TestClass#test3" // 3s
+          - test: app2-test2
+            cases:
+              - "class app2.test2.TestClass#test9" // 9s
+  - shard2:
+      - app: app2
+        tests:
+          - test: app2-test2
+            cases:
+              - "class app2.test2.TestClass#test7" // 7s
+              - "class app2.test2.TestClass#test8" // 8s
+```

corellium/shard/build.gradle.kts

@@ -0,0 +1,21 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+plugins {
+    kotlin(Plugins.Kotlin.PLUGIN_JVM)
+    kotlin(Plugins.Kotlin.PLUGIN_SERIALIZATION) version Versions.KOTLIN
+}
+
+repositories {
+    jcenter()
+    mavenCentral()
+    maven(url = "https://kotlin.bintray.com/kotlinx")
+}
+
+tasks.withType<KotlinCompile> { kotlinOptions.jvmTarget = "1.8" }
+
+dependencies {
+    implementation(Dependencies.KOTLIN_SERIALIZATION)
+    implementation(Dependencies.KOTLIN_COROUTINES_CORE)
+
+    testImplementation(Dependencies.JUNIT)
+}

corellium/shard/src/main/kotlin/flank/corellium/shard/Internal.kt

@@ -0,0 +1,105 @@
+package flank.corellium.shard
+
+import kotlin.math.min
+
+// Stage 1
+
+/**
+ * Create flat and easy to iterate list of [Chunk] without losing info about app and test related to test case.
+ * @receiver The [List] of [Shard.App].
+ * @return The [List] of [Chunk] which is just different representation of input data.
+ */
+internal fun List<Shard.App>.mapToInternalChunks(): List<Chunk> =
+    mutableListOf<Chunk>().also { result ->
+        forEach { app ->
+            app.tests.forEach { test ->
+                test.cases.forEach { case ->
+                    result.add(
+                        Chunk(
+                            app = app.name,
+                            test = test.name,
+                            case = case.name,
+                            duration = case.duration
+                        )
+                    )
+                }
+            }
+        }
+    }
+
+/**
+ * Internal intermediate structure which is representing test case with associated app and test module.
+ */
+internal class Chunk(
+    val app: String,
+    val test: String,
+    val case: String,
+    val duration: Long,
+)
+
+// Stage 2
+
+/**
+ * Group the chunks into sub-lists where the standard deviation of group duration should by possibly small.
+ * @receiver The flat [List] of [Chunk].
+ * @return The [List] of [Chunk] groups balanced by the summary duration of each.
+ */
+internal fun List<Chunk>.groupByDuration(maxCount: Int): List<List<Chunk>> {
+    class Chunks(
+        var duration: Long = 0,
+        val list: MutableList<Chunk> = mutableListOf()
+    )
+
+    // The real part of sharding calculations,
+    // everything else is just a mapping between structures.
+    // ===================================================
+    return sortedByDescending(Chunk::duration).fold(
+        initial = List(min(size, maxCount)) { Chunks() }
+    ) { acc, chunk ->
+        acc.first().apply {
+            duration += chunk.duration
+            list += chunk
+        }
+        acc.sortedBy(Chunks::duration)
+    }.map(Chunks::list)
+    // ===================================================
+}
+
+// Stage 3
+
+/**
+ * Build the final structure of shards.
+ */
+internal fun List<List<Chunk>>.mapToShards(): List<List<Shard.App>> {
+
+    // Structure the group of chunks mapping them by app and test.
+    val list: List<Map<String, Map<String, List<Chunk>>>> = map { group ->
+        group.groupBy { chunk ->
+            chunk.app
+        }.mapValues { (_, chunks) ->
+            chunks.groupBy { chunk ->
+                chunk.test
+            }
+        }
+    }
+
+    // Convert grouped chunks into the output structures.
+    return list.map { map ->
+        map.map { (app, tests) ->
+            Shard.App(
+                name = app,
+                tests = tests.map { (test, chunks) ->
+                    Shard.Test(
+                        name = test,
+                        cases = chunks.map { chunk ->
+                            Shard.Test.Case(
+                                name = chunk.case,
+                                duration = chunk.duration
+                            )
+                        }
+                    )
+                }
+            )
+        }
+    }
+}

corellium/shard/src/main/kotlin/flank/corellium/shard/Shard.kt

@@ -0,0 +1,36 @@
+package flank.corellium.shard
+
+/**
+ * Distribute the test cases into the [List] of shards where each shard have similar duration.
+ * @receiver The bunch of test cases grouped by test and app.
+ * @return [List] of shards where each shard may contains many apps and test cases.
+ */
+fun List<Shard.App>.calculateShards(
+    maxCount: Int
+): List<List<Shard.App>> = this
+    .mapToInternalChunks()
+    .groupByDuration(maxCount)
+    .mapToShards()
+
+/**
+ * Namespace for sharding input and output structures
+ */
+object Shard {
+
+    class App(
+        val name: String,
+        val tests: List<Test>
+    )
+
+    class Test(
+        val name: String,
+        val cases: List<Case>
+    ) {
+        class Case(
+            val name: String,
+            val duration: Long = DEFAULT_DURATION
+        )
+    }
+
+    private const val DEFAULT_DURATION = 120L
+}

corellium/shard/src/test/kotlin/flank/corellium/shard/ShardKtTest.kt

@@ -0,0 +1,107 @@
+package flank.corellium.shard
+
+import org.junit.Assert
+import org.junit.Test as JTest
+
+val apps = listOf(
+    Shard.App(
+        name = "app1",
+        tests = listOf(
+            Shard.Test(
+                name = "app1-test1",
+                cases = listOf(
+                    Shard.Test.Case(
+                        name = "class app1.test1.TestClass#test1",
+                        duration = 1
+                    ),
+                )
+            ),
+        )
+    ),
+    Shard.App(
+        name = "app2",
+        tests = listOf(
+            Shard.Test(
+                name = "app2-test1",
+                cases = listOf(
+                    Shard.Test.Case(
+                        name = "class app2.test1.TestClass#test2",
+                        duration = 2
+                    ),
+                    Shard.Test.Case(
+                        name = "class app2.test1.TestClass#test3",
+                        duration = 3
+                    ),
+                )
+            ),
+            Shard.Test(
+                name = "app2-test2",
+                cases = listOf(
+                    Shard.Test.Case(
+                        name = "class app2.test2.TestClass#test7",
+                        duration = 7
+                    ),
+                    Shard.Test.Case(
+                        name = "class app2.test2.TestClass#test8",
+                        duration = 8
+                    ),
+                    Shard.Test.Case(
+                        name = "class app2.test2.TestClass#test9",
+                        duration = 9
+                    ),
+                )
+            ),
+        )
+    )
+)
+
+class ShardKtTest {
+
+    @JTest
+    fun test2() {
+        apps.calculateShards(2).apply {
+            printShards()
+            verifyDurationEqual()
+        }
+    }
+
+    @JTest
+    fun test3() {
+        apps.calculateShards(3).apply {
+            printShards()
+            verifyDurationEqual()
+        }
+    }
+}
+
+private fun List<List<Shard.App>>.printShards() {
+    forEach {
+        it.forEach {
+            println(it.name)
+            it.tests.forEach {
+                println(it.name)
+                it.cases.forEach {
+                    println(it.name + ": " + it.duration)
+                }
+            }
+        }
+        println()
+    }
+}
+
+private fun List<List<Shard.App>>.verifyDurationEqual() {
+    map {
+        it.sumByDouble {
+            it.tests.sumByDouble {
+                it.cases.sumByDouble {
+                    it.duration.toDouble()
+                }
+            }
+        }
+    }.map {
+        it.toLong()
+    }.reduce { first, next ->
+        Assert.assertEquals(first, next)
+        first
+    }
+}