feat: Implement custom sharding -- iOS (#1779)

Flank · Apr 8, 2021 · 18a682f · 18a682f
1 parent e328dd0
commit 18a682f
Show file tree

Hide file tree

Showing 8 changed files with 371 additions and 49 deletions.
diff --git a/docs/feature/1665-custom-sharding.md b/docs/feature/1665-custom-sharding.md
@@ -1,7 +1,5 @@
 # Custom sharding
 
-##### NOTE: Currently for android only, iOS support will be released soon
-
 With [#1665](https://github.com/Flank/flank/issues/1665) Flank received the new feature called `Custom Sharding`. It
 enables Flank to consume predefined sharding and apply it during a test run. The feature gives flexibility and enables
 manual optimization. It also allows users to set up different sharding per app-test apk pair (android only).
@@ -220,6 +218,109 @@ You can now start a flank test run. With the updated config there will still be
     * `debug-2.apk` with 2 shards
     * `debug-3.apk` with 1 shard
 
+## iOS
+
+iOS custom sharding works exactly the same as Android (dump shards, modify, add a path to JSON file, etc) with some small exceptions:
+
+* there is no `additional-app-test-apks` feature for iOS
+* every shard in iOS is a separate matrix (FTL limitations)
+* dump shard JSON is different for xctestrun with test plans, therefore there is different custom sharding JSON
+  structure for both versions.
+
+#### Custom sharding JSON for xctestrun without Test Plans (example)
+
+```json
+[
+  {
+    "ExampleSwiftTests": [
+      "ExampleSwiftTestsClass/test1",
+      "ExampleSwiftTestsClass/test2",
+      "ExampleSwiftTestsClass/test3",
+      "ExampleSwiftTestsClass/test4",
+      "ExampleSwiftTestsClass/test5"
+    ]
+  },
+  {
+    "ExampleSwiftTests": [
+      "ExampleSwiftTestsClass/test6",
+      "ExampleSwiftTestsClass/test7",
+      "ExampleSwiftTestsClass/test8",
+      "ExampleSwiftTestsClass/test9"
+    ]
+  },
+  {
+    "ExampleSwiftTests": [
+      "ExampleSwiftTestsClass/test10",
+      "ExampleSwiftTestsClass/test11",
+      "ExampleSwiftTestsClass/test12",
+      "ExampleSwiftTestsClass/test13"
+    ]
+  },
+  {
+    "ExampleSwiftTests": [
+      "ExampleSwiftTestsClass/test14",
+      "ExampleSwiftTestsClass/test15",
+      "ExampleSwiftTestsClass/test16",
+      "ExampleSwiftTestsClass/test17"
+    ]
+  }
+]
+```
+
+#### Custom sharding JSON for xctestrun with Test Plans (example)
+
+```json
+{
+  "en": [
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_PLLocale",
+        "SecondUITestsClass/test2_3",
+        "SecondUITestsClass/test2_ENLocale"
+      ],
+      "UITests": [
+        "UITestsClass/test1_1",
+        "UITestsClass/test1_2",
+        "UITestsClass/test1_3"
+      ]
+    },
+    {
+      "UITests": [
+        "UITestsClass/test1_ENLocale",
+        "UITestsClass/test1_PLLocale"
+      ]
+    },
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_1",
+        "SecondUITestsClass/test2_2"
+      ]
+    }
+  ],
+  "pl": [
+    {
+      "UITests": [
+        "UITestsClass/test1_1",
+        "UITestsClass/test1_2",
+        "UITestsClass/test1_3",
+        "UITestsClass/test1_ENLocale",
+        "UITestsClass/test1_PLLocale"
+      ]
+    },
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_1",
+        "SecondUITestsClass/test2_2",
+        "SecondUITestsClass/test2_PLLocale",
+        "SecondUITestsClass/test2_3",
+        "SecondUITestsClass/test2_ENLocale"
+      ]
+    }
+  ]
+}
+
+```
+
 ## NOTE:
 
 * flank **DOES NOT** validate the provided custom sharding JSON -- it's your responsibility to provide a proper configuration

diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt
@@ -11,10 +11,13 @@ import ftl.ios.xctest.common.XctestrunMethods
 import ftl.ios.xctest.common.getXcTestRunVersion
 import ftl.ios.xctest.common.mapToRegex
 import ftl.ios.xctest.common.parseToNSDictionary
+import ftl.run.common.fromJson
 import ftl.shard.Chunk
+import ftl.shard.TestMethod
 import ftl.shard.testCases
 import ftl.util.FlankTestMethod
 import java.io.File
+import java.nio.file.Paths
 
 data class XcTestRunData(
     val rootDir: String,
@@ -33,13 +36,16 @@ private fun IosArgs.calculateXcTest(): XcTestRunData {
     val xcTestRoot: String = xcTestRunFile.parent + "/"
     val xcTestNsDictionary: NSDictionary = parseToNSDictionary(xcTestRunFile)
 
-    val calculatedShards: Map<String, Pair<List<Chunk>, List<XctestrunMethods>>> =
-        if (disableSharding) emptyMap()
-        else calculateConfigurationShards(
+    val calculatedShards: Map<String, Pair<List<Chunk>, List<XctestrunMethods>>> = when {
+        disableSharding -> emptyMap()
+        useCustomShardingV1(xcTestNsDictionary) -> shardsFromV1()
+        useCustomShardingV2(xcTestNsDictionary) -> shardsFromV2()
+        else -> calculateConfigurationShards(
             xcTestRoot = xcTestRoot,
             xcTestNsDictionary = xcTestNsDictionary,
             regexList = testTargets.mapToRegex()
         )
+    }
 
     return XcTestRunData(
         rootDir = xcTestRoot,
@@ -49,11 +55,38 @@ private fun IosArgs.calculateXcTest(): XcTestRunData {
     )
 }
 
+private inline fun <reified T> createCustomSharding(shardingJsonPath: String) =
+    fromJson<T>(Paths.get(shardingJsonPath).toFile().readText())
+
+private fun IosArgs.useCustomShardingV1(dictionary: NSDictionary) =
+    customShardingJson.isNotBlank() && dictionary.getXcTestRunVersion() == V1
+
+private fun IosArgs.useCustomShardingV2(dictionary: NSDictionary) =
+    customShardingJson.isNotBlank() && dictionary.getXcTestRunVersion() == V2
+
+private fun IosArgs.shardsFromV1() = mapOf(
+    "" to createCustomSharding<List<XctestrunMethods>>(commonArgs.customShardingJson)
+        .run {
+            map { xcMethods ->
+                Chunk(xcMethods.values.flatMap { it.map(::TestMethod) })
+            } to this
+        }
+)
+
+private fun IosArgs.shardsFromV2() =
+    createCustomSharding<Map<String, List<XctestrunMethods>>>(commonArgs.customShardingJson)
+        .mapValues { (_, xcMethodsList) ->
+            xcMethodsList.map { xcMethods ->
+                Chunk(xcMethods.values.flatMap { it.map(::TestMethod) })
+            } to xcMethodsList
+        }
+
 private fun emptyXcTestRunData() = XcTestRunData(
     rootDir = "",
     nsDict = NSDictionary(),
     version = V1
 )
+
 private fun IosArgs.filterTestConfigurationsIfNeeded(
     configurations: Map<String, Map<String, List<String>>>
 ): Map<String, Map<String, List<String>>> = when {

diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt
@@ -38,13 +38,9 @@ fun IosArgs.dumpShards(
     // VisibleForTesting
     shardFilePath: String = IOS_SHARD_FILE,
 ) {
-    val xcTestRunShards: Map<String, List<List<String>>> = xcTestRunData.shardTargets.mapValues {
-        it.value.flatMap { it.values }
-    }
-
     val rawShards: Any = when (xcTestRunData.version) {
-        V1 -> xcTestRunShards.values.first()
-        V2 -> xcTestRunShards
+        V1 -> xcTestRunData.shardTargets.values.first()
+        V2 -> xcTestRunData.shardTargets
     }
 
     val size = xcTestRunData.shardTargets.values

diff --git a/test_runner/src/main/kotlin/ftl/util/ObfuscationGson.kt b/test_runner/src/main/kotlin/ftl/util/ObfuscationGson.kt
@@ -3,12 +3,14 @@ package ftl.util
 import com.google.common.annotations.VisibleForTesting
 import com.google.gson.GsonBuilder
 import com.google.gson.JsonArray
+import com.google.gson.JsonObject
 import com.google.gson.JsonSerializationContext
 import com.google.gson.JsonSerializer
 import com.google.gson.reflect.TypeToken
-import ftl.args.ShardChunks
 import java.lang.reflect.Type
 
+private typealias CustomShardChunks = List<Map<String, List<String>>>
+
 val obfuscatePrettyPrinter = GsonBuilder()
     .registerTypeHierarchyAdapter(ListOfStringListTypeToken.rawType, ObfuscatedIosJsonSerializer)
     .registerTypeAdapter(ListOfStringTypeToken.rawType, ObfuscatedAndroidJsonSerializer)
@@ -19,7 +21,7 @@ val obfuscatePrettyPrinter = GsonBuilder()
 internal object ListOfStringTypeToken : TypeToken<List<String>>()
 
 @VisibleForTesting
-internal object ListOfStringListTypeToken : TypeToken<ShardChunks>()
+internal object ListOfStringListTypeToken : TypeToken<CustomShardChunks>()
 
 private object ObfuscatedAndroidJsonSerializer : JsonSerializer<List<String>> {
     private val obfuscationContext by lazy { mutableMapOf<String, MutableMap<String, String>>() }
@@ -36,20 +38,27 @@ private object ObfuscatedAndroidJsonSerializer : JsonSerializer<List<String>> {
     }
 }
 
-private object ObfuscatedIosJsonSerializer : JsonSerializer<List<List<String>>> {
+private object ObfuscatedIosJsonSerializer : JsonSerializer<CustomShardChunks> {
     private val obfuscationContext by lazy { mutableMapOf<String, MutableMap<String, String>>() }
 
     override fun serialize(
-        src: List<List<String>>,
+        src: CustomShardChunks,
         typeOfSrc: Type,
         context: JsonSerializationContext
     ) = JsonArray().also { jsonArray ->
-        src.forEach { list ->
-            jsonArray.add(
-                JsonArray().also { nestedJsonArray ->
-                    list.forEach { nestedJsonArray.add(obfuscationContext.obfuscateIosTestName(it)) }
-                }
-            )
+        src.forEach { xcTestMethodList ->
+            val xcTestMethodJson = JsonObject()
+            xcTestMethodList.forEach {
+                xcTestMethodJson.add(
+                    obfuscationContext.obfuscateIosTestName(it.key),
+                    JsonArray().also { nestedJsonArray ->
+                        it.value.forEach {
+                            nestedJsonArray.add(obfuscationContext.obfuscateIosTestName(it))
+                        }
+                    }
+                )
+            }
+            jsonArray.add(xcTestMethodJson)
         }
     }
 }
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/custom_sharding_ios_v1.json b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/custom_sharding_ios_v1.json
@@ -0,0 +1,39 @@
+[
+  {
+    "EarlGreyExampleSwiftTests": [
+      "EarlGreyExampleSwiftTests/testWithGreyAssertions",
+      "EarlGreyExampleSwiftTests/testWithCondition",
+      "EarlGreyExampleSwiftTests/testBasicSelection"
+    ]
+  },
+  {
+    "EarlGreyExampleSwiftTests": [
+      "EarlGreyExampleSwiftTests/testLayout",
+      "EarlGreyExampleSwiftTests/testCustomAction",
+      "EarlGreyExampleSwiftTests/testWithCustomMatcher",
+      "EarlGreyExampleSwiftTests/testWithCustomAssertion"
+    ]
+  },
+  {
+    "EarlGreyExampleSwiftTests": [
+      "EarlGreyExampleSwiftTests/testThatThrows",
+      "EarlGreyExampleSwiftTests/testWithInRoot",
+      "EarlGreyExampleSwiftTests/testCollectionMatchers",
+      "EarlGreyExampleSwiftTests/testCatchErrorOnFailure"
+    ]
+  },
+  {
+    "EarlGreyExampleSwiftTests": [
+      "EarlGreyExampleSwiftTests/testTableCellOutOfScreen",
+      "EarlGreyExampleSwiftTests/testBasicSelectionAndAction"
+    ]
+  },
+  {
+    "EarlGreyExampleSwiftTests": [
+      "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert",
+      "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements",
+      "EarlGreyExampleSwiftTests/testWithCustomFailureHandler",
+      "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert"
+    ]
+  }
+]
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/custom_sharding_ios_v2.json b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/custom_sharding_ios_v2.json
@@ -0,0 +1,48 @@
+{
+  "en": [
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_PLLocale",
+        "SecondUITestsClass/test2_3",
+        "SecondUITestsClass/test2_ENLocale"
+      ],
+      "UITests": [
+        "UITestsClass/test1_1",
+        "UITestsClass/test1_2",
+        "UITestsClass/test1_3"
+      ]
+    },
+    {
+      "UITests": [
+        "UITestsClass/test1_ENLocale",
+        "UITestsClass/test1_PLLocale"
+      ]
+    },
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_1",
+        "SecondUITestsClass/test2_2"
+      ]
+    }
+  ],
+  "pl": [
+    {
+      "UITests": [
+        "UITestsClass/test1_1",
+        "UITestsClass/test1_2",
+        "UITestsClass/test1_3",
+        "UITestsClass/test1_ENLocale",
+        "UITestsClass/test1_PLLocale"
+      ]
+    },
+    {
+      "SecondUITests": [
+        "SecondUITestsClass/test2_1",
+        "SecondUITestsClass/test2_2",
+        "SecondUITestsClass/test2_PLLocale",
+        "SecondUITestsClass/test2_3",
+        "SecondUITestsClass/test2_ENLocale"
+      ]
+    }
+  ]
+}