From 7457b20c88d8cad22b1d3c7665be56cdf60936a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Tue, 8 Dec 2020 14:29:54 +0100 Subject: [PATCH] feat: iOS support for testplans (#1321) Fixes #685 ## Documentation https://github.com/Flank/flank/blob/master/docs/feature/ios_test_plans.md ## Test Plan > How do we know the code works? From repository root: ``` . .env flankScripts testArtifacts -b master resolve flankScripts shell buildFlank cd test_runner flank ios run -c=./src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml ``` Feel free to modify `flank-xctestrunv2-all.yml` to test possible edge-cases. ## Checklist - [x] Documented - [x] Unit tested --- gradle.properties | 2 +- .../kotlin/ftl/args/CalculateShardsResult.kt | 5 +- .../src/main/kotlin/ftl/args/IosArgs.kt | 17 +- .../main/kotlin/ftl/args/ValidateIosArgs.kt | 30 ++ .../src/main/kotlin/ftl/gc/GcIosTestMatrix.kt | 19 +- .../ftl/ios/xctest/FindXcTestNamesV1.kt | 27 +- .../ftl/ios/xctest/FindXcTestNamesV2.kt | 41 +-- .../ftl/ios/xctest/ReduceXcTestRunV1.kt | 26 ++ .../ftl/ios/xctest/ReduceXcTestRunV2.kt | 46 +++ .../ftl/ios/xctest/RewriteXcTestRunV1.kt | 36 --- .../ftl/ios/xctest/RewriteXcTestRunV2.kt | 52 ---- .../main/kotlin/ftl/ios/xctest/XcTestData.kt | 98 ++++++ .../kotlin/ftl/ios/xctest/XcTestRunFlow.kt | 42 +++ .../main/kotlin/ftl/ios/xctest/common/Util.kt | 32 +- .../src/main/kotlin/ftl/run/DumpShards.kt | 22 +- .../kotlin/ftl/run/platform/RunIosTests.kt | 64 ++-- .../src/main/kotlin/ftl/shard/Chunk.kt | 6 +- .../main/kotlin/ftl/util/FlankTestMethod.kt | 6 +- .../src/main/kotlin/ftl/util/FlowExt.kt | 11 + test_runner/src/test/kotlin/Debug.kt | 10 +- .../test/kotlin/ftl/args/IosArgsFileTest.kt | 3 +- .../src/test/kotlin/ftl/args/IosArgsTest.kt | 7 +- .../kotlin/ftl/fixtures/ios_test_plan.yml | 6 +- .../test_app_cases/flank-xctestrunv2-all.yml | 6 + .../test/kotlin/ftl/gc/GcIosTestMatrixTest.kt | 18 +- .../ftl/ios/xctest/FindXcTestNamesV1KtTest.kt | 10 +- .../ftl/ios/xctest/FindXcTestNamesV2KtTest.kt | 10 +- .../ftl/ios/xctest/ReduceXcTestRunV1KtTest.kt | 199 ++++++++++++ .../ftl/ios/xctest/ReduceXcTestRunV2KtTest.kt | 283 ++++++++++++++++++ .../ios/xctest/RewriteXcTestRunV1KtTest.kt | 183 ----------- .../ios/xctest/RewriteXcTestRunV2KtTest.kt | 60 ---- .../src/test/kotlin/ftl/ios/xctest/Util.kt | 4 +- .../test/kotlin/ftl/run/DumpShardsKtTest.kt | 25 +- 33 files changed, 924 insertions(+), 482 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV1.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV2.kt delete mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt delete mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/XcTestRunFlow.kt create mode 100644 test_runner/src/main/kotlin/ftl/util/FlowExt.kt create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV1KtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV2KtTest.kt delete mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt delete mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt diff --git a/gradle.properties b/gradle.properties index e3ef2959d7..b5f8cf52a7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -13,4 +13,4 @@ org.gradle.jvmargs=-Xmx2560m -XX:MaxPermSize=256m -Dfile.encoding=UTF-8 kotlin.code.style=official org.gradle.parallel=true org.gradle.daemon=true -org.gradle.caching=true +org.gradle.caching=false diff --git a/test_runner/src/main/kotlin/ftl/args/CalculateShardsResult.kt b/test_runner/src/main/kotlin/ftl/args/CalculateShardsResult.kt index 2f4c8fd4b0..d07ba256f6 100644 --- a/test_runner/src/main/kotlin/ftl/args/CalculateShardsResult.kt +++ b/test_runner/src/main/kotlin/ftl/args/CalculateShardsResult.kt @@ -2,4 +2,7 @@ package ftl.args import ftl.shard.Chunk -data class CalculateShardsResult(val shardChunks: List, val ignoredTestCases: IgnoredTestCases) +data class CalculateShardsResult( + val shardChunks: List, + val ignoredTestCases: IgnoredTestCases +) diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 2684c94ffb..aefa3a30d0 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -1,11 +1,10 @@ package ftl.args import com.google.common.annotations.VisibleForTesting +import ftl.ios.xctest.XcTestRunData +import ftl.ios.xctest.calculateXcTestRunData import ftl.ios.xctest.common.XctestrunMethods -import ftl.ios.xctest.findXcTestNamesV1 import ftl.run.exception.FlankConfigurationError -import ftl.shard.Chunk -import ftl.util.FlankTestMethod data class IosArgs( val commonArgs: CommonArgs, @@ -20,7 +19,7 @@ data class IosArgs( ) : IArgs by commonArgs { override val useLegacyJUnitResult = true - val testShardChunks: List by lazy { calculateShardChunks() } + val xcTestRunData: XcTestRunData by lazy { calculateXcTestRunData() } companion object : IosArgsCompanion() @@ -77,16 +76,6 @@ IosArgs } } -private fun IosArgs.calculateShardChunks() = if (disableSharding) - emptyList() else - ArgsHelper.calculateShards( - filteredTests = filterTests(findXcTestNamesV1(xctestrunFile), testTargets) - .flatMap { it.value } - .distinct() - .map { FlankTestMethod(it, ignored = false) }, - args = this - ).shardChunks - @VisibleForTesting internal fun filterTests( validTestMethods: XctestrunMethods, diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt index db74f12cb7..b8c28519eb 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -3,6 +3,7 @@ package ftl.args import ftl.args.yml.Type import ftl.ios.IosCatalog import ftl.ios.IosCatalog.getSupportedVersionId +import ftl.ios.xctest.common.mapToRegex import ftl.run.exception.FlankConfigurationError import ftl.run.exception.IncompatibleTestDimensionError @@ -17,6 +18,7 @@ fun IosArgs.validate() = apply { assertAdditionalIpas() validType() assertGameloop() + assertXcTestRunData() } private fun IosArgs.assertGameloop() { @@ -84,3 +86,31 @@ private fun IosArgs.validType() { if (commonArgs.type !in validIosTypes) throw FlankConfigurationError("Type should be one of ${validIosTypes.joinToString(",")}") } + +private fun IosArgs.assertXcTestRunData() { + if (!disableSharding && testTargets.isNotEmpty()) { + val filteredMethods = xcTestRunData + .shardTargets.values + .flatten() + .flatMap { it.values } + .flatten() + + if (filteredMethods.isEmpty()) throw FlankConfigurationError( + "Empty shards. Cannot match any method to $testTargets" + ) + + if (filteredMethods.size < testTargets.size) { + val regexList = testTargets.mapToRegex() + + val notMatched = testTargets.filter { + filteredMethods.all { method -> + regexList.any { regex -> + regex.matches(method) + } + } + } + + println("WARNING: cannot match test_targets: $notMatched") + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index b804a0c79d..d998acac05 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -1,6 +1,5 @@ package ftl.gc -import com.dd.plist.NSDictionary import com.google.testing.Testing import com.google.testing.model.ClientInfo import com.google.testing.model.EnvironmentMatrix @@ -17,8 +16,6 @@ import ftl.args.IosArgs import ftl.gc.android.mapGcsPathsToFileReference import ftl.gc.android.mapToIosDeviceFiles import ftl.gc.android.toIosDeviceFile -import ftl.ios.xctest.common.toByteArray -import ftl.ios.xctest.rewriteXcTestRunV1 import ftl.run.exception.FlankGeneralError import ftl.util.ShardCounter import ftl.util.join @@ -30,10 +27,8 @@ object GcIosTestMatrix { fun build( iosDeviceList: IosDeviceList, testZipGcsPath: String, - runGcsPath: String, - xcTestParsed: NSDictionary, args: IosArgs, - testTargets: List, + xcTestRun: ByteArray, shardCounter: ShardCounter, toolResultsHistory: ToolResultsHistory, otherFiles: Map, @@ -45,23 +40,15 @@ object GcIosTestMatrix { val gcsBucket = args.resultsBucket val shardName = shardCounter.next() - val matrixGcsSuffix = join(runGcsPath, shardName) + val matrixGcsSuffix = join(args.resultsDir, shardName) val matrixGcsPath = join(gcsBucket, matrixGcsSuffix) - // Parameterized tests on iOS don't shard correctly. - // Avoid changing Xctestrun file when disableSharding is on. - val generatedXctestrun = if (args.disableSharding) { - xcTestParsed.toByteArray() - } else { - rewriteXcTestRunV1(args.xctestrunFile, testTargets) - } - // Add shard number to file name val xctestrunNewFileName = StringBuilder(args.xctestrunFile).insert(args.xctestrunFile.lastIndexOf("."), "_$shardName").toString() val xctestrunFileGcsPath = - GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, generatedXctestrun) + GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, xcTestRun) val iOSXCTest = IosXcTest() .setTestsZip(FileReference().setGcsPath(testZipGcsPath)) diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt index c0b10735a3..d8dae6ea23 100644 --- a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt @@ -1,23 +1,22 @@ package ftl.ios.xctest import com.dd.plist.NSDictionary -import ftl.ios.xctest.common.XctestrunMethods import ftl.ios.xctest.common.findTestsForTarget import ftl.ios.xctest.common.isMetadata -import ftl.ios.xctest.common.parseToNSDictionary -import java.io.File -internal fun findXcTestNamesV1(xctestrun: String): XctestrunMethods = - findXcTestNamesV1(File(xctestrun)) - -private fun findXcTestNamesV1(xctestrun: File): XctestrunMethods = - parseToNSDictionary(xctestrun).run { - val testRoot = xctestrun.parent + "/" - allKeys().filterNot(String::isMetadata).map { testTarget -> +internal fun findXcTestNamesV1( + xcTestRoot: String, + xcTestNsDictionary: NSDictionary +): Map> = + xcTestNsDictionary + .allKeys() + .filterNot(String::isMetadata) + .map { testTarget -> testTarget to findTestsForTarget( - testRoot = testRoot, - testTargetDict = get(testTarget) as NSDictionary, + testRoot = xcTestRoot, + testTargetDict = xcTestNsDictionary[testTarget] as NSDictionary, testTargetName = testTarget, ) - }.distinct().toMap() - } + } + .distinct() + .toMap() diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt index 1fc757370b..6a87bc63fa 100644 --- a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt @@ -1,27 +1,30 @@ package ftl.ios.xctest -import ftl.ios.xctest.common.XctestrunMethods +import com.dd.plist.NSDictionary import ftl.ios.xctest.common.findTestsForTarget import ftl.ios.xctest.common.getBlueprintName import ftl.ios.xctest.common.getTestConfigurations import ftl.ios.xctest.common.getTestTargets -import ftl.ios.xctest.common.parseToNSDictionary -import java.io.File -internal fun findXcTestNamesV2(xctestrun: String): Map = - findXcTestNamesV2(File(xctestrun)) +internal fun findXcTestNamesV2( + xcTestRoot: String, + xcTestNsDictionary: NSDictionary +): Map>> = + xcTestNsDictionary + .getTestConfigurations() + .mapValues { (_, configDict: NSDictionary) -> + configDict.findTestTargetMethods(xcTestRoot) + } -private fun findXcTestNamesV2(xctestrun: File): Map { - val testRoot = xctestrun.parent + "/" - return parseToNSDictionary(xctestrun).getTestConfigurations().mapValues { (_, configDict) -> - configDict.getTestTargets() - .associateBy { targetDict -> targetDict.getBlueprintName() } - .mapValues { (name, dict) -> - findTestsForTarget( - testRoot = testRoot, - testTargetName = name, - testTargetDict = dict, - ) - } - } -} +private fun NSDictionary.findTestTargetMethods( + xcTestRoot: String +): Map> = + getTestTargets() + .associateBy { targetDict -> targetDict.getBlueprintName() } + .mapValues { (name, dict) -> + findTestsForTarget( + testRoot = xcTestRoot, + testTargetName = name, + testTargetDict = dict, + ) + } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV1.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV1.kt new file mode 100644 index 0000000000..feb9dd609c --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV1.kt @@ -0,0 +1,26 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import ftl.ios.xctest.common.XCTEST_METADATA +import ftl.ios.xctest.common.setOnlyTestIdentifiers + +fun NSDictionary.reduceXcTestRunV1( + targets: Map>, +): NSDictionary = apply { + (keys - targets.keys - XCTEST_METADATA).forEach(this::remove) + targets.forEach { (testTarget, methods) -> + setOnlyTestIdentifiers(testTarget, methods) + } +} + +private fun NSDictionary.setOnlyTestIdentifiers( + testTarget: String, + methods: List +) { + set( + testTarget, + getValue(testTarget) + .let { it as NSDictionary } + .setOnlyTestIdentifiers(methods) + ) +} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV2.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV2.kt new file mode 100644 index 0000000000..b20d0e65db --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV2.kt @@ -0,0 +1,46 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.dd.plist.NSObject +import ftl.ios.xctest.common.TEST_CONFIGURATIONS +import ftl.ios.xctest.common.TEST_TARGETS +import ftl.ios.xctest.common.getBlueprintName +import ftl.ios.xctest.common.getTestConfigurations +import ftl.ios.xctest.common.getTestTargets +import ftl.ios.xctest.common.setOnlyTestIdentifiers + +fun NSDictionary.reduceXcTestRunV2( + configuration: String, + targets: Map> +): NSDictionary = apply { + set( + TEST_CONFIGURATIONS, + getTestConfigurations() + .getValue(configuration) + .reduceTestConfiguration(targets) + .inNSArray() + ) +} + +private fun NSDictionary.reduceTestConfiguration( + targetMethods: Map> +): NSDictionary = apply { + set( + TEST_TARGETS, + getTestTargets().mapNotNull { target -> + targetMethods[target.getBlueprintName()]?.let { methods -> + target.setOnlyTestIdentifiers(methods) + } + }.toNsArray() + ) +} + +private fun List.toNsArray(): NSArray = + NSArray(size).also { nsArray -> + forEachIndexed { index, t -> + nsArray.setValue(index, t) + } + } + +private fun NSObject.inNSArray() = NSArray(1).also { it.setValue(0, this) } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt deleted file mode 100644 index 4bfce6a2bd..0000000000 --- a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt +++ /dev/null @@ -1,36 +0,0 @@ -package ftl.ios.xctest - -import com.dd.plist.NSDictionary -import com.google.common.annotations.VisibleForTesting -import ftl.ios.xctest.common.XctestrunMethods -import ftl.ios.xctest.common.isMetadata -import ftl.ios.xctest.common.parseToNSDictionary -import ftl.ios.xctest.common.setOnlyTestIdentifiers -import ftl.ios.xctest.common.toByteArray - -fun rewriteXcTestRunV1( - xctestrun: String, - filterMethods: List -): ByteArray = - rewriteXcTestRunV1( - parseToNSDictionary(xctestrun), - findXcTestNamesV1(xctestrun).mapValues { (_, list) -> - list.filter(filterMethods::contains) - } - ) - -@VisibleForTesting -internal fun rewriteXcTestRunV1( - root: NSDictionary, - methods: XctestrunMethods -): ByteArray = - root.clone().apply { - allKeys().filterNot(String::isMetadata).map { testTarget -> - Pair( - get(testTarget) as NSDictionary, - methods[testTarget] ?: emptyList() - ) - }.forEach { (dict, methods) -> - dict.setOnlyTestIdentifiers(methods) - } - }.toByteArray() diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt deleted file mode 100644 index 10c35c0916..0000000000 --- a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt +++ /dev/null @@ -1,52 +0,0 @@ -package ftl.ios.xctest - -import com.dd.plist.NSArray -import com.dd.plist.NSDictionary -import com.dd.plist.NSObject -import com.google.common.annotations.VisibleForTesting -import ftl.ios.xctest.common.TEST_CONFIGURATIONS -import ftl.ios.xctest.common.XctestrunMethods -import ftl.ios.xctest.common.getBlueprintName -import ftl.ios.xctest.common.getTestConfigurations -import ftl.ios.xctest.common.getTestTargets -import ftl.ios.xctest.common.parseToNSDictionary -import ftl.ios.xctest.common.setOnlyTestIdentifiers -import ftl.ios.xctest.common.toByteArray - -fun rewriteXcTestRunV2( - xcTestPlan: String, - filterMethods: List = emptyList() -): Map = - rewriteXcTestRunV2( - parseToNSDictionary(xcTestPlan), - findXcTestNamesV2(xcTestPlan).filterXcTestMethods(filterMethods), - ) - -@VisibleForTesting -internal fun Map.filterXcTestMethods( - names: List -) = mapValues { (_, map) -> - map.mapValues { (_, list) -> - list.filter(names::contains) - } -} - -@VisibleForTesting -internal fun rewriteXcTestRunV2( - root: NSDictionary, - methods: Map -): Map = - root.getTestConfigurations().mapValues { (configName, configDict) -> - root.clone().also { rootClone -> - rootClone[TEST_CONFIGURATIONS] = configDict.clone().apply { - getTestTargets().forEach { target -> - val testMethods = methods[configName] - ?.get(target.getBlueprintName()) - ?: emptyList() - target.setOnlyTestIdentifiers(testMethods) - } - }.wrapInNSArray() - }.toByteArray() - }.toMap() - -private fun NSObject.wrapInNSArray() = NSArray(1).also { it.setValue(0, this) } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt new file mode 100644 index 0000000000..89a1a49705 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt @@ -0,0 +1,98 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import ftl.args.ArgsHelper.calculateShards +import ftl.args.IosArgs +import ftl.ios.xctest.common.XcTestRunVersion +import ftl.ios.xctest.common.XcTestRunVersion.V1 +import ftl.ios.xctest.common.XcTestRunVersion.V2 +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.common.getXcTestRunVersion +import ftl.ios.xctest.common.mapToRegex +import ftl.ios.xctest.common.parseToNSDictionary +import ftl.shard.Chunk +import ftl.shard.testCases +import ftl.util.FlankTestMethod +import java.io.File + +data class XcTestRunData( + val rootDir: String, + val nsDict: NSDictionary, + val version: XcTestRunVersion = nsDict.getXcTestRunVersion(), + val shardTargets: Map> = emptyMap(), + val shardChunks: Map> = emptyMap() +) + +fun IosArgs.calculateXcTestRunData(): XcTestRunData { + val xcTestRunFile = File(xctestrunFile) + val xcTestRoot: String = xcTestRunFile.parent + "/" + val xcTestNsDictionary: NSDictionary = parseToNSDictionary(xcTestRunFile) + + val calculatedShards: Map, List>> = + if (disableSharding) emptyMap() + else calculateConfigurationShards( + xcTestRoot = xcTestRoot, + xcTestNsDictionary = xcTestNsDictionary, + regexList = testTargets.mapToRegex() + ) + + return XcTestRunData( + rootDir = xcTestRoot, + nsDict = xcTestNsDictionary, + shardChunks = calculatedShards.mapValues { it.value.first }, + shardTargets = calculatedShards.mapValues { it.value.second }, + ) +} + +private fun IosArgs.calculateConfigurationShards( + xcTestRoot: String, + xcTestNsDictionary: NSDictionary, + regexList: List +): Map, List>> = + findXcTestNames( + xcTestRoot = xcTestRoot, + xcTestNsDictionary = xcTestNsDictionary + ).mapValues { (_, targets: Map>) -> + calculateConfigurationShards(targets, regexList) + } + +private fun findXcTestNames( + xcTestRoot: String, + xcTestNsDictionary: NSDictionary +): Map>> = + when (xcTestNsDictionary.getXcTestRunVersion()) { + V1 -> mapOf("" to findXcTestNamesV1(xcTestRoot, xcTestNsDictionary)) + V2 -> findXcTestNamesV2(xcTestRoot, xcTestNsDictionary) + } + +private fun IosArgs.calculateConfigurationShards( + targets: Map>, + regexList: List +): Pair, List> { + val shardChunks = calculateShards( + filteredTests = targets + .values.flatten() + .filterAnyMatches(regexList) + .map(::FlankTestMethod), + args = this + ).shardChunks + + val shardTargets = shardChunks.testCases.map { shardMethods -> + targets.mapValues { (_, methods) -> + methods.intersect(shardMethods).toList() + }.filterValues { + it.isNotEmpty() + } + } + + return shardChunks to shardTargets +} + +private fun List.filterAnyMatches( + list: List +): List = + if (list.isEmpty()) this + else filter { test -> list.any { regex -> regex.matches(test) } } + +fun XcTestRunData.flattenShardChunks(): List = + shardChunks.values.flatten() diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestRunFlow.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestRunFlow.kt new file mode 100644 index 0000000000..b3e5b3f23e --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/XcTestRunFlow.kt @@ -0,0 +1,42 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import ftl.args.IosArgs +import ftl.ios.xctest.common.XcTestRunVersion.V1 +import ftl.ios.xctest.common.XcTestRunVersion.V2 +import ftl.ios.xctest.common.toByteArray +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOf + +fun IosArgs.xcTestRunFlow(): Flow = +// Parameterized tests on iOS don't shard correctly. +// Avoid changing Xctestrun file when disableSharding is on. + if (disableSharding) flowOf(xcTestRunData.nsDict.toByteArray()) + else xcTestRunData.xcTestRunFlow() + +private fun XcTestRunData.xcTestRunFlow( + reduce: ReduceXCTestRun = createReducer() +): Flow = + flow { + shardTargets.forEach { (configName, shards: List>>) -> + shards.forEach { targets: Map> -> + nsDict.clone() + .reduce(configName, targets) + .toByteArray() + .let { emit(it) } + } + } + } + +private typealias ReduceXCTestRun = + NSDictionary.( + config: String, + targets: Map> + ) -> NSDictionary + +private fun XcTestRunData.createReducer(): ReduceXCTestRun = + when (version) { + V2 -> NSDictionary::reduceXcTestRunV2 + V1 -> { _, tests -> reduceXcTestRunV1(tests) } + } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt index b948a5c156..d4524a8f1f 100644 --- a/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt @@ -10,6 +10,8 @@ import java.io.File typealias XctestrunMethods = Map> +enum class XcTestRunVersion { V1, V2 } + internal const val XCTEST_METADATA = "__xctestrun_metadata__" internal const val FORMAT_VERSION = "FormatVersion" internal const val TEST_CONFIGURATIONS = "TestConfigurations" @@ -23,7 +25,13 @@ internal fun String.isMetadata(): Boolean = contentEquals(XCTEST_METADATA) internal fun String.isTestPlan(): Boolean = contentEquals(TEST_PLAN) -internal fun NSDictionary.getXcTestRunVersion(): Int = +internal fun NSDictionary.getXcTestRunVersion(): XcTestRunVersion = + getXcTestRunVersionNumber().let { versionNumber -> + XcTestRunVersion.values().getOrNull(versionNumber - 1) + ?: throw FlankGeneralError("Unsupported xctestrun version $versionNumber") + } + +private fun NSDictionary.getXcTestRunVersionNumber(): Int = (get(XCTEST_METADATA) as? NSDictionary) ?.get(FORMAT_VERSION)?.toJavaObject(Int::class.java) ?: throw FlankGeneralError("Given NSDictionary doesn't contains $FORMAT_VERSION") @@ -57,16 +65,18 @@ internal fun NSDictionary.toByteArray(): ByteArray { internal fun parseToNSDictionary(xctestrunPath: String): NSDictionary = parseToNSDictionary(File(xctestrunPath)) -internal fun parseToNSDictionary(xctestrun: File): NSDictionary = xctestrun.canonicalFile - .apply { if (!exists()) throw FlankGeneralError("$this doesn't exist") } - .let(PropertyListParser::parse) as NSDictionary +internal fun parseToNSDictionary(xctestrun: File): NSDictionary = + xctestrun.canonicalFile + .apply { if (!exists()) throw FlankGeneralError("$this doesn't exist") } + .let(PropertyListParser::parse) as NSDictionary internal fun parseToNSDictionary(xctestrun: ByteArray): NSDictionary = PropertyListParser.parse(xctestrun) as NSDictionary -internal fun parseXcMethodName(matcher: MatchResult): String = matcher.groupValues.last() - .replace('.', '/') - .replace(' ', '/') +internal fun parseXcMethodName(matcher: MatchResult): String = + matcher.groupValues.last() + .replace('.', '/') + .replace(' ', '/') internal fun String.quote() = "\"$this\"" @@ -84,3 +94,11 @@ internal fun NSDictionary.setOnlyTestIdentifiers(methods: Collection) = while (containsKey(ONLY_TEST_IDENTIFIERS)) remove(ONLY_TEST_IDENTIFIERS) this[ONLY_TEST_IDENTIFIERS] = NSArray(methods.size).also { methods.forEachIndexed(it::setValue) } } + +internal fun List.mapToRegex(): List = map { filter -> + try { + filter.toRegex() + } catch (e: Exception) { + throw FlankConfigurationError("Invalid regex: $filter", e) + } +} diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 63f2e8b823..f92a5a5186 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -5,11 +5,12 @@ import ftl.args.IosArgs import ftl.args.isInstrumentationTest import ftl.log.OutputLogLevel import ftl.log.logLn +import ftl.ios.xctest.common.XcTestRunVersion.V1 +import ftl.ios.xctest.common.XcTestRunVersion.V2 import ftl.run.common.prettyPrint import ftl.run.exception.FlankConfigurationError import ftl.run.model.AndroidMatrixTestShards import ftl.run.platform.android.getAndroidMatrixShards -import ftl.shard.testCases import ftl.util.obfuscatePrettyPrinter import java.nio.file.Files import java.nio.file.Paths @@ -34,10 +35,22 @@ fun IosArgs.dumpShards( // VisibleForTesting shardFilePath: String = IOS_SHARD_FILE, ) { + val xcTestRunShards: Map>> = xcTestRunData.shardTargets.mapValues { + it.value.flatMap { it.values } + } + + val rawShards: Any = when (xcTestRunData.version) { + V1 -> xcTestRunShards.values.first() + V2 -> xcTestRunShards + } + + val size = xcTestRunData.shardTargets.values + .flatten().size + saveShardChunks( shardFilePath = shardFilePath, - shards = testShardChunks.testCases, - size = testShardChunks.size, + shards = rawShards, + size = size, obfuscatedOutput = obfuscateDumpShards ) } @@ -55,7 +68,8 @@ fun saveShardChunks( logLn("Saved $size shards to $shardFilePath", OutputLogLevel.DETAILED) } -private fun getGson(obfuscatedOutput: Boolean) = if (obfuscatedOutput) obfuscatePrettyPrinter else prettyPrint +private fun getGson(obfuscatedOutput: Boolean) = + if (obfuscatedOutput) obfuscatePrettyPrinter else prettyPrint const val ANDROID_SHARD_FILE = "android_shards.json" const val IOS_SHARD_FILE = "ios_shards.json" diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index 44beb0952e..4d73012255 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -7,8 +7,9 @@ import ftl.gc.GcIosTestMatrix import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry -import ftl.ios.xctest.common.parseToNSDictionary import ftl.log.logLn +import ftl.ios.xctest.flattenShardChunks +import ftl.ios.xctest.xcTestRunFlow import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.model.TestResult @@ -20,60 +21,63 @@ import ftl.run.platform.common.beforeRunTests import ftl.shard.testCases import ftl.util.ShardCounter import ftl.util.asFileReference +import ftl.util.repeat import ftl.util.uploadIfNeeded -import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList // https://github.com/bootstraponline/gcloud_cli/blob/5bcba57e825fc98e690281cf69484b7ba4eb668a/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/ios/matrix_creator.py#L109 // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run // https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/ -internal suspend fun IosArgs.runIosTests(): TestResult = coroutineScope { - val args = this@runIosTests - val stopwatch = beforeRunTests() +internal suspend fun IosArgs.runIosTests(): TestResult = + coroutineScope { + val args = this@runIosTests + val stopwatch = beforeRunTests() - val iosDeviceList = GcIosMatrix.build(devices) - val xcTestParsed = parseToNSDictionary(xctestrunFile) + val iosDeviceList = GcIosMatrix.build(devices) - val jobs = arrayListOf>() - val runCount = repeatTests - val shardCounter = ShardCounter() - val history = GcToolResults.createToolResultsHistory(args) - val otherGcsFiles = uploadOtherFiles() - val additionalIpasGcsFiles = uploadAdditionalIpas() + val shardCounter = ShardCounter() + val history = GcToolResults.createToolResultsHistory(args) + val otherGcsFiles = uploadOtherFiles() + val additionalIpasGcsFiles = uploadAdditionalIpas() dumpShards() if (disableResultsUpload.not()) GcStorage.upload(IOS_SHARD_FILE, resultsBucket, resultsDir) - // Upload only after parsing shards to detect missing methods early. - val xcTestGcsPath = uploadIfNeeded(xctestrunZip.asFileReference()).gcs + // Upload only after parsing shards to detect missing methods early. + val xcTestGcsPath = uploadIfNeeded(xctestrunZip.asFileReference()).gcs + val testShardChunks = xcTestRunData.flattenShardChunks() - logLn(beforeRunMessage(testShardChunks)) + logLn(beforeRunMessage(testShardChunks)) - repeat(runCount) { - jobs += testShardChunks.map { testTargets -> - async(Dispatchers.IO) { + val result: List = xcTestRunFlow() + .map { xcTestRun -> GcIosTestMatrix.build( iosDeviceList = iosDeviceList, testZipGcsPath = xcTestGcsPath, - runGcsPath = resultsDir, - testTargets = testTargets.testMethodNames, - xcTestParsed = xcTestParsed, + xcTestRun = xcTestRun, args = args, shardCounter = shardCounter, toolResultsHistory = history, otherFiles = otherGcsFiles, additionalIpasGcsPaths = additionalIpasGcsFiles - ).executeWithRetry() + ) } - } - } + .repeat(repeatTests) + .map { async(Dispatchers.IO) { it.executeWithRetry() } } + .toList() + .awaitAll() - TestResult( - matrixMap = afterRunTests(jobs.awaitAll(), stopwatch), - shardChunks = testShardChunks.testCases - ) -} + TestResult( + matrixMap = afterRunTests( + testMatrices = result, + stopwatch = stopwatch + ), + shardChunks = testShardChunks.testCases + ) + } diff --git a/test_runner/src/main/kotlin/ftl/shard/Chunk.kt b/test_runner/src/main/kotlin/ftl/shard/Chunk.kt index b3624c5dd7..2e873c690d 100644 --- a/test_runner/src/main/kotlin/ftl/shard/Chunk.kt +++ b/test_runner/src/main/kotlin/ftl/shard/Chunk.kt @@ -1,9 +1,11 @@ package ftl.shard -data class Chunk(val testMethods: List) { +data class Chunk( + val testMethods: List +) { val testMethodNames = testMethods.map { it.name } val size get() = testMethods.size } -val List.testCases +val List.testCases: List> get() = map { it.testMethodNames } diff --git a/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt b/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt index fe1fdaaf23..cf751c2118 100644 --- a/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt +++ b/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt @@ -1,3 +1,7 @@ package ftl.util -data class FlankTestMethod(val testName: String, val ignored: Boolean = false, val isParameterizedClass: Boolean = false) +data class FlankTestMethod( + val testName: String, + val ignored: Boolean = false, + val isParameterizedClass: Boolean = false +) diff --git a/test_runner/src/main/kotlin/ftl/util/FlowExt.kt b/test_runner/src/main/kotlin/ftl/util/FlowExt.kt new file mode 100644 index 0000000000..2569c764a6 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/util/FlowExt.kt @@ -0,0 +1,11 @@ +@file:Suppress("EXPERIMENTAL_API_USAGE") + +package ftl.util + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.flatMapConcat +import kotlinx.coroutines.flow.map + +fun Flow.repeat(times: Int): Flow = + flatMapConcat { next -> (0 until times).asFlow().map { next } } diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index 083b831e27..dbe76c43f3 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -3,8 +3,10 @@ import ftl.Main import ftl.run.exception.withGlobalExceptionHandling import picocli.CommandLine +import java.io.File fun main() { + println(File("").absolutePath) // GoogleApiLogger.logAllToStdout() // for debugging. Run test from IntelliJ IDEA @@ -13,10 +15,6 @@ fun main() { val projectId = System.getenv("GOOGLE_CLOUD_PROJECT") ?: "YOUR PROJECT ID" - val quantity = "single" - val type = "success" - val extra = "ios" - // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 withGlobalExceptionHandling { @@ -24,14 +22,14 @@ fun main() { // "--debug", "firebase", "test", - "android", + "ios", "run", // "--dry", // "--dump-shards", "--output-style=single", // "--full-junit-result", // "--legacy-junit-result", - "-c=test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type-$extra.yml", + "-c=./src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml", "--project=$projectId" // "--client-details=key1=value1,key2=value2" ) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt index 0feafdf7a3..342b5cc33a 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt @@ -3,6 +3,7 @@ package ftl.args import com.google.common.truth.Truth.assertThat import ftl.config.Device import ftl.config.FtlConstants +import ftl.ios.xctest.flattenShardChunks import ftl.run.exception.FlankGeneralError import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner @@ -88,7 +89,7 @@ class IosArgsFileTest { "EarlGreyExampleSwiftTests/testWithCustomAssertion" ) - val testShardChunks = config.testShardChunks + val testShardChunks = config.xcTestRunData.flattenShardChunks() assertThat(testShardChunks.size).isEqualTo(2) assertThat(testShardChunks[0].testMethodNames).isEqualTo(chunk0) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 0797a7fdfd..43fba2c710 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -8,6 +8,7 @@ import ftl.config.FtlConstants.defaultIosModel import ftl.config.FtlConstants.defaultIosVersion import ftl.config.defaultIosConfig import ftl.gc.GcStorage +import ftl.ios.xctest.flattenShardChunks import ftl.run.exception.FlankConfigurationError import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner @@ -397,6 +398,8 @@ IosArgs ) with(args) { + val testShardChunks = xcTestRunData.flattenShardChunks() + assert(maxTestShards, IArgs.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE.last) assert(testShardChunks.size, 17) testShardChunks.forEach { chunk -> assert(chunk.size, 1) } @@ -430,7 +433,7 @@ IosArgs flank: disable-sharding: true """ - IosArgs.load(yaml).testShardChunks + IosArgs.load(yaml) } @Test(expected = RuntimeException::class) @@ -442,7 +445,7 @@ IosArgs flank: disable-sharding: false """ - IosArgs.load(yaml).testShardChunks + IosArgs.load(yaml).xcTestRunData } // gcloudYml diff --git a/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml index 84f920b04e..b4aa98ab1b 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml @@ -1,3 +1,7 @@ gcloud: test: "./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip" - xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/ios/flank_ios_example/FlankExampleTests.xctestrun" + xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/ios/multi_test_targets/AllTests_AllTests_iphoneos13.7-arm64e.xctestrun" + +flank: + disable-sharding: false + max-test-shards: 3 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml new file mode 100644 index 0000000000..aacbe11b32 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml @@ -0,0 +1,6 @@ +gcloud: + test: "./src/test/kotlin/ftl/fixtures/tmp/ios/FlankTestPlansExample/FlankTestPlansExample.zip" + xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/ios/FlankTestPlansExample/AllTests.xctestrun" +flank: + max-test-shards: 1 + repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt index 3af31000f0..ab28decb94 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt @@ -1,12 +1,9 @@ package ftl.gc -import com.dd.plist.NSDictionary import com.google.testing.model.IosDeviceList import ftl.args.IosArgs import ftl.config.FtlConstants.isWindows import ftl.ios.xctest.FIXTURES_PATH -import ftl.shard.Chunk -import ftl.shard.TestMethod import ftl.test.util.FlankTestRunner import ftl.util.ShardCounter import io.mockk.every @@ -29,15 +26,11 @@ class GcIosTestMatrixTest { fun `build negativeShardErrors`() { val iosArgs = mockk(relaxed = true) - every { iosArgs.testShardChunks } returns listOf(Chunk(listOf(TestMethod(name = "", time = 0.0)))) - GcIosTestMatrix.build( iosDeviceList = IosDeviceList(), testZipGcsPath = "", - runGcsPath = "", - xcTestParsed = NSDictionary(), args = iosArgs, - testTargets = emptyList(), + xcTestRun = ByteArray(0), shardCounter = ShardCounter(), toolResultsHistory = GcToolResults.createToolResultsHistory(iosArgs), otherFiles = mapOf(), @@ -51,10 +44,8 @@ class GcIosTestMatrixTest { GcIosTestMatrix.build( iosDeviceList = IosDeviceList(), testZipGcsPath = "", - runGcsPath = "", - xcTestParsed = NSDictionary(), args = iosArgs, - testTargets = listOf(""), + xcTestRun = ByteArray(0), shardCounter = ShardCounter(), toolResultsHistory = GcToolResults.createToolResultsHistory(iosArgs), otherFiles = mapOf(), @@ -67,7 +58,6 @@ class GcIosTestMatrixTest { assumeFalse(isWindows) // TODO enable it on #1180 val iosArgs = mockk(relaxed = true) - every { iosArgs.testShardChunks } returns listOf(Chunk(listOf(TestMethod(name = "", time = 0.0)))) every { iosArgs.testTimeout } returns "3m" every { iosArgs.resultsBucket } returns "/hi" every { iosArgs.project } returns "123" @@ -76,10 +66,8 @@ class GcIosTestMatrixTest { GcIosTestMatrix.build( iosDeviceList = IosDeviceList(), testZipGcsPath = "", - runGcsPath = "", - xcTestParsed = NSDictionary(), args = iosArgs, - testTargets = emptyList(), + xcTestRun = ByteArray(0), shardCounter = ShardCounter(), toolResultsHistory = GcToolResults.createToolResultsHistory(iosArgs), otherFiles = mapOf(), diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt index d48c824962..ef50bf75cf 100644 --- a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt @@ -2,8 +2,10 @@ package ftl.ios.xctest import com.google.common.truth.Truth.assertThat import ftl.config.FtlConstants +import ftl.ios.xctest.common.parseToNSDictionary import org.junit.Assume.assumeFalse import org.junit.Test +import java.io.File class FindXcTestNamesV1KtTest { @@ -11,8 +13,14 @@ class FindXcTestNamesV1KtTest { fun findTestNames() { assumeFalse(FtlConstants.isWindows) + // given + val xctestrun = File(swiftXcTestRunV1) + // when - val names = findXcTestNamesV1(swiftXcTestRunV1) + val names = findXcTestNamesV1( + xcTestRoot = xctestrun.parent + "/", + xcTestNsDictionary = parseToNSDictionary(xctestrun) + ) .flatMap { it.value } .sorted() diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt index 4b203c6852..759a3b97a6 100644 --- a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt @@ -2,8 +2,10 @@ package ftl.ios.xctest import com.google.common.truth.Truth.assertThat import ftl.config.FtlConstants +import ftl.ios.xctest.common.parseToNSDictionary import org.junit.Assume.assumeFalse import org.junit.Test +import java.io.File class FindXcTestNamesV2KtTest { @@ -11,8 +13,14 @@ class FindXcTestNamesV2KtTest { fun findTestNames() { assumeFalse(FtlConstants.isWindows) + // given + val xctestrun = File(swiftXcTestRunV2) + // when - val names = findXcTestNamesV2(swiftXcTestRunV2) + val names = findXcTestNamesV2( + xcTestRoot = xctestrun.parent + "/", + xcTestNsDictionary = parseToNSDictionary(xctestrun) + ) // then val sortedNames = names.mapValues { it.value.mapValues { entry -> entry.value.sorted() } } diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV1KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV1KtTest.kt new file mode 100644 index 0000000000..130d23a684 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV1KtTest.kt @@ -0,0 +1,199 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.google.common.truth.Truth.assertThat +import ftl.args.IosArgs +import ftl.config.FtlConstants +import ftl.ios.xctest.common.parseToNSDictionary +import ftl.ios.xctest.common.toByteArray +import ftl.test.util.TestHelper.normalizeLineEnding +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.Assume.assumeFalse +import org.junit.Test + +class ReduceXcTestRunV1KtTest { + + @Test + fun rewrite() { + assumeFalse(FtlConstants.isWindows) + + val results = runBlocking { + IosArgs.default().copy( + xctestrunFile = swiftXcTestRunV1, + testTargets = emptyList() + ).xcTestRunFlow() + .toList() + .first() + .let { String(it) } + } + + assertThat(results.contains("OnlyTestIdentifiers")).isTrue() + } + + @Test + fun rewriteImmutable() { + assumeFalse(FtlConstants.isWindows) + + val args = IosArgs.default().copy( + xctestrunFile = swiftXcTestRunV1, + testTargets = emptyList() + ) + + val expected = args.xcTestRunData.nsDict.toASCIIPropertyList() + + runBlocking { args.xcTestRunFlow().toList() } + + val actual = args.xcTestRunData.nsDict.toASCIIPropertyList() + + assertThat(actual).isEqualTo(expected) + } + + @Test + fun `rewrite preserves skip`() { + val inputXml = """ + + + + + EarlGreyExampleSwiftTests + + SkipTestIdentifiers + + testTwo + + OnlyTestIdentifiers + + testOne + testTwo + + + + + """.trimIndent() + val root = parseToNSDictionary(inputXml.toByteArray()) + + val tests = + mapOf("EarlGreyExampleSwiftTests" to listOf("testOne", "testTwo")) + val rewrittenXml = root.reduceXcTestRunV1(tests) + .toByteArray().let(::String) + + assertThat(inputXml).isEqualTo(rewrittenXml.normalizeLineEnding()) + } + + + @Test + fun `rewrite methods in single test target`() { + assumeFalse(FtlConstants.isWindows) + val methods = listOf( + "FlankExampleTests/test1", + "FlankExampleTests/test2" + ) + val result = runBlocking { + IosArgs.default().copy( + xctestrunFile = multiTargetsSwiftXcTestRunV1, + testTargets = methods + ).xcTestRunFlow().toList().first().also { + println(String(it)) + } + } + val resultXML = parseToNSDictionary(result) + val testTarget = resultXML["FlankExampleTests"] as NSDictionary + val resultMethods = testTarget["OnlyTestIdentifiers"] as NSArray + + assertThat( + resultMethods.array.map { it.toJavaObject() }.toSet() + ).isEqualTo( + methods.toSet() + ) + + assertThat( + resultXML["FlankExampleSecondTests"] + ).isNull() + } + + @Test + fun `rewrite methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val expectedMethods1 = + listOf("FlankExampleTests/test1", "FlankExampleTests/test2") + val expectedMethods2 = listOf("FlankExampleSecondTests/test3") + + val result = runBlocking { + IosArgs.default().copy( + xctestrunFile = multiTargetsSwiftXcTestRunV1, + testTargets = listOf( + expectedMethods1, + expectedMethods2 + ).flatten() + ).xcTestRunFlow().toList().first() + } + val resultXML = parseToNSDictionary(result) + + val target1 = resultXML["FlankExampleTests"] as NSDictionary + val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary + val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray + val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray + + assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() } + .toSet()) + assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() } + .toSet()) + } + + @Test + fun `rewrite incorrect methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val methods = listOf( + "incorrect1", + "incorrect2", + "incorrect3" + ) + + val result = runBlocking { + IosArgs.default().copy( + xctestrunFile = multiTargetsSwiftXcTestRunV1, + testTargets = methods + ).xcTestRunFlow().toList() + } + + assertThat(result).isEmpty() + } + + @Test + fun `rewrite mix of correct and incorrect methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val methods1 = listOf( + "FlankExampleTests/test1", + "FlankExampleTests/test2", + "incorrect1" + ) + val methods2 = listOf("FlankExampleSecondTests/test3", "incorrect2") + + val expectedMethods1 = + listOf("FlankExampleTests/test1", "FlankExampleTests/test2") + val expectedMethods2 = listOf("FlankExampleSecondTests/test3") + + val result = runBlocking { + IosArgs.default().copy( + xctestrunFile = multiTargetsSwiftXcTestRunV1, + testTargets = listOf(methods1, methods2).flatten() + ).xcTestRunFlow().toList().first() + } + val resultXML = parseToNSDictionary(result) + + val target1 = resultXML["FlankExampleTests"] as NSDictionary + val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary + val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray + val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray + + assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() } + .toSet()) + assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() } + .toSet()) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV2KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV2KtTest.kt new file mode 100644 index 0000000000..6e435e44d4 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/ReduceXcTestRunV2KtTest.kt @@ -0,0 +1,283 @@ +package ftl.ios.xctest + +import com.google.common.truth.Truth.assertThat +import ftl.args.IosArgs +import ftl.config.FtlConstants +import ftl.ios.xctest.common.ONLY_TEST_IDENTIFIERS +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assume.assumeFalse +import org.junit.Test + +class ReduceXcTestRunV2KtTest { + + @Test + fun rewrite() { + assumeFalse(FtlConstants.isWindows) + // given + val uiTestsMethods = listOf("UITestsClass/test1_1") + val expected = listOf( + """ + + + + + TestConfigurations + + + Name + en + TestTargets + + + BlueprintName + UITests + BundleIdentifiersForCrashReportEmphasis + + io.gogoapps.FlankMultiTestTargetsExample + io.gogoapps.SecondUITests + io.gogoapps.UITests + + CommandLineArguments + + -AppleLanguages + (en) + -AppleTextDirection + NO + -AppleLocale + en_GB + + DefaultTestExecutionTimeAllowance + 600 + DependentProductPaths + + __TESTROOT__/Debug-iphoneos/UITests-Runner.app/PlugIns/UITests.xctest + __TESTROOT__/Debug-iphoneos/SecondUITests-Runner.app/PlugIns/SecondUITests.xctest + __TESTROOT__/Debug-iphoneos/FlankTestPlansExample.app + + EnvironmentVariables + + OS_ACTIVITY_DT_MODE + YES + SQLITE_ENABLE_THREAD_ASSERTIONS + 1 + + IsUITestBundle + + IsXCTRunnerHostedTestBundle + + ProductModuleName + UITests + SystemAttachmentLifetime + deleteOnSuccess + TestBundlePath + __TESTHOST__/PlugIns/UITests.xctest + TestHostBundleIdentifier + io.gogoapps.UITests.xctrunner + TestHostPath + __TESTROOT__/Debug-iphoneos/UITests-Runner.app + TestLanguage + en + TestRegion + GB + TestTimeoutsEnabled + + TestingEnvironmentVariables + + + ToolchainsSettingValue + + + UITargetAppCommandLineArguments + + -AppleLanguages + (en) + -AppleTextDirection + NO + -AppleLocale + en_GB + + UITargetAppEnvironmentVariables + + + UITargetAppPath + __TESTROOT__/Debug-iphoneos/FlankTestPlansExample.app + UserAttachmentLifetime + deleteOnSuccess + OnlyTestIdentifiers + + UITestsClass/test1_1 + + + + + + TestPlan + + IsDefault + + Name + AllTests + + __xctestrun_metadata__ + + FormatVersion + 2 + + + + """.trimIndent(), + """ + + + + + TestConfigurations + + + Name + pl + TestTargets + + + BlueprintName + UITests + BundleIdentifiersForCrashReportEmphasis + + io.gogoapps.FlankMultiTestTargetsExample + io.gogoapps.SecondUITests + io.gogoapps.UITests + + CommandLineArguments + + -AppleLanguages + (pl) + -AppleTextDirection + NO + -AppleLocale + pl_PL + + DefaultTestExecutionTimeAllowance + 600 + DependentProductPaths + + __TESTROOT__/Debug-iphoneos/UITests-Runner.app/PlugIns/UITests.xctest + __TESTROOT__/Debug-iphoneos/SecondUITests-Runner.app/PlugIns/SecondUITests.xctest + __TESTROOT__/Debug-iphoneos/FlankTestPlansExample.app + + EnvironmentVariables + + OS_ACTIVITY_DT_MODE + YES + SQLITE_ENABLE_THREAD_ASSERTIONS + 1 + + IsUITestBundle + + IsXCTRunnerHostedTestBundle + + ProductModuleName + UITests + SystemAttachmentLifetime + deleteOnSuccess + TestBundlePath + __TESTHOST__/PlugIns/UITests.xctest + TestHostBundleIdentifier + io.gogoapps.UITests.xctrunner + TestHostPath + __TESTROOT__/Debug-iphoneos/UITests-Runner.app + TestLanguage + pl + TestRegion + PL + TestTimeoutsEnabled + + TestingEnvironmentVariables + + + ToolchainsSettingValue + + + UITargetAppCommandLineArguments + + -AppleLanguages + (pl) + -AppleTextDirection + NO + -AppleLocale + pl_PL + + UITargetAppEnvironmentVariables + + + UITargetAppPath + __TESTROOT__/Debug-iphoneos/FlankTestPlansExample.app + UserAttachmentLifetime + deleteOnSuccess + OnlyTestIdentifiers + + UITestsClass/test1_1 + + + + + + TestPlan + + IsDefault + + Name + AllTests + + __xctestrun_metadata__ + + FormatVersion + 2 + + + + """.trimIndent() + ) + + // when + val result: List = runBlocking { + IosArgs.default() + .copy( + xctestrunFile = swiftXcTestRunV2, + testTargets = uiTestsMethods + ) + .xcTestRunFlow() + .map { String(it) } + .toList() + } + + // then + assertEquals(expected, result) + } + + @Test + fun `ensure rewrite not mutates root`() { + assumeFalse(FtlConstants.isWindows) + + // given + val args = IosArgs.default() + .copy( + xctestrunFile = swiftXcTestRunV2, + testTargets = emptyList() + ) + + assertThat( + args.xcTestRunData.nsDict.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS) + ).isFalse() + + // when + runBlocking { args.xcTestRunFlow().toList() } + + // then + assertThat( + args.xcTestRunData.nsDict.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS) + ).isFalse() + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt deleted file mode 100644 index 721944b2cd..0000000000 --- a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt +++ /dev/null @@ -1,183 +0,0 @@ -package ftl.ios.xctest - -import com.dd.plist.NSArray -import com.dd.plist.NSDictionary -import com.google.common.truth.Truth.assertThat -import ftl.config.FtlConstants -import ftl.ios.xctest.common.findTestsForTestTarget -import ftl.ios.xctest.common.parseToNSDictionary -import ftl.test.util.TestHelper.normalizeLineEnding -import org.junit.Assume.assumeFalse -import org.junit.Test -import java.io.File - -class RewriteXcTestRunV1KtTest { - - @Test - fun rewrite() { - assumeFalse(FtlConstants.isWindows) - - val root = parseToNSDictionary(swiftXcTestRunV1) - val methods = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)) - val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods) - - val results = String(rewriteXcTestRunV1(root, methodsData)) - - assertThat(results.contains("OnlyTestIdentifiers")).isTrue() - } - - @Test - fun rewriteImmutable() { - assumeFalse(FtlConstants.isWindows) - - val root = parseToNSDictionary(swiftXcTestRunV1) - val methods = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)) - val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods) - - // ensure root object isn't modified. Rewrite should return a new object. - val key = "OnlyTestIdentifiers" - - assertThat(root.toASCIIPropertyList().contains(key)).isFalse() - - rewriteXcTestRunV1(root, methodsData) - - assertThat(root.toASCIIPropertyList().contains(key)).isFalse() - } - - @Test - fun `rewrite idempotent`() { - val root = NSDictionary() - root["EarlGreyExampleSwiftTests"] = NSDictionary() - root["EarlGreyExampleTests"] = NSDictionary() - val methods = listOf("testOne", "testTwo") - val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods, "EarlGreyExampleTests" to methods) - rewriteXcTestRunV1(root, methodsData) - rewriteXcTestRunV1(root, methodsData) - val result = rewriteXcTestRunV1(root, methodsData) - val expected = """ - - - - EarlGreyExampleSwiftTests - - OnlyTestIdentifiers - - testOne - testTwo - - - EarlGreyExampleTests - - OnlyTestIdentifiers - - testOne - testTwo - - - -""" - assertThat(expected).isEqualTo(String(result).normalizeLineEnding()) - } - - @Test - fun `rewrite preserves skip`() { - val inputXml = """ - - - - - EarlGreyExampleSwiftTests - - SkipTestIdentifiers - - testTwo - - OnlyTestIdentifiers - - testOne - testTwo - - - - - """.trimIndent() - val root = parseToNSDictionary(inputXml.toByteArray()) - - val tests = mapOf("EarlGreyExampleSwiftTests" to listOf("testOne", "testTwo")) - val rewrittenXml = String(rewriteXcTestRunV1(root, tests)) - - assertThat(inputXml).isEqualTo(rewrittenXml.normalizeLineEnding()) - } - - @Test - fun `rewrite methods in single test target`() { - assumeFalse(FtlConstants.isWindows) - val methods = listOf("EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", "EarlGreyExampleSwiftTests/testBasicSelection") - val result = rewriteXcTestRunV1(xctestrun = swiftXcTestRunV1, methods) - val resultXML = parseToNSDictionary(result) - val testTarget = resultXML["EarlGreyExampleSwiftTests"] as NSDictionary - val resultMethods = testTarget["OnlyTestIdentifiers"] as NSArray - - assertThat(methods.toSet()).isEqualTo(resultMethods.array.map { it.toJavaObject() }.toSet()) - } - - @Test - fun `rewrite methods in multiple test targets`() { - assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 - - val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") - val expectedMethods2 = listOf("FlankExampleSecondTests/test3") - - val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(expectedMethods1, expectedMethods2).flatMap { it }) - val resultXML = parseToNSDictionary(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) - assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) - } - - @Test - fun `rewrite incorrect methods in multiple test targets`() { - assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 - - val methods1 = listOf("incorrect1", "incorrect2") - val methods2 = listOf("incorrect3") - - val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(methods1, methods2).flatMap { it }) - val resultXML = parseToNSDictionary(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(resultMethods1.array.isEmpty()).isTrue() - assertThat(resultMethods2.array.isEmpty()).isTrue() - } - - @Test - fun `rewrite mix of correct and incorrect methods in multiple test targets`() { - assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 - - val methods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2", "incorrect1") - val methods2 = listOf("FlankExampleSecondTests/test3", "incorrect2") - - val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") - val expectedMethods2 = listOf("FlankExampleSecondTests/test3") - - val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(methods1, methods2).flatMap { it }) - val resultXML = parseToNSDictionary(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) - assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) - } -} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt deleted file mode 100644 index 73bf4da96f..0000000000 --- a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt +++ /dev/null @@ -1,60 +0,0 @@ -package ftl.ios.xctest - -import com.google.common.truth.Truth.assertThat -import ftl.config.FtlConstants -import ftl.ios.xctest.common.ONLY_TEST_IDENTIFIERS -import ftl.ios.xctest.common.getOnlyTestIdentifiers -import ftl.ios.xctest.common.getTestConfigurations -import ftl.ios.xctest.common.getTestTargets -import ftl.ios.xctest.common.parseToNSDictionary -import org.junit.Assume.assumeFalse -import org.junit.Test - -class RewriteXcTestRunV2KtTest { - - @Test - fun rewrite() { - assumeFalse(FtlConstants.isWindows) - // given - val uiTestsMethods = listOf("UITestsClass/test1_1") - val secondUiTestsMethods = emptyList() - val expectedTargetNames = listOf("en", "pl") - - // when - val result = rewriteXcTestRunV2( - swiftXcTestRunV2, - uiTestsMethods - ).mapValues { (_, bytes) -> - parseToNSDictionary(bytes) - } - - // then - assertThat(result.keys.toList().sorted()).isEqualTo(expectedTargetNames) - expectedTargetNames.forEach { targetName -> - result.getValue(targetName).getTestConfigurations().values.let { configurations -> - assertThat(configurations.size).isEqualTo(1) - configurations.first().getTestTargets().let { targets -> - assertThat(targets.size).isEqualTo(2) - assertThat(targets[0].getOnlyTestIdentifiers()).isEqualTo(secondUiTestsMethods) - assertThat(targets[1].getOnlyTestIdentifiers()).isEqualTo(uiTestsMethods) - } - } - } - } - - @Test - fun `ensure rewrite not mutates root`() { - assumeFalse(FtlConstants.isWindows) - - // given - val root = parseToNSDictionary(swiftXcTestRunV2) - val methods = findXcTestNamesV2(swiftXcTestRunV2) - assertThat(root.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS)).isFalse() - - // when - rewriteXcTestRunV2(root, methods) - - // then - assertThat(root.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS)).isFalse() - } -} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt index a7f95fd300..65c924d77d 100644 --- a/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt @@ -56,7 +56,7 @@ val swiftTestsV2 = mapOf( "UITestsClass/test1_2", "UITestsClass/test1_3", "UITestsClass/test1_ENLocale", - "UITestsClass/test1_PLLocale", + "UITestsClass/test1_PLLocale" ), "SecondUITests" to listOf( "SecondUITestsClass/test2_1", @@ -72,7 +72,7 @@ val swiftTestsV2 = mapOf( "UITestsClass/test1_2", "UITestsClass/test1_3", "UITestsClass/test1_ENLocale", - "UITestsClass/test1_PLLocale", + "UITestsClass/test1_PLLocale" ), "SecondUITests" to listOf( "SecondUITestsClass/test2_1", diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index dff39bf7a2..a744f3922d 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -19,6 +19,7 @@ import org.junit.Test import org.junit.contrib.java.lang.system.SystemOutRule import org.junit.runner.RunWith import java.io.File +import org.junit.Assume.assumeFalse @RunWith(FlankTestRunner::class) class DumpShardsKtTest { @@ -183,24 +184,25 @@ class DumpShardsKtTest { @Test fun `dump shards obfuscated ios`() { + assumeFalse(FtlConstants.isWindows) // TODO Windows Linux subsytem does not contain all expected commands + // given - val notExpected = """ + val expected = """ [ [ - "EarlGreyExampleSwiftTests/testWithGreyAssertions", - "EarlGreyExampleSwiftTests/testWithInRoot", - "EarlGreyExampleSwiftTests/testWithCondition", - "EarlGreyExampleSwiftTests/testWithCustomFailureHandler" + "A/a", + "A/b", + "A/c", + "A/d" ], [ - "EarlGreyExampleSwiftTests/testWithGreyAssertions", - "EarlGreyExampleSwiftTests/testWithCustomMatcher", - "EarlGreyExampleSwiftTests/testWithCustomAssertion" + "A/a", + "A/e", + "A/f" ] ] """.trimIndent() - if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { IosArgs.load( @@ -211,10 +213,7 @@ class DumpShardsKtTest { } // then - assertNotEquals(notExpected, actual) - assertThat(notExpected.split(System.lineSeparator()).size).isEqualTo( - actual.split(System.lineSeparator()).size - ) // same line count + assertEquals(expected, actual) assertThat(output.log).contains("Saved 2 shards to $TEST_SHARD_FILE") } }