From 501ea710b7c949a294f4a35e75ccfa619655e90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Thu, 19 Nov 2020 14:36:01 +0100 Subject: [PATCH] refactor: Prepare flank codebase for supporting iOS testplans (#1281) Co-authored-by: Axel Zuziak Co-authored-by: Axel Zuziak <31246956+axelzuziak-gogo@users.noreply.github.com> --- docs/feature/ios_test_plans.md | 187 +++++++++++ .../src/main/kotlin/ftl/args/IosArgs.kt | 6 +- .../src/main/kotlin/ftl/gc/GcIosTestMatrix.kt | 12 +- test_runner/src/main/kotlin/ftl/ios/Parse.kt | 92 ------ .../src/main/kotlin/ftl/ios/Xctestrun.kt | 133 -------- .../ftl/ios/xctest/FindXcTestNamesV1.kt | 23 ++ .../ftl/ios/xctest/FindXcTestNamesV2.kt | 27 ++ .../ftl/ios/xctest/RewriteXcTestRunV1.kt | 36 +++ .../ftl/ios/xctest/RewriteXcTestRunV2.kt | 52 +++ .../ios/xctest/common/FindTestsForTarget.kt | 50 +++ .../ios/xctest/common/InstallParseBinaries.kt | 13 + .../ftl/ios/xctest/common/ParseObjTests.kt | 27 ++ .../ftl/ios/xctest/common/ParseSwiftTests.kt | 35 ++ .../main/kotlin/ftl/ios/xctest/common/Util.kt | 86 +++++ .../kotlin/ftl/run/platform/RunIosTests.kt | 4 +- .../kotlin/ftl/fixtures/ios_test_plan.yml | 3 + .../test/kotlin/ftl/gc/GcIosTestMatrixTest.kt | 2 +- .../src/test/kotlin/ftl/ios/Constants.kt | 3 - .../src/test/kotlin/ftl/ios/ParseTest.kt | 112 ------- .../src/test/kotlin/ftl/ios/XctestrunTest.kt | 305 ------------------ .../ftl/ios/xctest/FindXcTestNamesV1KtTest.kt | 24 ++ .../ftl/ios/xctest/FindXcTestNamesV2KtTest.kt | 21 ++ .../ios/xctest/RewriteXcTestRunV1KtTest.kt | 184 +++++++++++ .../ios/xctest/RewriteXcTestRunV2KtTest.kt | 60 ++++ .../src/test/kotlin/ftl/ios/xctest/Util.kt | 99 ++++++ .../xctest/common/FindTestsForTargetKtTest.kt | 88 +++++ .../ios/xctest/common/ParseObjTestsKtTest.kt | 33 ++ .../xctest/common/ParseSwiftTestsKtTest.kt | 38 +++ .../ftl/ios/xctest/common/UtilKtTest.kt | 23 ++ 29 files changed, 1122 insertions(+), 656 deletions(-) create mode 100644 docs/feature/ios_test_plans.md delete mode 100644 test_runner/src/main/kotlin/ftl/ios/Parse.kt delete mode 100644 test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/common/FindTestsForTarget.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/common/InstallParseBinaries.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseObjTests.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseSwiftTests.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml delete mode 100644 test_runner/src/test/kotlin/ftl/ios/Constants.kt delete mode 100644 test_runner/src/test/kotlin/ftl/ios/ParseTest.kt delete mode 100644 test_runner/src/test/kotlin/ftl/ios/XctestrunTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/common/FindTestsForTargetKtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseObjTestsKtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseSwiftTestsKtTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/ios/xctest/common/UtilKtTest.kt diff --git a/docs/feature/ios_test_plans.md b/docs/feature/ios_test_plans.md new file mode 100644 index 0000000000..35947345a3 --- /dev/null +++ b/docs/feature/ios_test_plans.md @@ -0,0 +1,187 @@ +# Flow + +Flow starts by parsing .xctestrun file. +Search for: `__xctestrun_metadata__` key. + +```xml +__xctestrun_metadata__ + + FormatVersion + 1 + +``` + +- **FormatVersion: `1` -** old version of .xctestrun +- **FormatVersion: `2` -** the newest version with test plans +- If format is different than 1 or 2 throw an error. + +--- + +### FormatVersion: 1 + +Any other key than metadata should have corresponding **TestTarget** dictionary. In example below `EarlGreyExampleSwiftTests` has a **TestTarget** dictionary. + +```xml + + + EarlGreyExampleSwiftTests + + BlueprintName + EarlGreyExampleSwiftTests + ... + + __xctestrun_metadata__ + + FormatVersion + 1 + + + +``` + +### FormatVersion: 2 + +In this version, XML contains two keys: `TestConfigurations` and `TestPlan` in addition to `__xctestrun_metadata__`. + +`TestPlan` is just a dictionary containing basic informations about current **TestPlan.** We can ignore it. So it's excluded from example xml below. + +`TestConfigurations` is an array of different test configurations. Test configuration contains name property and array of TestTargets. + +```xml + + + Name + pl + TestTargets + + + BlueprintName + UITests + + + + BlueprintName + SecondUITests + + + + + +``` + +Each configuration may contain different Environment Variables, languages, regions or any other properties. Those properties are stored under TestTarget. + +Currently **FTL** doesn't support specifying TestConfiguration for test execution. + +If there is more than one configuration FTL will probably choose one arbitrarily. + +For now Flank will allow specifying which test configuration should run with `only-test-configuration` argument. + +--- + +# Running test plan locally + +## Build Xcode project + +To build example project run command below. + +```bash +xcodebuild build-for-testing \ +-allowProvisioningUpdates \ +-project "FlankMultiTestTargetsExample.xcodeproj" \ +-scheme "AllTests" \ #Scheme should have test plans enabled +-derivedDataPath "build_testplan_device" \ +-sdk iphoneos | xcpretty +``` + +This command will generate directory: **Debug-iphoneos** containing binaries and .xctestrun file for each TestPlan. + +In this example scheme `AllTests` has have only one test plan: **AllTests** with two test configurations: `pl` and `en`. + +**Test Plan** contains two **Test Targets: `UITests` and `SecondUITests`** +Outputted .xctestrun should looks like this: + +```xml + + + TestConfigurations + + + Name + en + TestTargets + + + BlueprintName + UITests + TestLanguage + en + TestRegion + GB + + + + BlueprintName + SecondUITests + TestLanguage + en + TestRegion + GB + + + + + + Name + pl + TestTargets + + + BlueprintName + UITests + TestLanguage + pl + TestRegion + PL + + + + BlueprintName + SecondUITests + TestLanguage + pl + TestRegion + PL + + + + + + TestPlan + + IsDefault + + Name + AllTests + + __xctestrun_metadata__ + + FormatVersion + 2 + + + +``` + +## Running tests on a local device + +After generating binaries and .xctestrun file we can run tests using command. + +```bash +xcodebuild test-without-building \ +-xctestrun "build_testplan_device/Build/Products/testrun.xctestrun" \ +-destination "platform=iOS,id=00008030-000209DC1A50802E" \ +-only-test-configuration pl | xcpretty +``` + +Option: `-only-test-configuration pl` allows to specify which test configuration should Xcode run. diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index f5141f42ae..0a27c2c2fb 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -1,8 +1,8 @@ package ftl.args import com.google.common.annotations.VisibleForTesting -import ftl.ios.Xctestrun.findTestNames -import ftl.ios.XctestrunMethods +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.findXcTestNamesV1 import ftl.run.exception.FlankConfigurationError import ftl.shard.Chunk import ftl.util.FlankTestMethod @@ -79,7 +79,7 @@ IosArgs private fun IosArgs.calculateShardChunks() = if (disableSharding) emptyList() else ArgsHelper.calculateShards( - filteredTests = filterTests(findTestNames(xctestrunFile), testTargets) + filteredTests = filterTests(findXcTestNamesV1(xctestrunFile), testTargets) .flatMap { it.value } .distinct() .map { FlankTestMethod(it, ignored = false) }, diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index 5bafc39e60..b17e3139f4 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -17,8 +17,8 @@ import ftl.args.IosArgs import ftl.gc.android.mapGcsPathsToFileReference import ftl.gc.android.mapToIosDeviceFiles import ftl.gc.android.toIosDeviceFile -import ftl.ios.Xctestrun -import ftl.ios.Xctestrun.toByteArray +import ftl.ios.xctest.rewriteXcTestRunV1 +import ftl.ios.xctest.common.toByteArray import ftl.run.exception.FlankGeneralError import ftl.util.ShardCounter import ftl.util.join @@ -53,13 +53,15 @@ object GcIosTestMatrix { val generatedXctestrun = if (args.disableSharding) { xcTestParsed.toByteArray() } else { - Xctestrun.rewrite(args.xctestrunFile, testTargets) + rewriteXcTestRunV1(args.xctestrunFile, testTargets) } // Add shard number to file name - val xctestrunNewFileName = StringBuilder(args.xctestrunFile).insert(args.xctestrunFile.lastIndexOf("."), "_$shardName").toString() + val xctestrunNewFileName = + StringBuilder(args.xctestrunFile).insert(args.xctestrunFile.lastIndexOf("."), "_$shardName").toString() - val xctestrunFileGcsPath = GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, generatedXctestrun) + val xctestrunFileGcsPath = + GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, generatedXctestrun) val iOSXCTest = IosXcTest() .setTestsZip(FileReference().setGcsPath(testZipGcsPath)) diff --git a/test_runner/src/main/kotlin/ftl/ios/Parse.kt b/test_runner/src/main/kotlin/ftl/ios/Parse.kt deleted file mode 100644 index 7b1a16c6d9..0000000000 --- a/test_runner/src/main/kotlin/ftl/ios/Parse.kt +++ /dev/null @@ -1,92 +0,0 @@ -package ftl.ios - -import ftl.config.FtlConstants.isMacOS -import ftl.run.exception.FlankGeneralError -import ftl.util.Bash -import ftl.util.copyBinaryResource -import java.io.File - -object Parse { - - private val installBinaries by lazy { - if (!isMacOS) { - copyBinaryResource("nm") - copyBinaryResource("swift-demangle") - copyBinaryResource("libatomic.so.1") // swift-demangle dependency - copyBinaryResource("libatomic.so.1.2.0") - } - } - - private fun validateFile(path: String) { - val file = File(path) - if (!file.exists()) { - throw FlankGeneralError("File $path does not exist!") - } - - if (file.isDirectory) throw FlankGeneralError("$path is a directory!") - } - - private fun methodName(matcher: MatchResult): String { - return matcher.groupValues.last() - .replace('.', '/') - .replace(' ', '/') - } - - private fun String.quote(): String { - return "\"$this\"" - } - - internal fun parseObjcTests(binary: String): List { - installBinaries - validateFile(binary) - - val results = mutableListOf() - // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L18 - // must quote binary path in case there are spaces - var cmd = "nm -U ${binary.quote()}" - if (!isMacOS) cmd = "PATH=~/.flank $cmd" - val output = Bash.execute(cmd) - - output.lines().forEach { line -> - // 000089b0 t -[EarlGreyExampleTests testLayout] - // 00008330 t -[EarlGreyExampleTests testCustomAction] - val pattern = """.+\st\s-\[(.+\stest.+)]""".toRegex() - val matcher = pattern.find(line) - if (matcher != null && matcher.groupValues.size == 2) { - results.add(methodName(matcher)) - } - } - return results.distinct() - } - - internal fun parseSwiftTests(binary: String): List { - installBinaries - validateFile(binary) - val results = mutableListOf() - // The OS limits the list of arguments to ARG_MAX. Setting the xargs limit avoids a fatal - // 'argument too long' error. xargs will split the args and run the command for each chunk. - // getconf ARG_MAX - val argMax = 262_144 - - val cmd = if (isMacOS) { - "nm -gU ${binary.quote()} | xargs -s $argMax xcrun swift-demangle" - } else { - "export LD_LIBRARY_PATH=~/.flank; export PATH=~/.flank:\$PATH; nm -gU ${binary.quote()} | xargs -s $argMax swift-demangle" - } - - // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L17-18 - val demangledOutput = Bash.execute(cmd) - demangledOutput.lines().forEach { line -> - // _T025EarlGreyExampleTestsSwift0abceD0C10testLayoutyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testLayout() -> () - // _T025EarlGreyExampleTestsSwift0abceD0C16testCustomActionyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testCustomAction() -> () - // _$S25EarlGreyExampleTestsSwift0abceD0C14testThatThrowsyyKF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testThatThrows() throws -> () - val pattern = """.+\s--->\s.+\.(.+\.test.+)\(\)\s.*->\s\(\)""".toRegex() - val matcher = pattern.find(line) - - if (matcher != null && matcher.groupValues.size == 2) { - results.add(methodName(matcher)) - } - } - return results.distinct() - } -} diff --git a/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt b/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt deleted file mode 100644 index cbd783f120..0000000000 --- a/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt +++ /dev/null @@ -1,133 +0,0 @@ -package ftl.ios - -import com.dd.plist.NSArray -import com.dd.plist.NSDictionary -import com.dd.plist.NSString -import com.dd.plist.PropertyListParser -import com.google.common.annotations.VisibleForTesting -import ftl.run.exception.FlankGeneralError -import java.io.ByteArrayOutputStream -import java.io.File -import java.nio.file.Paths - -typealias XctestrunMethods = Map> - -object Xctestrun { - - private fun String.isMetadata(): Boolean { - return this.contentEquals("__xctestrun_metadata__") - } - - // Parses all tests for a given target - private fun testsForTarget(testDictionary: NSDictionary, testTarget: String, testRoot: String): List { - if (testTarget.isMetadata()) return emptyList() - val skipTestIdentifiers: NSArray? = testDictionary["SkipTestIdentifiers"] as NSArray? - val skipTests: List = skipTestIdentifiers?.array?.mapNotNull { (it as NSString?)?.content } ?: listOf() - val productPaths = testDictionary["DependentProductPaths"] as NSArray - for (product in productPaths.array) { - val productString = product.toString() - if (productString.contains("/$testTarget.xctest")) { - val binaryRoot = productString.replace("__TESTROOT__/", testRoot) - println("Found xctest: $binaryRoot") - - val binaryName = File(binaryRoot).nameWithoutExtension - val binaryPath = Paths.get(binaryRoot, binaryName).toString() - - val tests = (Parse.parseObjcTests(binaryPath) + Parse.parseSwiftTests(binaryPath)).distinct() - - return tests.minus(skipTests) - } - } - - throw FlankGeneralError("No tests found") - } - - // https://github.com/google/xctestrunner/blob/51dbb6b7eb35f2ed55439459ca49e06992bc4da0/xctestrunner/test_runner/xctestrun.py#L129 - private const val onlyTestIdentifiers = "OnlyTestIdentifiers" - - // Rewrites tests so that only the listed tests execute - private fun setOnlyTestIdentifiers(testDictionary: NSDictionary, tests: Collection) { - val nsArray = NSArray(tests.size) - tests.forEachIndexed { index, test -> nsArray.setValue(index, test) } - - while (testDictionary.containsKey(onlyTestIdentifiers)) { - testDictionary.remove(onlyTestIdentifiers) - } - - testDictionary[onlyTestIdentifiers] = nsArray - } - - fun parse(xctestrun: String): NSDictionary { - return parse(File(xctestrun)) - } - - // Parses xctestrun file into a dictonary - fun parse(xctestrun: File): NSDictionary { - val testrun = xctestrun.canonicalFile - if (!testrun.exists()) throw FlankGeneralError("$testrun doesn't exist") - - return PropertyListParser.parse(testrun) as NSDictionary - } - - fun parse(xctestrun: ByteArray): NSDictionary { - return PropertyListParser.parse(xctestrun) as NSDictionary - } - - fun findTestNames(xctestrun: String): XctestrunMethods { - return findTestNames(File(xctestrun)) - } - - fun findTestNames(testTarget: String, xctestrun: String): List = - findTestNamesForTarget( - testTarget = testTarget, - xctestrun = File(xctestrun) - ) - - private fun findTestNames(xctestrun: File): XctestrunMethods = - parse(xctestrun).allKeys().associate { testTarget -> - testTarget to findTestNamesForTarget( - testTarget = testTarget, - xctestrun = xctestrun - ) - } - - private fun findTestNamesForTarget( - testTarget: String, - xctestrun: File - ): List = - testsForTarget( - testDictionary = parse(xctestrun)[testTarget] - as? NSDictionary - ?: throw FlankGeneralError("XCTestrun does not contain $testTarget test target."), - testRoot = xctestrun.parent + "/", - testTarget = testTarget - ).distinct() - - fun rewrite(xctestrun: String, methods: List): ByteArray { - val xctestrunFile = File(xctestrun) - val methodsToRun = findTestNames(xctestrunFile).mapValues { (_, list) -> list.filter(methods::contains) } - return rewrite(parse(xctestrunFile), methodsToRun) - } - - @VisibleForTesting - internal fun rewrite(root: NSDictionary, data: XctestrunMethods): ByteArray { - val rootClone = root.clone() - - for (testTarget in rootClone.allKeys()) { - if (testTarget.isMetadata()) continue - val methods = data[testTarget] - if (methods != null) { - val testDictionary = rootClone[testTarget] as NSDictionary - setOnlyTestIdentifiers(testDictionary, methods) - } - } - - return rootClone.toByteArray() - } - - fun NSDictionary.toByteArray(): ByteArray { - val out = ByteArrayOutputStream() - PropertyListParser.saveAsXML(this, out) - return out.toByteArray() - } -} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt new file mode 100644 index 0000000000..c0b10735a3 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt @@ -0,0 +1,23 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.common.findTestsForTarget +import ftl.ios.xctest.common.isMetadata +import ftl.ios.xctest.common.parseToNSDictionary +import java.io.File + +internal fun findXcTestNamesV1(xctestrun: String): XctestrunMethods = + findXcTestNamesV1(File(xctestrun)) + +private fun findXcTestNamesV1(xctestrun: File): XctestrunMethods = + parseToNSDictionary(xctestrun).run { + val testRoot = xctestrun.parent + "/" + allKeys().filterNot(String::isMetadata).map { testTarget -> + testTarget to findTestsForTarget( + testRoot = testRoot, + testTargetDict = get(testTarget) as NSDictionary, + testTargetName = testTarget, + ) + }.distinct().toMap() + } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt new file mode 100644 index 0000000000..1fc757370b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt @@ -0,0 +1,27 @@ +package ftl.ios.xctest + +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.common.findTestsForTarget +import ftl.ios.xctest.common.getBlueprintName +import ftl.ios.xctest.common.getTestConfigurations +import ftl.ios.xctest.common.getTestTargets +import ftl.ios.xctest.common.parseToNSDictionary +import java.io.File + +internal fun findXcTestNamesV2(xctestrun: String): Map = + findXcTestNamesV2(File(xctestrun)) + +private fun findXcTestNamesV2(xctestrun: File): Map { + val testRoot = xctestrun.parent + "/" + return parseToNSDictionary(xctestrun).getTestConfigurations().mapValues { (_, configDict) -> + configDict.getTestTargets() + .associateBy { targetDict -> targetDict.getBlueprintName() } + .mapValues { (name, dict) -> + findTestsForTarget( + testRoot = testRoot, + testTargetName = name, + testTargetDict = dict, + ) + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt new file mode 100644 index 0000000000..4bfce6a2bd --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt @@ -0,0 +1,36 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import com.google.common.annotations.VisibleForTesting +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.common.isMetadata +import ftl.ios.xctest.common.parseToNSDictionary +import ftl.ios.xctest.common.setOnlyTestIdentifiers +import ftl.ios.xctest.common.toByteArray + +fun rewriteXcTestRunV1( + xctestrun: String, + filterMethods: List +): ByteArray = + rewriteXcTestRunV1( + parseToNSDictionary(xctestrun), + findXcTestNamesV1(xctestrun).mapValues { (_, list) -> + list.filter(filterMethods::contains) + } + ) + +@VisibleForTesting +internal fun rewriteXcTestRunV1( + root: NSDictionary, + methods: XctestrunMethods +): ByteArray = + root.clone().apply { + allKeys().filterNot(String::isMetadata).map { testTarget -> + Pair( + get(testTarget) as NSDictionary, + methods[testTarget] ?: emptyList() + ) + }.forEach { (dict, methods) -> + dict.setOnlyTestIdentifiers(methods) + } + }.toByteArray() diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt new file mode 100644 index 0000000000..10c35c0916 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV2.kt @@ -0,0 +1,52 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.dd.plist.NSObject +import com.google.common.annotations.VisibleForTesting +import ftl.ios.xctest.common.TEST_CONFIGURATIONS +import ftl.ios.xctest.common.XctestrunMethods +import ftl.ios.xctest.common.getBlueprintName +import ftl.ios.xctest.common.getTestConfigurations +import ftl.ios.xctest.common.getTestTargets +import ftl.ios.xctest.common.parseToNSDictionary +import ftl.ios.xctest.common.setOnlyTestIdentifiers +import ftl.ios.xctest.common.toByteArray + +fun rewriteXcTestRunV2( + xcTestPlan: String, + filterMethods: List = emptyList() +): Map = + rewriteXcTestRunV2( + parseToNSDictionary(xcTestPlan), + findXcTestNamesV2(xcTestPlan).filterXcTestMethods(filterMethods), + ) + +@VisibleForTesting +internal fun Map.filterXcTestMethods( + names: List +) = mapValues { (_, map) -> + map.mapValues { (_, list) -> + list.filter(names::contains) + } +} + +@VisibleForTesting +internal fun rewriteXcTestRunV2( + root: NSDictionary, + methods: Map +): Map = + root.getTestConfigurations().mapValues { (configName, configDict) -> + root.clone().also { rootClone -> + rootClone[TEST_CONFIGURATIONS] = configDict.clone().apply { + getTestTargets().forEach { target -> + val testMethods = methods[configName] + ?.get(target.getBlueprintName()) + ?: emptyList() + target.setOnlyTestIdentifiers(testMethods) + } + }.wrapInNSArray() + }.toByteArray() + }.toMap() + +private fun NSObject.wrapInNSArray() = NSArray(1).also { it.setValue(0, this) } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/FindTestsForTarget.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/FindTestsForTarget.kt new file mode 100644 index 0000000000..d7a569fdad --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/FindTestsForTarget.kt @@ -0,0 +1,50 @@ +package ftl.ios.xctest.common + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.dd.plist.NSObject +import com.dd.plist.NSString +import com.google.common.annotations.VisibleForTesting +import ftl.run.exception.FlankGeneralError +import java.io.File +import java.nio.file.Paths + +internal fun findTestsForTarget( + testRoot: String, + testTargetDict: NSDictionary, + testTargetName: String, +): List = testTargetDict + .findXcTestTargets(testTargetName) + .findBinaryTests(testRoot) - testTargetDict + .findTestsToSkip() + +private fun NSDictionary.findXcTestTargets( + testTarget: String +): NSObject = + get("DependentProductPaths") + ?.let { it as? NSArray }?.array + ?.first { product -> product.toString().contains("/$testTarget.xctest") } + ?: throw FlankGeneralError("Test target $testTarget doesn't exist") + +private fun NSObject.findBinaryTests(testRoot: String): List { + val binaryRoot = toString().replace("__TESTROOT__/", testRoot) + println("Found xctest: $binaryRoot") + + val binaryName = File(binaryRoot).nameWithoutExtension + val binaryPath = Paths.get(binaryRoot, binaryName).toString() + + return (parseObjcTests(binaryPath) + parseSwiftTests(binaryPath)).distinct() +} + +private fun NSDictionary.findTestsToSkip(): List = + get("SkipTestIdentifiers") + ?.let { it as? NSArray }?.array + ?.mapNotNull { (it as? NSString)?.content } + ?: emptyList() + +@VisibleForTesting // TODO Remove it, cause is used only in tests +internal fun findTestsForTestTarget(testTarget: String, xctestrun: File): List = + parseToNSDictionary(xctestrun)[testTarget] + ?.let { it as? NSDictionary } + ?.let { findTestsForTarget(xctestrun.parent + "/", it, testTarget) } + ?: throw FlankGeneralError("Test target $testTarget doesn't exist") diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/InstallParseBinaries.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/InstallParseBinaries.kt new file mode 100644 index 0000000000..76a2f35247 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/InstallParseBinaries.kt @@ -0,0 +1,13 @@ +package ftl.ios.xctest.common + +import ftl.config.FtlConstants +import ftl.util.copyBinaryResource + +internal val installBinaries by lazy { + if (!FtlConstants.isMacOS) { + copyBinaryResource("nm") + copyBinaryResource("swift-demangle") + copyBinaryResource("libatomic.so.1") // swift-demangle dependency + copyBinaryResource("libatomic.so.1.2.0") + } +} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseObjTests.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseObjTests.kt new file mode 100644 index 0000000000..dee46a47a3 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseObjTests.kt @@ -0,0 +1,27 @@ +package ftl.ios.xctest.common + +import ftl.config.FtlConstants +import ftl.util.Bash + +internal fun parseObjcTests(binary: String): List { + installBinaries + validateIsFile(binary) + + val results = mutableListOf() + // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L18 + // must quote binary path in case there are spaces + var cmd = "nm -U ${binary.quote()}" + if (!FtlConstants.isMacOS) cmd = "PATH=~/.flank $cmd" + val output = Bash.execute(cmd) + + output.lines().forEach { line -> + // 000089b0 t -[EarlGreyExampleTests testLayout] + // 00008330 t -[EarlGreyExampleTests testCustomAction] + val pattern = """.+\st\s-\[(.+\stest.+)]""".toRegex() + val matcher = pattern.find(line) + if (matcher != null && matcher.groupValues.size == 2) { + results.add(parseXcMethodName(matcher)) + } + } + return results.distinct() +} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseSwiftTests.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseSwiftTests.kt new file mode 100644 index 0000000000..e47a30c511 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseSwiftTests.kt @@ -0,0 +1,35 @@ +package ftl.ios.xctest.common + +import ftl.config.FtlConstants.isMacOS +import ftl.util.Bash + +internal fun parseSwiftTests(binary: String): List { + installBinaries + validateIsFile(binary) + val results = mutableListOf() + // The OS limits the list of arguments to ARG_MAX. Setting the xargs limit avoids a fatal + // 'argument too long' error. xargs will split the args and run the command for each chunk. + // getconf ARG_MAX + val argMax = 262_144 + + val cmd = if (isMacOS) { + "nm -gU ${binary.quote()} | xargs -s $argMax xcrun swift-demangle" + } else { + "export LD_LIBRARY_PATH=~/.flank; export PATH=~/.flank:\$PATH; nm -gU ${binary.quote()} | xargs -s $argMax swift-demangle" + } + + // https://github.com/linkedin/bluepill/blob/37e7efa42472222b81adaa0e88f2bd82aa289b44/Source/Shared/BPXCTestFile.m#L17-18 + val demangledOutput = Bash.execute(cmd) + demangledOutput.lines().forEach { line -> + // _T025EarlGreyExampleTestsSwift0abceD0C10testLayoutyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testLayout() -> () + // _T025EarlGreyExampleTestsSwift0abceD0C16testCustomActionyyF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testCustomAction() -> () + // _$S25EarlGreyExampleTestsSwift0abceD0C14testThatThrowsyyKF ---> EarlGreyExampleTestsSwift.EarlGreyExampleSwiftTests.testThatThrows() throws -> () + val pattern = """.+\s--->\s.+\.(.+\.test.+)\(\)\s.*->\s\(\)""".toRegex() + val matcher = pattern.find(line) + + if (matcher != null && matcher.groupValues.size == 2) { + results.add(parseXcMethodName(matcher)) + } + } + return results.distinct() +} diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt new file mode 100644 index 0000000000..b948a5c156 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/common/Util.kt @@ -0,0 +1,86 @@ +package ftl.ios.xctest.common + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.dd.plist.PropertyListParser +import ftl.run.exception.FlankConfigurationError +import ftl.run.exception.FlankGeneralError +import java.io.ByteArrayOutputStream +import java.io.File + +typealias XctestrunMethods = Map> + +internal const val XCTEST_METADATA = "__xctestrun_metadata__" +internal const val FORMAT_VERSION = "FormatVersion" +internal const val TEST_CONFIGURATIONS = "TestConfigurations" +internal const val TEST_TARGETS = "TestTargets" +internal const val TEST_PLAN = "TestPlan" +internal const val NAME = "Name" +internal const val BLUEPRINT_NAME = "BlueprintName" +internal const val ONLY_TEST_IDENTIFIERS = "OnlyTestIdentifiers" + +internal fun String.isMetadata(): Boolean = contentEquals(XCTEST_METADATA) + +internal fun String.isTestPlan(): Boolean = contentEquals(TEST_PLAN) + +internal fun NSDictionary.getXcTestRunVersion(): Int = + (get(XCTEST_METADATA) as? NSDictionary) + ?.get(FORMAT_VERSION)?.toJavaObject(Int::class.java) + ?: throw FlankGeneralError("Given NSDictionary doesn't contains $FORMAT_VERSION") + +internal fun NSDictionary.getTestConfigurations(): Map = + testConfigurationsNSArray().array.map { it as NSDictionary }.associateBy { + it.getName() + } + +private fun NSDictionary.testConfigurationsNSArray(): NSArray = + get(TEST_CONFIGURATIONS) as NSArray + +internal fun NSDictionary.getTestTargets(): List = + (get(TEST_TARGETS) as NSArray).array.map { it as NSDictionary } + +internal fun NSDictionary.getOnlyTestIdentifiers() = + (get(ONLY_TEST_IDENTIFIERS) as NSArray).array.map { it.toString() } + +private fun NSDictionary.getName(): String = get(NAME) + ?.toJavaObject(String::class.java) + ?: throw FlankConfigurationError("Cannot get Name key from NSDictionary:\n ${toXMLPropertyList()}") + +internal fun NSDictionary.getBlueprintName() = get(BLUEPRINT_NAME).toString() + +internal fun NSDictionary.toByteArray(): ByteArray { + val out = ByteArrayOutputStream() + PropertyListParser.saveAsXML(this, out) + return out.toByteArray() +} + +internal fun parseToNSDictionary(xctestrunPath: String): NSDictionary = + parseToNSDictionary(File(xctestrunPath)) + +internal fun parseToNSDictionary(xctestrun: File): NSDictionary = xctestrun.canonicalFile + .apply { if (!exists()) throw FlankGeneralError("$this doesn't exist") } + .let(PropertyListParser::parse) as NSDictionary + +internal fun parseToNSDictionary(xctestrun: ByteArray): NSDictionary = + PropertyListParser.parse(xctestrun) as NSDictionary + +internal fun parseXcMethodName(matcher: MatchResult): String = matcher.groupValues.last() + .replace('.', '/') + .replace(' ', '/') + +internal fun String.quote() = "\"$this\"" + +internal fun validateIsFile(path: String) = File(path).run { + when { + !exists() -> throw FlankGeneralError("File $path does not exist!") + isDirectory -> throw FlankGeneralError("$path is a directory!") + else -> Unit + } +} + +// https://github.com/google/xctestrunner/blob/51dbb6b7eb35f2ed55439459ca49e06992bc4da0/xctestrunner/test_runner/xctestrun.py#L129 +// Rewrites tests so that only the listed tests execute +internal fun NSDictionary.setOnlyTestIdentifiers(methods: Collection) = apply { + while (containsKey(ONLY_TEST_IDENTIFIERS)) remove(ONLY_TEST_IDENTIFIERS) + this[ONLY_TEST_IDENTIFIERS] = NSArray(methods.size).also { methods.forEachIndexed(it::setValue) } +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index d683a6d13c..7c0ae26cbd 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -8,7 +8,7 @@ import ftl.gc.GcIosTestMatrix import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry -import ftl.ios.Xctestrun +import ftl.ios.xctest.common.parseToNSDictionary import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.model.TestResult @@ -32,7 +32,7 @@ internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope val (stopwatch, runGcsPath) = beforeRunTests(iosArgs) val iosDeviceList = GcIosMatrix.build(iosArgs.devices) - val xcTestParsed = Xctestrun.parse(iosArgs.xctestrunFile) + val xcTestParsed = parseToNSDictionary(iosArgs.xctestrunFile) val jobs = arrayListOf>() val runCount = iosArgs.repeatTests diff --git a/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml new file mode 100644 index 0000000000..84f920b04e --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml @@ -0,0 +1,3 @@ +gcloud: + test: "./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip" + xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/ios/flank_ios_example/FlankExampleTests.xctestrun" diff --git a/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt index e8da56641c..2911f8fc83 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcIosTestMatrixTest.kt @@ -4,7 +4,7 @@ import com.dd.plist.NSDictionary import com.google.api.services.testing.model.IosDeviceList import ftl.args.IosArgs import ftl.config.FtlConstants.isWindows -import ftl.ios.FIXTURES_PATH +import ftl.ios.xctest.FIXTURES_PATH import ftl.shard.Chunk import ftl.shard.TestMethod import ftl.test.util.FlankTestRunner diff --git a/test_runner/src/test/kotlin/ftl/ios/Constants.kt b/test_runner/src/test/kotlin/ftl/ios/Constants.kt deleted file mode 100644 index f85d25e180..0000000000 --- a/test_runner/src/test/kotlin/ftl/ios/Constants.kt +++ /dev/null @@ -1,3 +0,0 @@ -package ftl.ios - -const val FIXTURES_PATH = "./src/test/kotlin/ftl/fixtures/tmp" diff --git a/test_runner/src/test/kotlin/ftl/ios/ParseTest.kt b/test_runner/src/test/kotlin/ftl/ios/ParseTest.kt deleted file mode 100644 index dec00410ff..0000000000 --- a/test_runner/src/test/kotlin/ftl/ios/ParseTest.kt +++ /dev/null @@ -1,112 +0,0 @@ -package ftl.ios - -import com.google.common.truth.Truth.assertThat -import ftl.config.FtlConstants.isWindows -import ftl.test.util.FlankTestRunner -import ftl.run.exception.FlankGeneralError -import org.junit.Assume.assumeFalse -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(FlankTestRunner::class) -class ParseTest { - - private val objcBinary = "$FIXTURES_PATH/ios/earl_grey_example/objc/EarlGreyExampleTests" - private val swiftBinary = "$FIXTURES_PATH/ios/earl_grey_example/swift/EarlGreyExampleSwiftTests" - - private val objcTests = listOf( - "EarlGreyExampleTests/testBasicSelection", - "EarlGreyExampleTests/testBasicSelectionActionAssert", - "EarlGreyExampleTests/testBasicSelectionAndAction", - "EarlGreyExampleTests/testBasicSelectionAndAssert", - "EarlGreyExampleTests/testCatchErrorOnFailure", - "EarlGreyExampleTests/testCollectionMatchers", - "EarlGreyExampleTests/testCustomAction", - "EarlGreyExampleTests/testLayout", - "EarlGreyExampleTests/testSelectionOnMultipleElements", - "EarlGreyExampleTests/testTableCellOutOfScreen", - "EarlGreyExampleTests/testWithCondition", - "EarlGreyExampleTests/testWithCustomAssertion", - "EarlGreyExampleTests/testWithCustomFailureHandler", - "EarlGreyExampleTests/testWithCustomMatcher", - "EarlGreyExampleTests/testWithInRoot" - ) - - private val swiftTests = listOf( - "EarlGreyExampleSwiftTests/testBasicSelection", - "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAction", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert", - "EarlGreyExampleSwiftTests/testCatchErrorOnFailure", - "EarlGreyExampleSwiftTests/testCollectionMatchers", - "EarlGreyExampleSwiftTests/testCustomAction", - "EarlGreyExampleSwiftTests/testLayout", - "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements", - "EarlGreyExampleSwiftTests/testTableCellOutOfScreen", - "EarlGreyExampleSwiftTests/testThatThrows", - "EarlGreyExampleSwiftTests/testWithCondition", - "EarlGreyExampleSwiftTests/testWithCustomAssertion", - "EarlGreyExampleSwiftTests/testWithCustomFailureHandler", - "EarlGreyExampleSwiftTests/testWithCustomMatcher", - "EarlGreyExampleSwiftTests/testWithGreyAssertions", - "EarlGreyExampleSwiftTests/testWithInRoot" - ) - - private fun checkObjcTests(actual: List) { - actual.forEachIndexed { index, result -> - assertThat(objcTests[index]).isEqualTo(result) - } - - assertThat(objcTests.size).isEqualTo(actual.size) - } - - private fun checkSwiftTests(actual: List) { - actual.forEachIndexed { index, result -> - assertThat(result).isEqualTo(swiftTests[index]) - } - - assertThat(actual.size).isEqualTo(swiftTests.size) - } - - @Test - fun `Parse ObjC and Swift with space in path`() { - assumeFalse(isWindows) - - var results = Parse.parseObjcTests("$FIXTURES_PATH/sp ace/objc/EarlGreyExampleTests").sorted() - checkObjcTests(results) - - results = Parse.parseSwiftTests("$FIXTURES_PATH/sp ace/swift/EarlGreyExampleSwiftTests").sorted() - checkSwiftTests(results) - } - - @Test - fun parseObjcTests() { - assumeFalse(isWindows) - - val results = Parse.parseObjcTests(objcBinary).sorted() - checkObjcTests(results) - } - - @Test(expected = FlankGeneralError::class) - fun `parseObjcTests fileNotFound`() { - Parse.parseObjcTests("./BinaryThatDoesNotExist") - } - - @Test - fun parseSwiftTests() { - assumeFalse(isWindows) - - val results = Parse.parseSwiftTests(swiftBinary).sorted() - checkSwiftTests(results) - } - - @Test(expected = FlankGeneralError::class) - fun `parseSwiftTests fileNotFound`() { - Parse.parseSwiftTests("./BinaryThatDoesNotExist") - } - - @Test(expected = FlankGeneralError::class) - fun `parseSwiftTests tmpFolder`() { - Parse.parseSwiftTests("/tmp") - } -} diff --git a/test_runner/src/test/kotlin/ftl/ios/XctestrunTest.kt b/test_runner/src/test/kotlin/ftl/ios/XctestrunTest.kt deleted file mode 100644 index a34b957c59..0000000000 --- a/test_runner/src/test/kotlin/ftl/ios/XctestrunTest.kt +++ /dev/null @@ -1,305 +0,0 @@ -package ftl.ios - -import com.dd.plist.NSArray -import com.dd.plist.NSDictionary -import com.google.common.truth.Truth.assertThat -import ftl.config.FtlConstants.isWindows -import ftl.test.util.FlankTestRunner -import ftl.test.util.TestHelper.normalizeLineEnding -import ftl.run.exception.FlankGeneralError -import org.junit.Assume.assumeFalse -import org.junit.Test -import org.junit.runner.RunWith -import java.nio.file.Files -import java.nio.file.Paths - -@RunWith(FlankTestRunner::class) -class XctestrunTest { - - private val swiftXctestrun = "$FIXTURES_PATH/ios/earl_grey_example/EarlGreyExampleSwiftTests.xctestrun" - private val multipleTargetsSwiftXctestrun = "$FIXTURES_PATH/ios/flank_ios_example/FlankExampleTests.xctestrun" - - private val swiftTests = listOf( - "EarlGreyExampleSwiftTests/testBasicSelection", - "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAction", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert", - "EarlGreyExampleSwiftTests/testCatchErrorOnFailure", - "EarlGreyExampleSwiftTests/testCollectionMatchers", - "EarlGreyExampleSwiftTests/testCustomAction", - "EarlGreyExampleSwiftTests/testLayout", - "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements", - "EarlGreyExampleSwiftTests/testTableCellOutOfScreen", - "EarlGreyExampleSwiftTests/testThatThrows", - "EarlGreyExampleSwiftTests/testWithCondition", - "EarlGreyExampleSwiftTests/testWithCustomAssertion", - "EarlGreyExampleSwiftTests/testWithCustomFailureHandler", - "EarlGreyExampleSwiftTests/testWithCustomMatcher", - "EarlGreyExampleSwiftTests/testWithGreyAssertions", - "EarlGreyExampleSwiftTests/testWithInRoot" - ) - - @Test - fun parse() { - assumeFalse(isWindows) - val result = Xctestrun.parse(swiftXctestrun) - assertThat(arrayOf("EarlGreyExampleSwiftTests", "__xctestrun_metadata__")).isEqualTo(result.allKeys()) - val dict = result["EarlGreyExampleSwiftTests"] as NSDictionary - assertThat(dict.count()).isEqualTo(20) - assertThat(dict.containsKey("OnlyTestIdentifiers")).isFalse() - } - - @Test(expected = FlankGeneralError::class) - fun `parse fileNotFound`() { - Xctestrun.parse("./XctestrunThatDoesNotExist") - } - - @Test - fun findTestNames() { - assumeFalse(isWindows) - - val names = Xctestrun.findTestNames(swiftXctestrun) - .flatMap { it.value } - .sorted() - assertThat(swiftTests).isEqualTo(names) - } - - @Test - fun rewrite() { - assumeFalse(isWindows) - - val root = Xctestrun.parse(swiftXctestrun) - val methods = Xctestrun.findTestNames(testTarget = "EarlGreyExampleSwiftTests", xctestrun = swiftXctestrun) - val methodsData = mapOf>("EarlGreyExampleSwiftTests" to methods) - - val results = String(Xctestrun.rewrite(root, methodsData)) - - assertThat(results.contains("OnlyTestIdentifiers")).isTrue() - } - - @Test - fun rewriteImmutable() { - assumeFalse(isWindows) - - val root = Xctestrun.parse(swiftXctestrun) - val methods = Xctestrun.findTestNames(testTarget = "EarlGreyExampleSwiftTests", xctestrun = swiftXctestrun) - val methodsData = mapOf>("EarlGreyExampleSwiftTests" to methods) - - // ensure root object isn't modified. Rewrite should return a new object. - val key = "OnlyTestIdentifiers" - - assertThat(root.toASCIIPropertyList().contains(key)).isFalse() - - Xctestrun.rewrite(root, methodsData) - - assertThat(root.toASCIIPropertyList().contains(key)).isFalse() - } - - @Test - fun `rewrite idempotent`() { - val root = NSDictionary() - root["EarlGreyExampleSwiftTests"] = NSDictionary() - root["EarlGreyExampleTests"] = NSDictionary() - val methods = listOf("testOne", "testTwo") - val methodsData = mapOf>("EarlGreyExampleSwiftTests" to methods, "EarlGreyExampleTests" to methods) - Xctestrun.rewrite(root, methodsData) - Xctestrun.rewrite(root, methodsData) - val result = Xctestrun.rewrite(root, methodsData) - val expected = """ - - - - EarlGreyExampleSwiftTests - - OnlyTestIdentifiers - - testOne - testTwo - - - EarlGreyExampleTests - - OnlyTestIdentifiers - - testOne - testTwo - - - -""" - assertThat(expected).isEqualTo(String(result).normalizeLineEnding()) - } - - @Test - fun `rewrite preserves skip`() { - val inputXml = """ - - - - - EarlGreyExampleSwiftTests - - SkipTestIdentifiers - - testTwo - - OnlyTestIdentifiers - - testOne - testTwo - - - - - """.trimIndent() - val root = Xctestrun.parse(inputXml.toByteArray()) - - val tests = mapOf>("EarlGreyExampleSwiftTests" to listOf("testOne", "testTwo")) - val rewrittenXml = String(Xctestrun.rewrite(root, tests)) - - assertThat(inputXml).isEqualTo(rewrittenXml.normalizeLineEnding()) - } - - @Test - fun `findTestNames respects skip`() { - assumeFalse(isWindows) - - val inputXml = """ - - - - - EarlGreyExampleSwiftTests - - DependentProductPaths - - __TESTROOT__/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest - __TESTROOT__/Debug-iphoneos/EarlGreyExampleSwift.app - - SkipTestIdentifiers - - EarlGreyExampleSwiftTests/testBasicSelectionActionAssert - EarlGreyExampleSwiftTests/testBasicSelectionAndAction - EarlGreyExampleSwiftTests/testBasicSelectionAndAssert - EarlGreyExampleSwiftTests/testCatchErrorOnFailure - EarlGreyExampleSwiftTests/testCollectionMatchers - EarlGreyExampleSwiftTests/testCustomAction - EarlGreyExampleSwiftTests/testLayout - EarlGreyExampleSwiftTests/testSelectionOnMultipleElements - EarlGreyExampleSwiftTests/testTableCellOutOfScreen - EarlGreyExampleSwiftTests/testThatThrows - EarlGreyExampleSwiftTests/testWithCondition - EarlGreyExampleSwiftTests/testWithCustomAssertion - EarlGreyExampleSwiftTests/testWithCustomFailureHandler - EarlGreyExampleSwiftTests/testWithCustomMatcher - EarlGreyExampleSwiftTests/testWithGreyAssertions - EarlGreyExampleSwiftTests/testWithInRoot - - - - - """.trimIndent() - - val tmpXml = Paths.get(FIXTURES_PATH, "skip.xctestrun") - Files.write(tmpXml, inputXml.toByteArray()) - tmpXml.toFile().deleteOnExit() - - val actualTests = Xctestrun.findTestNames("EarlGreyExampleSwiftTests", tmpXml.toString()).sorted() - assertThat(actualTests).isEqualTo(listOf("EarlGreyExampleSwiftTests/testBasicSelection")) - } - - @Test - fun findTestNamesForTestTarget() { - assumeFalse(isWindows) - val names = Xctestrun.findTestNames(testTarget = "EarlGreyExampleSwiftTests", xctestrun = swiftXctestrun).sorted() - assertThat(swiftTests).isEqualTo(names) - } - - @Test(expected = FlankGeneralError::class) - fun `findTestNames for nonexisting test target`() { - assumeFalse(isWindows) - Xctestrun.findTestNames(testTarget = "Incorrect", xctestrun = swiftXctestrun).sorted() - } - - @Test - fun `find test names for xctestrun file containing multiple test targets`() { - assumeFalse(isWindows) - val names = Xctestrun.findTestNames(testTarget = "FlankExampleTests", xctestrun = multipleTargetsSwiftXctestrun).sorted() - assertThat(names).isEqualTo(listOf("FlankExampleTests/test1", "FlankExampleTests/test2")) - - val names2 = Xctestrun.findTestNames(testTarget = "FlankExampleSecondTests", xctestrun = multipleTargetsSwiftXctestrun).sorted() - assertThat(names2).isEqualTo(listOf("FlankExampleSecondTests/test3", "FlankExampleSecondTests/test4")) - } - - @Test - fun `rewrite methods in single test target`() { - assumeFalse(isWindows) - val methods = listOf("EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", "EarlGreyExampleSwiftTests/testBasicSelection") - val result = Xctestrun.rewrite(xctestrun = swiftXctestrun, methods) - val resultXML = Xctestrun.parse(result) - val testTarget = resultXML["EarlGreyExampleSwiftTests"] as NSDictionary - val resultMethods = testTarget["OnlyTestIdentifiers"] as NSArray - - assertThat(methods.toSet()).isEqualTo(resultMethods.array.map { it.toJavaObject() }.toSet()) - } - - @Test - fun `rewrite methods in multiple test targets`() { - assumeFalse(isWindows) // TODO enable it on #1180 - - val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") - val expectedMethods2 = listOf("FlankExampleSecondTests/test3") - - val result = Xctestrun.rewrite(xctestrun = multipleTargetsSwiftXctestrun, listOf(expectedMethods1, expectedMethods2).flatMap { it }) - val resultXML = Xctestrun.parse(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) - assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) - } - - @Test - fun `rewrite incorrect methods in multiple test targets`() { - assumeFalse(isWindows) // TODO enable it on #1180 - - val methods1 = listOf("incorrect1", "incorrect2") - val methods2 = listOf("incorrect3") - - val result = Xctestrun.rewrite(xctestrun = multipleTargetsSwiftXctestrun, listOf(methods1, methods2).flatMap { it }) - val resultXML = Xctestrun.parse(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(resultMethods1.array.isEmpty()).isTrue() - assertThat(resultMethods2.array.isEmpty()).isTrue() - } - - @Test - fun `rewrite mix of correct and incorrect methods in multiple test targets`() { - assumeFalse(isWindows) // TODO enable it on #1180 - - val methods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2", "incorrect1") - val methods2 = listOf("FlankExampleSecondTests/test3", "incorrect2") - - val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") - val expectedMethods2 = listOf("FlankExampleSecondTests/test3") - - val result = Xctestrun.rewrite(xctestrun = multipleTargetsSwiftXctestrun, listOf(methods1, methods2).flatMap { it }) - val resultXML = Xctestrun.parse(result) - - val target1 = resultXML["FlankExampleTests"] as NSDictionary - val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary - val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray - val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray - - assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) - assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) - } -} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt new file mode 100644 index 0000000000..046a09a5fc --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV1KtTest.kt @@ -0,0 +1,24 @@ +package ftl.ios.xctest + +import com.google.common.truth.Truth.assertThat +import ftl.config.FtlConstants +import org.junit.Assume.assumeFalse +import org.junit.Test + +class FindXcTestNamesV1KtTest { + + @Test + fun findTestNames() { + assumeFalse(FtlConstants.isWindows) + + // when + val names = findXcTestNamesV1(swiftXcTestRunV1) + .flatMap { it.value } + .sorted() + + // then + assertThat(swiftTestsV1).isEqualTo(names) + } +} + + diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt new file mode 100644 index 0000000000..4b203c6852 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt @@ -0,0 +1,21 @@ +package ftl.ios.xctest + +import com.google.common.truth.Truth.assertThat +import ftl.config.FtlConstants +import org.junit.Assume.assumeFalse +import org.junit.Test + +class FindXcTestNamesV2KtTest { + + @Test + fun findTestNames() { + assumeFalse(FtlConstants.isWindows) + + // when + val names = findXcTestNamesV2(swiftXcTestRunV2) + + // then + val sortedNames = names.mapValues { it.value.mapValues { entry -> entry.value.sorted() } } + assertThat(sortedNames).isEqualTo(swiftTestsV2) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt new file mode 100644 index 0000000000..6e7ff76c18 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV1KtTest.kt @@ -0,0 +1,184 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.google.common.truth.Truth.assertThat +import ftl.config.FtlConstants +import ftl.ios.xctest.common.findTestsForTestTarget +import ftl.ios.xctest.common.parseToNSDictionary +import ftl.test.util.TestHelper.normalizeLineEnding +import org.junit.Assume.assumeFalse +import org.junit.Test +import java.io.File + +class RewriteXcTestRunV1KtTest { + + @Test + fun rewrite() { + assumeFalse(FtlConstants.isWindows) + + val root = parseToNSDictionary(swiftXcTestRunV1) + val methods = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)) + val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods) + + val results = String(rewriteXcTestRunV1(root, methodsData)) + + assertThat(results.contains("OnlyTestIdentifiers")).isTrue() + } + + @Test + fun rewriteImmutable() { + assumeFalse(FtlConstants.isWindows) + + val root = parseToNSDictionary(swiftXcTestRunV1) + val methods = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)) + val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods) + + // ensure root object isn't modified. Rewrite should return a new object. + val key = "OnlyTestIdentifiers" + + assertThat(root.toASCIIPropertyList().contains(key)).isFalse() + + rewriteXcTestRunV1(root, methodsData) + + assertThat(root.toASCIIPropertyList().contains(key)).isFalse() + } + + @Test + fun `rewrite idempotent`() { + val root = NSDictionary() + root["EarlGreyExampleSwiftTests"] = NSDictionary() + root["EarlGreyExampleTests"] = NSDictionary() + val methods = listOf("testOne", "testTwo") + val methodsData = mapOf("EarlGreyExampleSwiftTests" to methods, "EarlGreyExampleTests" to methods) + rewriteXcTestRunV1(root, methodsData) + rewriteXcTestRunV1(root, methodsData) + val result = rewriteXcTestRunV1(root, methodsData) + val expected = """ + + + + EarlGreyExampleSwiftTests + + OnlyTestIdentifiers + + testOne + testTwo + + + EarlGreyExampleTests + + OnlyTestIdentifiers + + testOne + testTwo + + + +""" + assertThat(expected).isEqualTo(String(result).normalizeLineEnding()) + } + + @Test + fun `rewrite preserves skip`() { + val inputXml = """ + + + + + EarlGreyExampleSwiftTests + + SkipTestIdentifiers + + testTwo + + OnlyTestIdentifiers + + testOne + testTwo + + + + + """.trimIndent() + val root = parseToNSDictionary(inputXml.toByteArray()) + + val tests = mapOf("EarlGreyExampleSwiftTests" to listOf("testOne", "testTwo")) + val rewrittenXml = String(rewriteXcTestRunV1(root, tests)) + + assertThat(inputXml).isEqualTo(rewrittenXml.normalizeLineEnding()) + } + + + @Test + fun `rewrite methods in single test target`() { + assumeFalse(FtlConstants.isWindows) + val methods = listOf("EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", "EarlGreyExampleSwiftTests/testBasicSelection") + val result = rewriteXcTestRunV1(xctestrun = swiftXcTestRunV1, methods) + val resultXML = parseToNSDictionary(result) + val testTarget = resultXML["EarlGreyExampleSwiftTests"] as NSDictionary + val resultMethods = testTarget["OnlyTestIdentifiers"] as NSArray + + assertThat(methods.toSet()).isEqualTo(resultMethods.array.map { it.toJavaObject() }.toSet()) + } + + @Test + fun `rewrite methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") + val expectedMethods2 = listOf("FlankExampleSecondTests/test3") + + val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(expectedMethods1, expectedMethods2).flatMap { it }) + val resultXML = parseToNSDictionary(result) + + val target1 = resultXML["FlankExampleTests"] as NSDictionary + val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary + val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray + val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray + + assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) + assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) + } + + @Test + fun `rewrite incorrect methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val methods1 = listOf("incorrect1", "incorrect2") + val methods2 = listOf("incorrect3") + + val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(methods1, methods2).flatMap { it }) + val resultXML = parseToNSDictionary(result) + + val target1 = resultXML["FlankExampleTests"] as NSDictionary + val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary + val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray + val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray + + assertThat(resultMethods1.array.isEmpty()).isTrue() + assertThat(resultMethods2.array.isEmpty()).isTrue() + } + + @Test + fun `rewrite mix of correct and incorrect methods in multiple test targets`() { + assumeFalse(FtlConstants.isWindows) // TODO enable it on #1180 + + val methods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2", "incorrect1") + val methods2 = listOf("FlankExampleSecondTests/test3", "incorrect2") + + val expectedMethods1 = listOf("FlankExampleTests/test1", "FlankExampleTests/test2") + val expectedMethods2 = listOf("FlankExampleSecondTests/test3") + + val result = rewriteXcTestRunV1(xctestrun = multiTargetsSwiftXcTestRunV1, listOf(methods1, methods2).flatMap { it }) + val resultXML = parseToNSDictionary(result) + + val target1 = resultXML["FlankExampleTests"] as NSDictionary + val target2 = resultXML["FlankExampleSecondTests"] as NSDictionary + val resultMethods1 = target1["OnlyTestIdentifiers"] as NSArray + val resultMethods2 = target2["OnlyTestIdentifiers"] as NSArray + + assertThat(expectedMethods1.toSet()).isEqualTo(resultMethods1.array.map { it.toJavaObject() }.toSet()) + assertThat(expectedMethods2.toSet()).isEqualTo(resultMethods2.array.map { it.toJavaObject() }.toSet()) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt new file mode 100644 index 0000000000..457ac15656 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/RewriteXcTestRunV2KtTest.kt @@ -0,0 +1,60 @@ +package ftl.ios.xctest + +import com.google.common.truth.Truth.assertThat +import ftl.config.FtlConstants +import ftl.ios.xctest.common.ONLY_TEST_IDENTIFIERS +import ftl.ios.xctest.common.getOnlyTestIdentifiers +import ftl.ios.xctest.common.getTestConfigurations +import ftl.ios.xctest.common.getTestTargets +import ftl.ios.xctest.common.parseToNSDictionary +import org.junit.Assume.assumeFalse +import org.junit.Test + +class RewriteXcTestRunV2KtTest { + + @Test + fun rewrite() { + assumeFalse(FtlConstants.isWindows) + // given + val uiTestsMethods = listOf("UITests/test1") + val secondUiTestsMethods = emptyList() + val expectedTargetNames = listOf("en", "pl") + + // when + val result = rewriteXcTestRunV2( + swiftXcTestRunV2, + uiTestsMethods + ).mapValues { (_, bytes) -> + parseToNSDictionary(bytes) + } + + // then + assertThat(result.keys.toList().sorted()).isEqualTo(expectedTargetNames) + expectedTargetNames.forEach { targetName -> + result.getValue(targetName).getTestConfigurations().values.let { configurations -> + assertThat(configurations.size).isEqualTo(1) + configurations.first().getTestTargets().let { targets -> + assertThat(targets.size).isEqualTo(2) + assertThat(targets[0].getOnlyTestIdentifiers()).isEqualTo(uiTestsMethods) + assertThat(targets[1].getOnlyTestIdentifiers()).isEqualTo(secondUiTestsMethods) + } + } + } + } + + @Test + fun `ensure rewrite not mutates root`() { + assumeFalse(FtlConstants.isWindows) + + // given + val root = parseToNSDictionary(swiftXcTestRunV2) + val methods = findXcTestNamesV2(swiftXcTestRunV2) + assertThat(root.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS)).isFalse() + + // when + rewriteXcTestRunV2(root, methods) + + // then + assertThat(root.toASCIIPropertyList().contains(ONLY_TEST_IDENTIFIERS)).isFalse() + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt new file mode 100644 index 0000000000..dd6261ffe8 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/Util.kt @@ -0,0 +1,99 @@ +package ftl.ios.xctest + +import com.google.common.truth.Truth.assertThat + +const val FIXTURES_PATH = "./src/test/kotlin/ftl/fixtures/tmp" + +const val swiftXcTestRunV1 = "$FIXTURES_PATH/ios/earl_grey_example/EarlGreyExampleSwiftTests.xctestrun" +const val multiTargetsSwiftXcTestRunV1 = "$FIXTURES_PATH/ios/flank_ios_example/FlankExampleTests.xctestrun" +const val swiftXcTestRunV2 = "$FIXTURES_PATH/ios/multi_test_targets/AllTests_AllTests_iphoneos13.7-arm64e.xctestrun" + +const val objcBinary = "$FIXTURES_PATH/ios/earl_grey_example/objc/EarlGreyExampleTests" +const val swiftBinary = "$FIXTURES_PATH/ios/earl_grey_example/swift/EarlGreyExampleSwiftTests" + +val objcTestsV1 = listOf( + "EarlGreyExampleTests/testBasicSelection", + "EarlGreyExampleTests/testBasicSelectionActionAssert", + "EarlGreyExampleTests/testBasicSelectionAndAction", + "EarlGreyExampleTests/testBasicSelectionAndAssert", + "EarlGreyExampleTests/testCatchErrorOnFailure", + "EarlGreyExampleTests/testCollectionMatchers", + "EarlGreyExampleTests/testCustomAction", + "EarlGreyExampleTests/testLayout", + "EarlGreyExampleTests/testSelectionOnMultipleElements", + "EarlGreyExampleTests/testTableCellOutOfScreen", + "EarlGreyExampleTests/testWithCondition", + "EarlGreyExampleTests/testWithCustomAssertion", + "EarlGreyExampleTests/testWithCustomFailureHandler", + "EarlGreyExampleTests/testWithCustomMatcher", + "EarlGreyExampleTests/testWithInRoot" +) + +val swiftTestsV1 = listOf( + "EarlGreyExampleSwiftTests/testBasicSelection", + "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", + "EarlGreyExampleSwiftTests/testBasicSelectionAndAction", + "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert", + "EarlGreyExampleSwiftTests/testCatchErrorOnFailure", + "EarlGreyExampleSwiftTests/testCollectionMatchers", + "EarlGreyExampleSwiftTests/testCustomAction", + "EarlGreyExampleSwiftTests/testLayout", + "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements", + "EarlGreyExampleSwiftTests/testTableCellOutOfScreen", + "EarlGreyExampleSwiftTests/testThatThrows", + "EarlGreyExampleSwiftTests/testWithCondition", + "EarlGreyExampleSwiftTests/testWithCustomAssertion", + "EarlGreyExampleSwiftTests/testWithCustomFailureHandler", + "EarlGreyExampleSwiftTests/testWithCustomMatcher", + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testWithInRoot" +) + +val swiftTestsV2 = mapOf( + "en" to mapOf( + "UITests" to listOf( + "UITests/test1", + "UITests/test1ENLocale", + "UITests/test1PLLocale", + "UITests/test3", + ), + "SecondUITests" to listOf( + "SecondUITests/test11", + "SecondUITests/test12", + "SecondUITests/test13", + "SecondUITests/test2ENLocale", + "SecondUITests/test2PLLocale", + ) + ), + "pl" to mapOf( + "UITests" to listOf( + "UITests/test1", + "UITests/test1ENLocale", + "UITests/test1PLLocale", + "UITests/test3", + ), + "SecondUITests" to listOf( + "SecondUITests/test11", + "SecondUITests/test12", + "SecondUITests/test13", + "SecondUITests/test2ENLocale", + "SecondUITests/test2PLLocale", + ) + ) +) + +fun checkObjcTests(actual: List) { + actual.forEachIndexed { index, result -> + assertThat(objcTestsV1[index]).isEqualTo(result) + } + + assertThat(objcTestsV1.size).isEqualTo(actual.size) +} + +fun checkSwiftTests(actual: List) { + actual.forEachIndexed { index, result -> + assertThat(result).isEqualTo(swiftTestsV1[index]) + } + + assertThat(actual.size).isEqualTo(swiftTestsV1.size) +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/common/FindTestsForTargetKtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/common/FindTestsForTargetKtTest.kt new file mode 100644 index 0000000000..23917ed889 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/common/FindTestsForTargetKtTest.kt @@ -0,0 +1,88 @@ +package ftl.ios.xctest.common + +import com.google.common.truth.Truth.assertThat +import ftl.config.FtlConstants +import ftl.ios.xctest.FIXTURES_PATH +import ftl.ios.xctest.multiTargetsSwiftXcTestRunV1 +import ftl.ios.xctest.swiftTestsV1 +import ftl.ios.xctest.swiftXcTestRunV1 +import ftl.run.exception.FlankGeneralError +import org.junit.Assume.assumeFalse +import org.junit.Test +import java.io.File +import java.nio.file.Files +import java.nio.file.Paths + +class FindTestsForTargetKtTest { + + @Test + fun `findTestNames respects skip`() { + assumeFalse(FtlConstants.isWindows) + + val inputXml = """ + + + + + EarlGreyExampleSwiftTests + + DependentProductPaths + + __TESTROOT__/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest + __TESTROOT__/Debug-iphoneos/EarlGreyExampleSwift.app + + SkipTestIdentifiers + + EarlGreyExampleSwiftTests/testBasicSelectionActionAssert + EarlGreyExampleSwiftTests/testBasicSelectionAndAction + EarlGreyExampleSwiftTests/testBasicSelectionAndAssert + EarlGreyExampleSwiftTests/testCatchErrorOnFailure + EarlGreyExampleSwiftTests/testCollectionMatchers + EarlGreyExampleSwiftTests/testCustomAction + EarlGreyExampleSwiftTests/testLayout + EarlGreyExampleSwiftTests/testSelectionOnMultipleElements + EarlGreyExampleSwiftTests/testTableCellOutOfScreen + EarlGreyExampleSwiftTests/testThatThrows + EarlGreyExampleSwiftTests/testWithCondition + EarlGreyExampleSwiftTests/testWithCustomAssertion + EarlGreyExampleSwiftTests/testWithCustomFailureHandler + EarlGreyExampleSwiftTests/testWithCustomMatcher + EarlGreyExampleSwiftTests/testWithGreyAssertions + EarlGreyExampleSwiftTests/testWithInRoot + + + + + """.trimIndent() + + val tmpXml = Paths.get(FIXTURES_PATH, "skip.xctestrun") + Files.write(tmpXml, inputXml.toByteArray()) + tmpXml.toFile().deleteOnExit() + + val actualTests = findTestsForTestTarget("EarlGreyExampleSwiftTests", tmpXml.toFile()).sorted() + assertThat(actualTests).isEqualTo(listOf("EarlGreyExampleSwiftTests/testBasicSelection")) + } + + @Test + fun findTestNamesForTestTarget() { + assumeFalse(FtlConstants.isWindows) + val names = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)).sorted() + assertThat(swiftTestsV1).isEqualTo(names) + } + + @Test(expected = FlankGeneralError::class) + fun `findTestNames for nonexisting test target`() { + assumeFalse(FtlConstants.isWindows) + findTestsForTestTarget(testTarget = "Incorrect", xctestrun = File(swiftXcTestRunV1)).sorted() + } + + @Test + fun `find test names for xctestrun file containing multiple test targets`() { + assumeFalse(FtlConstants.isWindows) + val names = findTestsForTestTarget(testTarget = "FlankExampleTests", xctestrun = File(multiTargetsSwiftXcTestRunV1)).sorted() + assertThat(names).isEqualTo(listOf("FlankExampleTests/test1", "FlankExampleTests/test2")) + + val names2 = findTestsForTestTarget(testTarget = "FlankExampleSecondTests", xctestrun = File(multiTargetsSwiftXcTestRunV1)).sorted() + assertThat(names2).isEqualTo(listOf("FlankExampleSecondTests/test3", "FlankExampleSecondTests/test4")) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseObjTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseObjTestsKtTest.kt new file mode 100644 index 0000000000..e8fc7f5d60 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseObjTestsKtTest.kt @@ -0,0 +1,33 @@ +package ftl.ios.xctest.common + +import ftl.config.FtlConstants +import ftl.ios.xctest.FIXTURES_PATH +import ftl.ios.xctest.checkObjcTests +import ftl.ios.xctest.objcBinary +import ftl.run.exception.FlankGeneralError +import org.junit.Assume.assumeFalse +import org.junit.Test + +class ParseObjTestsKtTest { + + @Test + fun parseObjcTests() { + assumeFalse(FtlConstants.isWindows) + + val results = parseObjcTests(objcBinary).sorted() + checkObjcTests(results) + } + + @Test(expected = FlankGeneralError::class) + fun `parseObjcTests fileNotFound`() { + parseObjcTests("./BinaryThatDoesNotExist") + } + + @Test + fun `Parse ObjC with space in path`() { + assumeFalse(FtlConstants.isWindows) + + val results = parseObjcTests("$FIXTURES_PATH/sp ace/objc/EarlGreyExampleTests").sorted() + checkObjcTests(results) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseSwiftTestsKtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseSwiftTestsKtTest.kt new file mode 100644 index 0000000000..fa4d6e2818 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseSwiftTestsKtTest.kt @@ -0,0 +1,38 @@ +package ftl.ios.xctest.common + +import ftl.config.FtlConstants +import ftl.ios.xctest.FIXTURES_PATH +import ftl.ios.xctest.checkSwiftTests +import ftl.ios.xctest.swiftBinary +import ftl.run.exception.FlankGeneralError +import org.junit.Assume.assumeFalse +import org.junit.Test + +class ParseSwiftTestsKtTest { + + @Test + fun parseSwiftTests() { + assumeFalse(FtlConstants.isWindows) + + val results = parseSwiftTests(swiftBinary).sorted() + checkSwiftTests(results) + } + + @Test(expected = FlankGeneralError::class) + fun `parseSwiftTests fileNotFound`() { + parseSwiftTests("./BinaryThatDoesNotExist") + } + + @Test(expected = FlankGeneralError::class) + fun `parseSwiftTests tmpFolder`() { + parseSwiftTests("/tmp") + } + + @Test + fun `Parse Swift with space in path`() { + assumeFalse(FtlConstants.isWindows) + + val results = parseSwiftTests("$FIXTURES_PATH/sp ace/swift/EarlGreyExampleSwiftTests").sorted() + checkSwiftTests(results) + } +} diff --git a/test_runner/src/test/kotlin/ftl/ios/xctest/common/UtilKtTest.kt b/test_runner/src/test/kotlin/ftl/ios/xctest/common/UtilKtTest.kt new file mode 100644 index 0000000000..7500005083 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/ios/xctest/common/UtilKtTest.kt @@ -0,0 +1,23 @@ +package ftl.ios.xctest.common + +import com.dd.plist.NSDictionary +import com.google.common.truth.Truth.assertThat +import ftl.ios.xctest.swiftXcTestRunV1 +import ftl.run.exception.FlankGeneralError +import org.junit.Test + +class UtilKtTest { + + @Test + fun parse() { + val result = parseToNSDictionary(swiftXcTestRunV1) + assertThat(arrayOf("EarlGreyExampleSwiftTests", "__xctestrun_metadata__")).isEqualTo(result.allKeys()) + val dict = result["EarlGreyExampleSwiftTests"] as NSDictionary + assertThat(dict.containsKey("OnlyTestIdentifiers")).isFalse() + } + + @Test(expected = FlankGeneralError::class) + fun `parse fileNotFound`() { + parseToNSDictionary("./XctestrunThatDoesNotExist") + } +}