From a9f6144cf60dd85d69d38576548c5208b52cf4ce Mon Sep 17 00:00:00 2001 From: Axel Zuziak Date: Wed, 21 Oct 2020 10:07:52 +0200 Subject: [PATCH] Added sample iOS Project with test plans * added feature doc template * refactored Xctestru * added ios_test_plan.yml --- docs/feature/ios_test_plans.md | 187 ++++++++++++++++++ .../src/main/kotlin/ftl/gc/GcIosTestMatrix.kt | 2 +- .../src/main/kotlin/ftl/ios/Xctestrun.kt | 127 +----------- .../kotlin/ftl/ios/xctest/FindTestNames.kt | 51 +++++ .../kotlin/ftl/ios/xctest/RewriteXcTestRun.kt | 22 +++ .../src/main/kotlin/ftl/ios/xctest/Util.kt | 22 +++ .../kotlin/ftl/fixtures/ios_test_plan.yml | 3 + 7 files changed, 293 insertions(+), 121 deletions(-) create mode 100644 docs/feature/ios_test_plans.md create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/FindTestNames.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRun.kt create mode 100644 test_runner/src/main/kotlin/ftl/ios/xctest/Util.kt create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml diff --git a/docs/feature/ios_test_plans.md b/docs/feature/ios_test_plans.md new file mode 100644 index 0000000000..62cd5a9131 --- /dev/null +++ b/docs/feature/ios_test_plans.md @@ -0,0 +1,187 @@ +# Flow + +Flow starts by parsing .xctestrun file. +Search for: `__xctestrun_metadata__` key. + +```xml +__xctestrun_metadata__ + + FormatVersion + 1 + +``` + +- **FormatVersion: `1` -** old version of .xctestrun +- **FormatVersion: `2` -** the newest version with test plans +- If format is different than 1 or 2 throw an error. + +--- + +### FormatVersion: 1 + +Any other key than metadata should have corresponding **TestTarget** dictionary. In example below `EarlGreyExampleSwiftTests` has a **TestTarget** dictionary. + +```xml + + + EarlGreyExampleSwiftTests + + BlueprintName + EarlGreyExampleSwiftTests + ... + + __xctestrun_metadata__ + + FormatVersion + 1 + + + +``` + +### FormatVersion: 2 + +In this version, XML contains two keys: `TestConfigurations` and `TestPlan` in addition to `__xctestrun_metadata__`. + +`TestPlan` is just dictionary containing basic informations about current **TestPlan.** We can ignore it. + +`TestConfigurations` is an array of different test configurations. Test configuration contains name property and array of TestTargets. + +```xml + + + Name + pl + TestTargets + + + BlueprintName + UITests + + + + BlueprintName + SecondUITests + + + + + +``` + +Each configuration may contain different Environment Variables, languages, regions or any other properties. Those properties are stored under TestTarget. + +Currently **FTL** doesn't support specifying TestConfiguration for test execution. + +If there is more than one configuration FTL will probably choose one arbitrarily. + +For now Flank will allow specifying which test configuration should run with `only-test-configuration` argument. + +--- + +# Running test plan locally + +## Build Xcode project + +To build example project run command below. + +```bash +xcodebuild build-for-testing \ +-allowProvisioningUpdates \ +-project "FlankMultiTestTargetsExample.xcodeproj" \ +-scheme "AllTests" \ #Scheme should have test plans enabled +-derivedDataPath "build_testplan_device" \ +-sdk iphoneos | xcpretty +``` + +This command will generate directory: **Debug-iphoneos** containing binaries and .xctestrun file for each TestPlan. + +In this example scheme `AllTests` has have only one test plan: **AllTests** with two test configurations: `pl` and `en`. + +**Test Plan** contains two **Test Targets: `UITests` and `SecondUITests`** +Outputted .xctestrun should looks like this: + +```xml + + + TestConfigurations + + + Name + en + TestTargets + + + BlueprintName + UITests + TestLanguage + en + TestRegion + GB + + + + BlueprintName + SecondUITests + TestLanguage + en + TestRegion + GB + + + + + + Name + pl + TestTargets + + + BlueprintName + UITests + TestLanguage + pl + TestRegion + PL + + + + BlueprintName + SecondUITests + TestLanguage + pl + TestRegion + PL + + + + + + TestPlan + + IsDefault + + Name + AllTests + + __xctestrun_metadata__ + + FormatVersion + 2 + + + +``` + +## Running tests on a local device + +After generating binaries and .xctestrun file we can run tests using command. + +```bash +xcodebuild test-without-building \ +-xctestrun "build_testplan_device/Build/Products/testrun.xctestrun" \ +-destination "platform=iOS,id=00008030-000209DC1A50802E" \ +-only-test-configuration pl | xcpretty +``` + +Option: `-only-test-configuration pl` allows to specify which test configuration should Xcode run. \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index 0ae83f4a52..975161df4c 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -17,7 +17,7 @@ import ftl.args.IosArgs import ftl.gc.android.mapGcsPathsToFileReference import ftl.gc.android.mapToIosDeviceFiles import ftl.ios.Xctestrun -import ftl.ios.Xctestrun.toByteArray +import ftl.ios.xctest.toByteArray import ftl.run.exception.FlankGeneralError import ftl.util.ShardCounter import ftl.util.join diff --git a/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt b/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt index 52e6e29869..bec83ae254 100644 --- a/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt +++ b/test_runner/src/main/kotlin/ftl/ios/Xctestrun.kt @@ -1,133 +1,20 @@ package ftl.ios -import com.dd.plist.NSArray import com.dd.plist.NSDictionary -import com.dd.plist.NSString -import com.dd.plist.PropertyListParser -import com.google.common.annotations.VisibleForTesting -import ftl.run.exception.FlankGeneralError -import java.io.ByteArrayOutputStream +import ftl.ios.xctest.findTestNames +import ftl.ios.xctest.parseToNSDictionary +import ftl.ios.xctest.rewriteXcTestRun import java.io.File -import java.nio.file.Paths typealias XctestrunMethods = Map> object Xctestrun { - private fun String.isMetadata(): Boolean { - return this.contentEquals("__xctestrun_metadata__") - } + fun parse(xctestrun: String): NSDictionary = parseToNSDictionary(File(xctestrun)) - // Parses all tests for a given target - private fun testsForTarget(testDictionary: NSDictionary, testTarget: String, testRoot: String): List { - if (testTarget.isMetadata()) return emptyList() - val skipTestIdentifiers: NSArray? = testDictionary["SkipTestIdentifiers"] as NSArray? - val skipTests: List = skipTestIdentifiers?.array?.mapNotNull { (it as NSString?)?.content } ?: listOf() - val productPaths = testDictionary["DependentProductPaths"] as NSArray - for (product in productPaths.array) { - val productString = product.toString() - if (productString.contains("/$testTarget.xctest")) { - val binaryRoot = productString.replace("__TESTROOT__/", testRoot) - println("Found xctest: $binaryRoot") + fun parse(xctestrun: ByteArray): NSDictionary = parseToNSDictionary(xctestrun) - val binaryName = File(binaryRoot).nameWithoutExtension - val binaryPath = Paths.get(binaryRoot, binaryName).toString() + fun findTestNames(xctestrun: String): List = findTestNames(File(xctestrun)) - val tests = (Parse.parseObjcTests(binaryPath) + Parse.parseSwiftTests(binaryPath)).distinct() - - return tests.minus(skipTests) - } - } - - throw FlankGeneralError("No tests found") - } - - // https://github.com/google/xctestrunner/blob/51dbb6b7eb35f2ed55439459ca49e06992bc4da0/xctestrunner/test_runner/xctestrun.py#L129 - private const val onlyTestIdentifiers = "OnlyTestIdentifiers" - - // Rewrites tests so that only the listed tests execute - private fun setOnlyTestIdentifiers(testDictionary: NSDictionary, tests: Collection) { - val nsArray = NSArray(tests.size) - tests.forEachIndexed { index, test -> nsArray.setValue(index, test) } - - while (testDictionary.containsKey(onlyTestIdentifiers)) { - testDictionary.remove(onlyTestIdentifiers) - } - - testDictionary[onlyTestIdentifiers] = nsArray - } - - fun parse(xctestrun: String): NSDictionary { - return parse(File(xctestrun)) - } - - // Parses xctestrun file into a dictonary - fun parse(xctestrun: File): NSDictionary { - val testrun = xctestrun.canonicalFile - if (!testrun.exists()) throw FlankGeneralError("$testrun doesn't exist") - - return PropertyListParser.parse(testrun) as NSDictionary - } - - fun parse(xctestrun: ByteArray): NSDictionary { - return PropertyListParser.parse(xctestrun) as NSDictionary - } - - fun findTestNames(xctestrun: String): XctestrunMethods { - return findTestNames(File(xctestrun)) - } - - fun findTestNames(testTarget: String, xctestrun: String): List = - findTestNamesForTarget( - testTarget = testTarget, - xctestrun = File(xctestrun) - ) - - private fun findTestNames(xctestrun: File): XctestrunMethods = - parse(xctestrun).allKeys().associate { testTarget -> - testTarget to findTestNamesForTarget( - testTarget = testTarget, - xctestrun = xctestrun - ) - } - - private fun findTestNamesForTarget( - testTarget: String, - xctestrun: File - ): List = - testsForTarget( - testDictionary = parse(xctestrun)[testTarget] - as? NSDictionary - ?: throw FlankGeneralError("XCTestrun does not contain $testTarget test target."), - testRoot = xctestrun.parent + "/", - testTarget = testTarget - ).distinct() - - fun rewrite(xctestrun: String, methods: List): ByteArray { - val xctestrunFile = File(xctestrun) - val methodsToRun = findTestNames(xctestrunFile).mapValues { (_, list) -> list.filter(methods::contains) } - return rewrite(parse(xctestrunFile), methodsToRun) - } - - @VisibleForTesting - internal fun rewrite(root: NSDictionary, data: XctestrunMethods): ByteArray { - val rootClone = root.clone() - - for (testTarget in rootClone.allKeys()) { - if (testTarget.isMetadata()) continue - val methods = data[testTarget] - if (methods != null) { - val testDictionary = rootClone[testTarget] as NSDictionary - setOnlyTestIdentifiers(testDictionary, methods) - } - } - - return rootClone.toByteArray() - } - - fun NSDictionary.toByteArray(): ByteArray { - val out = ByteArrayOutputStream() - PropertyListParser.saveAsXML(this, out) - return out.toByteArray() - } + fun rewrite(root: NSDictionary, methods: Collection) = rewriteXcTestRun(root, methods) } diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/FindTestNames.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/FindTestNames.kt new file mode 100644 index 0000000000..07c0e4b9eb --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/FindTestNames.kt @@ -0,0 +1,51 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary +import com.dd.plist.NSObject +import com.dd.plist.NSString +import ftl.ios.Parse +import ftl.run.exception.FlankGeneralError +import java.io.File +import java.nio.file.Paths + +// Finds tests in a xctestrun file +internal fun findTestNames(xctestrun: File): List = + parseToNSDictionary(xctestrun).run { + val testRoot = xctestrun.parent + "/" + allKeys().map { testTarget -> + (get(testTarget) as NSDictionary).findTestsForTarget( + testRoot = testRoot, + testTarget = testTarget + ) + }.flatten().distinct() + } + +private fun NSDictionary.findTestsForTarget(testTarget: String, testRoot: String): List = + if (testTarget.isMetadata()) emptyList() + else findXcTestTargets(testTarget) + ?.run { findBinaryTests(testRoot) - findTestsToSkip() } + ?: throw FlankGeneralError("No tests found") + +private fun NSDictionary.findXcTestTargets(testTarget: String): NSObject? = + get("DependentProductPaths") + ?.let { it as? NSArray }?.array + ?.first { product -> product.toString().containsTestTarget(testTarget) } + +private fun String.containsTestTarget(name: String): Boolean = contains("/$name.xctest") + +private fun NSObject.findBinaryTests(testRoot: String): List { + val binaryRoot = toString().replace("__TESTROOT__/", testRoot) + println("Found xctest: $binaryRoot") + + val binaryName = File(binaryRoot).nameWithoutExtension + val binaryPath = Paths.get(binaryRoot, binaryName).toString() + + return (Parse.parseObjcTests(binaryPath) + Parse.parseSwiftTests(binaryPath)).distinct() +} + +private fun NSDictionary.findTestsToSkip(): List = + get("SkipTestIdentifiers") + ?.let { it as? NSArray }?.array + ?.mapNotNull { (it as? NSString)?.content } + ?: emptyList() diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRun.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRun.kt new file mode 100644 index 0000000000..df663f10f9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRun.kt @@ -0,0 +1,22 @@ +package ftl.ios.xctest + +import com.dd.plist.NSArray +import com.dd.plist.NSDictionary + +fun rewriteXcTestRun( + root: NSDictionary, + methods: Collection +): ByteArray = root.clone().apply { + allKeys().filterNot(String::isMetadata).forEach { testTarget -> + (get(testTarget) as NSDictionary).setOnlyTestIdentifiers(methods) + } +}.toByteArray() + +// Rewrites tests so that only the listed tests execute +private fun NSDictionary.setOnlyTestIdentifiers(tests: Collection) { + while (containsKey(ONLY_TEST_IDENTIFIERS)) remove(ONLY_TEST_IDENTIFIERS) + this[ONLY_TEST_IDENTIFIERS] = NSArray(tests.size).also { tests.forEachIndexed(it::setValue) } +} + +// https://github.com/google/xctestrunner/blob/51dbb6b7eb35f2ed55439459ca49e06992bc4da0/xctestrunner/test_runner/xctestrun.py#L129 +private const val ONLY_TEST_IDENTIFIERS = "OnlyTestIdentifiers" diff --git a/test_runner/src/main/kotlin/ftl/ios/xctest/Util.kt b/test_runner/src/main/kotlin/ftl/ios/xctest/Util.kt new file mode 100644 index 0000000000..64c5c47c35 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/ios/xctest/Util.kt @@ -0,0 +1,22 @@ +package ftl.ios.xctest + +import com.dd.plist.NSDictionary +import com.dd.plist.PropertyListParser +import ftl.run.exception.FlankGeneralError +import java.io.ByteArrayOutputStream +import java.io.File + +internal fun String.isMetadata(): Boolean = contentEquals("__xctestrun_metadata__") + +internal fun NSDictionary.toByteArray(): ByteArray { + val out = ByteArrayOutputStream() + PropertyListParser.saveAsXML(this, out) + return out.toByteArray() +} + +internal fun parseToNSDictionary(xctestrun: File): NSDictionary = xctestrun.canonicalFile + .apply { if (!exists()) throw FlankGeneralError("$this doesn't exist") } + .let(PropertyListParser::parse) as NSDictionary + +internal fun parseToNSDictionary(xctestrun: ByteArray): NSDictionary = + PropertyListParser.parse(xctestrun) as NSDictionary diff --git a/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml new file mode 100644 index 0000000000..84f920b04e --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/ios_test_plan.yml @@ -0,0 +1,3 @@ +gcloud: + test: "./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip" + xctestrun-file: "./src/test/kotlin/ftl/fixtures/tmp/ios/flank_ios_example/FlankExampleTests.xctestrun"