Skip to content

Commit

Permalink
feat: iOS support for testplans (#1321)
Browse files Browse the repository at this point in the history
Fixes #685 

## Documentation
https://github.com/Flank/flank/blob/master/docs/feature/ios_test_plans.md

## Test Plan
> How do we know the code works?

From repository root:
```
. .env
flankScripts testArtifacts -b master resolve
flankScripts shell buildFlank
cd test_runner
flank ios run -c=./src/test/kotlin/ftl/fixtures/test_app_cases/flank-xctestrunv2-all.yml
```
Feel free to modify `flank-xctestrunv2-all.yml` to test possible edge-cases.

## Checklist

- [x] Documented
- [x] Unit tested
  • Loading branch information
jan-goral authored Dec 8, 2020
1 parent 2ceb0e0 commit 7457b20
Show file tree
Hide file tree
Showing 33 changed files with 924 additions and 482 deletions.
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@ org.gradle.jvmargs=-Xmx2560m -XX:MaxPermSize=256m -Dfile.encoding=UTF-8
kotlin.code.style=official
org.gradle.parallel=true
org.gradle.daemon=true
org.gradle.caching=true
org.gradle.caching=false
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,7 @@ package ftl.args

import ftl.shard.Chunk

data class CalculateShardsResult(val shardChunks: List<Chunk>, val ignoredTestCases: IgnoredTestCases)
data class CalculateShardsResult(
val shardChunks: List<Chunk>,
val ignoredTestCases: IgnoredTestCases
)
17 changes: 3 additions & 14 deletions test_runner/src/main/kotlin/ftl/args/IosArgs.kt
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
package ftl.args

import com.google.common.annotations.VisibleForTesting
import ftl.ios.xctest.XcTestRunData
import ftl.ios.xctest.calculateXcTestRunData
import ftl.ios.xctest.common.XctestrunMethods
import ftl.ios.xctest.findXcTestNamesV1
import ftl.run.exception.FlankConfigurationError
import ftl.shard.Chunk
import ftl.util.FlankTestMethod

data class IosArgs(
val commonArgs: CommonArgs,
Expand All @@ -20,7 +19,7 @@ data class IosArgs(
) : IArgs by commonArgs {

override val useLegacyJUnitResult = true
val testShardChunks: List<Chunk> by lazy { calculateShardChunks() }
val xcTestRunData: XcTestRunData by lazy { calculateXcTestRunData() }

companion object : IosArgsCompanion()

Expand Down Expand Up @@ -77,16 +76,6 @@ IosArgs
}
}

private fun IosArgs.calculateShardChunks() = if (disableSharding)
emptyList() else
ArgsHelper.calculateShards(
filteredTests = filterTests(findXcTestNamesV1(xctestrunFile), testTargets)
.flatMap { it.value }
.distinct()
.map { FlankTestMethod(it, ignored = false) },
args = this
).shardChunks

@VisibleForTesting
internal fun filterTests(
validTestMethods: XctestrunMethods,
Expand Down
30 changes: 30 additions & 0 deletions test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package ftl.args
import ftl.args.yml.Type
import ftl.ios.IosCatalog
import ftl.ios.IosCatalog.getSupportedVersionId
import ftl.ios.xctest.common.mapToRegex
import ftl.run.exception.FlankConfigurationError
import ftl.run.exception.IncompatibleTestDimensionError

Expand All @@ -17,6 +18,7 @@ fun IosArgs.validate() = apply {
assertAdditionalIpas()
validType()
assertGameloop()
assertXcTestRunData()
}

private fun IosArgs.assertGameloop() {
Expand Down Expand Up @@ -84,3 +86,31 @@ private fun IosArgs.validType() {
if (commonArgs.type !in validIosTypes)
throw FlankConfigurationError("Type should be one of ${validIosTypes.joinToString(",")}")
}

private fun IosArgs.assertXcTestRunData() {
if (!disableSharding && testTargets.isNotEmpty()) {
val filteredMethods = xcTestRunData
.shardTargets.values
.flatten()
.flatMap { it.values }
.flatten()

if (filteredMethods.isEmpty()) throw FlankConfigurationError(
"Empty shards. Cannot match any method to $testTargets"
)

if (filteredMethods.size < testTargets.size) {
val regexList = testTargets.mapToRegex()

val notMatched = testTargets.filter {
filteredMethods.all { method ->
regexList.any { regex ->
regex.matches(method)
}
}
}

println("WARNING: cannot match test_targets: $notMatched")
}
}
}
19 changes: 3 additions & 16 deletions test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package ftl.gc

import com.dd.plist.NSDictionary
import com.google.testing.Testing
import com.google.testing.model.ClientInfo
import com.google.testing.model.EnvironmentMatrix
Expand All @@ -17,8 +16,6 @@ import ftl.args.IosArgs
import ftl.gc.android.mapGcsPathsToFileReference
import ftl.gc.android.mapToIosDeviceFiles
import ftl.gc.android.toIosDeviceFile
import ftl.ios.xctest.common.toByteArray
import ftl.ios.xctest.rewriteXcTestRunV1
import ftl.run.exception.FlankGeneralError
import ftl.util.ShardCounter
import ftl.util.join
Expand All @@ -30,10 +27,8 @@ object GcIosTestMatrix {
fun build(
iosDeviceList: IosDeviceList,
testZipGcsPath: String,
runGcsPath: String,
xcTestParsed: NSDictionary,
args: IosArgs,
testTargets: List<String>,
xcTestRun: ByteArray,
shardCounter: ShardCounter,
toolResultsHistory: ToolResultsHistory,
otherFiles: Map<String, String>,
Expand All @@ -45,23 +40,15 @@ object GcIosTestMatrix {

val gcsBucket = args.resultsBucket
val shardName = shardCounter.next()
val matrixGcsSuffix = join(runGcsPath, shardName)
val matrixGcsSuffix = join(args.resultsDir, shardName)
val matrixGcsPath = join(gcsBucket, matrixGcsSuffix)

// Parameterized tests on iOS don't shard correctly.
// Avoid changing Xctestrun file when disableSharding is on.
val generatedXctestrun = if (args.disableSharding) {
xcTestParsed.toByteArray()
} else {
rewriteXcTestRunV1(args.xctestrunFile, testTargets)
}

// Add shard number to file name
val xctestrunNewFileName =
StringBuilder(args.xctestrunFile).insert(args.xctestrunFile.lastIndexOf("."), "_$shardName").toString()

val xctestrunFileGcsPath =
GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, generatedXctestrun)
GcStorage.uploadXCTestFile(xctestrunNewFileName, gcsBucket, matrixGcsSuffix, xcTestRun)

val iOSXCTest = IosXcTest()
.setTestsZip(FileReference().setGcsPath(testZipGcsPath))
Expand Down
27 changes: 13 additions & 14 deletions test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt
Original file line number Diff line number Diff line change
@@ -1,23 +1,22 @@
package ftl.ios.xctest

import com.dd.plist.NSDictionary
import ftl.ios.xctest.common.XctestrunMethods
import ftl.ios.xctest.common.findTestsForTarget
import ftl.ios.xctest.common.isMetadata
import ftl.ios.xctest.common.parseToNSDictionary
import java.io.File

internal fun findXcTestNamesV1(xctestrun: String): XctestrunMethods =
findXcTestNamesV1(File(xctestrun))

private fun findXcTestNamesV1(xctestrun: File): XctestrunMethods =
parseToNSDictionary(xctestrun).run {
val testRoot = xctestrun.parent + "/"
allKeys().filterNot(String::isMetadata).map { testTarget ->
internal fun findXcTestNamesV1(
xcTestRoot: String,
xcTestNsDictionary: NSDictionary
): Map<String, List<String>> =
xcTestNsDictionary
.allKeys()
.filterNot(String::isMetadata)
.map { testTarget ->
testTarget to findTestsForTarget(
testRoot = testRoot,
testTargetDict = get(testTarget) as NSDictionary,
testRoot = xcTestRoot,
testTargetDict = xcTestNsDictionary[testTarget] as NSDictionary,
testTargetName = testTarget,
)
}.distinct().toMap()
}
}
.distinct()
.toMap()
41 changes: 22 additions & 19 deletions test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt
Original file line number Diff line number Diff line change
@@ -1,27 +1,30 @@
package ftl.ios.xctest

import ftl.ios.xctest.common.XctestrunMethods
import com.dd.plist.NSDictionary
import ftl.ios.xctest.common.findTestsForTarget
import ftl.ios.xctest.common.getBlueprintName
import ftl.ios.xctest.common.getTestConfigurations
import ftl.ios.xctest.common.getTestTargets
import ftl.ios.xctest.common.parseToNSDictionary
import java.io.File

internal fun findXcTestNamesV2(xctestrun: String): Map<String, XctestrunMethods> =
findXcTestNamesV2(File(xctestrun))
internal fun findXcTestNamesV2(
xcTestRoot: String,
xcTestNsDictionary: NSDictionary
): Map<String, Map<String, List<String>>> =
xcTestNsDictionary
.getTestConfigurations()
.mapValues { (_, configDict: NSDictionary) ->
configDict.findTestTargetMethods(xcTestRoot)
}

private fun findXcTestNamesV2(xctestrun: File): Map<String, XctestrunMethods> {
val testRoot = xctestrun.parent + "/"
return parseToNSDictionary(xctestrun).getTestConfigurations().mapValues { (_, configDict) ->
configDict.getTestTargets()
.associateBy { targetDict -> targetDict.getBlueprintName() }
.mapValues { (name, dict) ->
findTestsForTarget(
testRoot = testRoot,
testTargetName = name,
testTargetDict = dict,
)
}
}
}
private fun NSDictionary.findTestTargetMethods(
xcTestRoot: String
): Map<String, List<String>> =
getTestTargets()
.associateBy { targetDict -> targetDict.getBlueprintName() }
.mapValues { (name, dict) ->
findTestsForTarget(
testRoot = xcTestRoot,
testTargetName = name,
testTargetDict = dict,
)
}
26 changes: 26 additions & 0 deletions test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV1.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package ftl.ios.xctest

import com.dd.plist.NSDictionary
import ftl.ios.xctest.common.XCTEST_METADATA
import ftl.ios.xctest.common.setOnlyTestIdentifiers

fun NSDictionary.reduceXcTestRunV1(
targets: Map<String, List<String>>,
): NSDictionary = apply {
(keys - targets.keys - XCTEST_METADATA).forEach(this::remove)
targets.forEach { (testTarget, methods) ->
setOnlyTestIdentifiers(testTarget, methods)
}
}

private fun NSDictionary.setOnlyTestIdentifiers(
testTarget: String,
methods: List<String>
) {
set(
testTarget,
getValue(testTarget)
.let { it as NSDictionary }
.setOnlyTestIdentifiers(methods)
)
}
46 changes: 46 additions & 0 deletions test_runner/src/main/kotlin/ftl/ios/xctest/ReduceXcTestRunV2.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package ftl.ios.xctest

import com.dd.plist.NSArray
import com.dd.plist.NSDictionary
import com.dd.plist.NSObject
import ftl.ios.xctest.common.TEST_CONFIGURATIONS
import ftl.ios.xctest.common.TEST_TARGETS
import ftl.ios.xctest.common.getBlueprintName
import ftl.ios.xctest.common.getTestConfigurations
import ftl.ios.xctest.common.getTestTargets
import ftl.ios.xctest.common.setOnlyTestIdentifiers

fun NSDictionary.reduceXcTestRunV2(
configuration: String,
targets: Map<String, List<String>>
): NSDictionary = apply {
set(
TEST_CONFIGURATIONS,
getTestConfigurations()
.getValue(configuration)
.reduceTestConfiguration(targets)
.inNSArray()
)
}

private fun NSDictionary.reduceTestConfiguration(
targetMethods: Map<String, List<String>>
): NSDictionary = apply {
set(
TEST_TARGETS,
getTestTargets().mapNotNull { target ->
targetMethods[target.getBlueprintName()]?.let { methods ->
target.setOnlyTestIdentifiers(methods)
}
}.toNsArray()
)
}

private fun <T : NSObject> List<T>.toNsArray(): NSArray =
NSArray(size).also { nsArray ->
forEachIndexed { index, t ->
nsArray.setValue(index, t)
}
}

private fun NSObject.inNSArray() = NSArray(1).also { it.setValue(0, this) }
36 changes: 0 additions & 36 deletions test_runner/src/main/kotlin/ftl/ios/xctest/RewriteXcTestRunV1.kt

This file was deleted.

Loading

0 comments on commit 7457b20

Please sign in to comment.