Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Smart Flank #385

Merged
merged 17 commits into from
Nov 27, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions release_notes.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
## v4.1 (unreleased)
- `app`, `test`, and `xctestrun-file` now support `~`, environment variables, and globs (`*`, `**`) when resolving paths. [#386](https://github.com/TestArmada/flank/pull/386)
- Update `flank android run` to support `--app`, `--test`, `--test-targets`, `--use-orchestrator` and `--no-use-orchestrator`.
- Add `smartFlankGcsPath` to shard iOS and Android tests by time using historical run data. The amount of shards used is set by `testShards`.

## v4.0.0

4 changes: 3 additions & 1 deletion test_runner/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -38,7 +38,9 @@ tasks.withType<JacocoReport> {
}

tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.allWarningsAsErrors = true
// https://devcenter.bitrise.io/builds/available-environment-variables/
val runningOnBitrise = System.getenv("BITRISE_IO") != null
kotlinOptions.allWarningsAsErrors = runningOnBitrise
}

apply {
19 changes: 14 additions & 5 deletions test_runner/flank.ios.yml
Original file line number Diff line number Diff line change
@@ -11,24 +11,33 @@ gcloud:
# test and xctestrun-file are the only required args
test: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip
xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun
xcode-version: 9.2
xcode-version: 10.1
device:
- model: iphone8
version: 11.2
locale: en
orientation: portrait

flank:
# # Google cloud storage path to store the JUnit XML results from the last run.
#
# smartFlankGcsPath: gs://tmp_flank/flank/test_app_ios.xml

# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
testShards: 1
testShards: 2

# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
repeatTests: 1
# always run - these tests are inserted at the beginning of every shard
# useful if you need to grant permissions or login before other tests run

# # always run - these tests are inserted at the beginning of every shard
# # useful if you need to grant permissions or login before other tests run
#
# test-targets-always-run:
# - a/testGrantPermissions
# test targets - a list of tests to run. omit to run all tests.

# # test targets - a list of tests to run. omit to run all tests.
#
# test-targets:
# - b/testBasicSelection
15 changes: 12 additions & 3 deletions test_runner/flank.yml
Original file line number Diff line number Diff line change
@@ -27,13 +27,22 @@ gcloud:
version: 28

flank:
# # Google cloud storage path to store the JUnit XML results from the last run.
#
# smartFlankGcsPath: gs://tmp_flank/flank/test_app_android.xml

# test shards - the amount of groups to split the test suite into
# set to -1 to use one shard per test.
#
testShards: 1

# repeat tests - the amount of times to run the tests.
# 1 runs the tests once. 10 runs all the tests 10x
#
repeatTests: 1
# always run - these tests are inserted at the beginning of every shard
# useful if you need to grant permissions or login before other tests run

# # always run - these tests are inserted at the beginning of every shard
# # useful if you need to grant permissions or login before other tests run
#
# test-targets-always-run:
# - class com.example.app.ExampleUiTest#testPasses
# - class com.example.app.ExampleUiTest#testPasses
19 changes: 10 additions & 9 deletions test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
Original file line number Diff line number Diff line change
@@ -10,8 +10,9 @@ import ftl.android.UnsupportedVersionId
import ftl.args.ArgsHelper.assertFileExists
import ftl.args.ArgsHelper.assertGcsFileExists
import ftl.args.ArgsHelper.calculateShards
import ftl.args.ArgsHelper.createGcsBucket
import ftl.args.ArgsHelper.createJunitBucket
import ftl.args.ArgsHelper.evaluateFilePath
import ftl.args.ArgsHelper.getGcsBucket
import ftl.args.ArgsHelper.mergeYmlMaps
import ftl.args.ArgsHelper.yamlMapper
import ftl.args.ArgsToString.devicesToString
@@ -63,6 +64,7 @@ class AndroidArgs(
private val flank = flankYml.flank
override val testShards = flank.testShards
override val repeatTests = flank.repeatTests
override val smartFlankGcsPath = flank.smartFlankGcsPath
override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun

// computed properties not specified in yaml
@@ -76,16 +78,12 @@ class AndroidArgs(
}

val filteredTests = getTestMethods(testLocalApk)

calculateShards(
filteredTests,
testTargetsAlwaysRun,
testShards
)
calculateShards(filteredTests, this)
}

init {
resultsBucket = getGcsBucket(projectId, gcloud.resultsBucket)
resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
createJunitBucket(projectId, flank.smartFlankGcsPath)

if (appApk.startsWith(FtlConstants.GCS_PREFIX)) {
assertGcsFileExists(appApk)
@@ -110,6 +108,7 @@ class AndroidArgs(
val testFilter = TestFilters.fromTestTargets(testTargets)
val filteredTests = allTestMethods
.asSequence()
.distinct()
.filter(testFilter.shouldRun)
.map(TestMethod::testName)
.map { "class $it" }
@@ -157,6 +156,7 @@ ${devicesToString(devices)}
flank:
testShards: $testShards
repeatTests: $repeatTests
smartFlankGcsPath: $smartFlankGcsPath
test-targets-always-run:
${listToString(testTargetsAlwaysRun)}
""".trimIndent()
@@ -167,7 +167,8 @@ ${listToString(testTargetsAlwaysRun)}
mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml)
}

fun load(data: Path, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs = load(String(Files.readAllBytes(data)), cli)
fun load(data: Path, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs =
load(String(Files.readAllBytes(data)), cli)

fun load(data: String, cli: AndroidRunCommand = AndroidRunCommand()): AndroidArgs {
val flankYml = yamlMapper.readValue(data, FlankYml::class.java)
4 changes: 2 additions & 2 deletions test_runner/src/main/kotlin/ftl/args/ArgsFileVisitor.kt
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package ftl.args

import ftl.util.Utils.fatalError
import java.io.IOException
import java.nio.file.FileSystems
import java.nio.file.FileVisitOption
@@ -26,7 +25,8 @@ class ArgsFileVisitor(glob: String) : SimpleFileVisitor<Path>() {
}

override fun visitFileFailed(file: Path?, exc: IOException?): FileVisitResult {
fatalError("Failed to visit $file $exc")
// java.nio.file.AccessDeniedException: /tmp/systemd-private-2bc4cd4c824142ab95fb18cbb14165f5-systemd-timesyncd.service-epYUoK
System.err.println("Failed to visit $file ${exc?.message}")
return FileVisitResult.CONTINUE
}

70 changes: 34 additions & 36 deletions test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt
Original file line number Diff line number Diff line change
@@ -11,16 +11,18 @@ import com.google.cloud.storage.BucketInfo
import com.google.cloud.storage.Storage
import com.google.cloud.storage.StorageClass
import com.google.cloud.storage.StorageOptions
import com.google.common.math.IntMath
import ftl.args.yml.IYmlMap
import ftl.config.FtlConstants
import ftl.config.FtlConstants.GCS_PREFIX
import ftl.config.FtlConstants.JSON_FACTORY
import ftl.config.FtlConstants.defaultCredentialPath
import ftl.gc.GcStorage
import ftl.reports.xml.model.JUnitTestResult
import ftl.shard.Shard
import ftl.shard.StringShards
import ftl.shard.stringShards
import ftl.util.Utils
import java.io.File
import java.math.RoundingMode
import java.net.URI
import java.nio.file.Files
import java.nio.file.Path
@@ -95,54 +97,31 @@ object ArgsHelper {
if (validTestMethods.isEmpty()) Utils.fatalError("$from has no tests")
}

fun calculateShards(
testMethodsToShard: Collection<String>,
testMethodsAlwaysRun: Collection<String>,
testShards: Int
): List<List<String>> {
val testShardMethods = testMethodsToShard.distinct().toMutableList()
testShardMethods.removeAll(testMethodsAlwaysRun)

val oneTestPerChunk = testShards == -1
var chunkSize = IntMath.divide(testShardMethods.size, testShards, RoundingMode.UP)

if (oneTestPerChunk || chunkSize < 1) {
chunkSize = 1
}

val testShardChunks = testShardMethods.asSequence()
.chunked(chunkSize)
.map { testMethodsAlwaysRun + it }
.toList()

// Ensure we don't create more VMs than requested. VM count per run should be <= testShards
if (!oneTestPerChunk && testShardChunks.size > testShards) {
Utils.fatalError("Calculated chunks $testShardChunks is > requested $testShards testShards.")
}
if (testShardChunks.isEmpty()) Utils.fatalError("Failed to populate test shard chunks")

return testShardChunks
fun createJunitBucket(projectId: String, junitGcsPath: String) {
if (FtlConstants.useMock || junitGcsPath.isEmpty()) return
val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/')
createGcsBucket(projectId, bucket)
}

fun getGcsBucket(projectId: String, resultsBucket: String): String {
fun createGcsBucket(projectId: String, bucket: String): String {
// com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list
// when testing, use a hard coded results bucket instead.
if (FtlConstants.useMock) return resultsBucket
if (FtlConstants.useMock) return bucket
// test lab supports using a special free storage bucket
// because we don't have access to the root account, it won't show up in the storage list.
if (resultsBucket.startsWith("test-lab-")) return resultsBucket
if (bucket.startsWith("test-lab-")) return bucket
bootstraponline marked this conversation as resolved.
Show resolved Hide resolved

val storage = StorageOptions.newBuilder().setProjectId(projectId).build().service
val bucketLabel = mapOf(Pair("flank", ""))
val storageLocation = "us-central1"

val bucketListOption = Storage.BucketListOption.prefix(resultsBucket)
val bucketListOption = Storage.BucketListOption.prefix(bucket)
val storageList = storage.list(bucketListOption).values?.map { it.name } ?: emptyList()
val bucket = storageList.find { it == resultsBucket }
if (bucket != null) return bucket
val targetBucket = storageList.find { it == bucket }
if (targetBucket != null) return targetBucket

return storage.create(
BucketInfo.newBuilder(resultsBucket)
BucketInfo.newBuilder(targetBucket)
.setStorageClass(StorageClass.REGIONAL)
.setLocation(storageLocation)
.setLabels(bucketLabel)
@@ -176,6 +155,7 @@ object ArgsHelper {

// https://stackoverflow.com/a/2821201/2450315
private val envRegex = Pattern.compile("\\$([a-zA-Z_]+[a-zA-Z0-9_]*)")

private fun evaluateEnvVars(text: String): String {
val buffer = StringBuffer()
val matcher = envRegex.matcher(text)
@@ -196,4 +176,22 @@ object ArgsHelper {

return ArgsFileVisitor("glob:$filePath").walk(searchDir)
}

fun calculateShards(filteredTests: List<String>, args: IArgs): List<List<String>> {
val oldTestResult = GcStorage.downloadJunitXml(args) ?: JUnitTestResult(mutableListOf())
val shardsByTime = Shard.calculateShardsByTime(filteredTests, oldTestResult, args)

return testMethodsAlwaysRun(shardsByTime.stringShards(), args)
}

private fun testMethodsAlwaysRun(shards: StringShards, args: IArgs): StringShards {
val alwaysRun = args.testTargetsAlwaysRun

shards.forEach { shard ->
shard.removeAll(alwaysRun)
shard.addAll(0, alwaysRun)
}

return shards
}
}
1 change: 1 addition & 0 deletions test_runner/src/main/kotlin/ftl/args/IArgs.kt
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@ interface IArgs {
// FlankYml
val testShards: Int
val repeatTests: Int
val smartFlankGcsPath: String
val testTargetsAlwaysRun: List<String>

// computed property
17 changes: 10 additions & 7 deletions test_runner/src/main/kotlin/ftl/args/IosArgs.kt
Original file line number Diff line number Diff line change
@@ -2,6 +2,8 @@ package ftl.args

import ftl.args.ArgsHelper.assertFileExists
import ftl.args.ArgsHelper.assertGcsFileExists
import ftl.args.ArgsHelper.createGcsBucket
import ftl.args.ArgsHelper.createJunitBucket
import ftl.args.ArgsHelper.evaluateFilePath
import ftl.args.ArgsHelper.mergeYmlMaps
import ftl.args.ArgsHelper.validateTestMethods
@@ -29,7 +31,7 @@ class IosArgs(
) : IArgs {

private val gcloud = gcloudYml.gcloud
override val resultsBucket = gcloud.resultsBucket
override val resultsBucket: String
override val recordVideo = gcloud.recordVideo
override val testTimeout = gcloud.timeout
override val async = gcloud.async
@@ -45,6 +47,7 @@ class IosArgs(
private val flank = flankYml.flank
override val testShards = flank.testShards
override val repeatTests = flank.repeatTests
override val smartFlankGcsPath = flank.smartFlankGcsPath
override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun

private val iosFlank = iosFlankYml.flank
@@ -58,16 +61,15 @@ class IosArgs(
validTestMethods
} else {
testTargets
}
}.distinct()

ArgsHelper.calculateShards(
testMethodsToShard = testsToShard,
testMethodsAlwaysRun = testTargetsAlwaysRun,
testShards = testShards
)
ArgsHelper.calculateShards(testsToShard, this)
}

init {
resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket)
createJunitBucket(projectId, flank.smartFlankGcsPath)

if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) {
assertGcsFileExists(xctestrunZip)
} else {
@@ -114,6 +116,7 @@ ${devicesToString(devices)}
flank:
testShards: $testShards
repeatTests: $repeatTests
smartFlankGcsPath: $smartFlankGcsPath
test-targets-always-run:
${listToString(testTargetsAlwaysRun)}
# iOS flank
13 changes: 12 additions & 1 deletion test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt
Original file line number Diff line number Diff line change
@@ -2,24 +2,35 @@ package ftl.args.yml

import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import ftl.config.FtlConstants.GCS_PREFIX
import ftl.util.Utils.fatalError

/** Flank specific parameters for both iOS and Android */
@JsonIgnoreProperties(ignoreUnknown = true)
class FlankYmlParams(
val testShards: Int = 1,
val repeatTests: Int = 1,
val smartFlankGcsPath: String = "",

@field:JsonProperty("test-targets-always-run")
val testTargetsAlwaysRun: List<String> = emptyList()
) {
companion object : IYmlKeys {
override val keys = listOf("testShards", "repeatTests", "test-targets-always-run")
override val keys = listOf("testShards", "repeatTests", "smartFlankGcsPath", "test-targets-always-run")
}

init {
if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1")
if (repeatTests < 1) fatalError("repeatTests must be >= 1")

if (smartFlankGcsPath.isNotEmpty()) {
if (!smartFlankGcsPath.startsWith(GCS_PREFIX)) {
fatalError("smartFlankGcsPath must start with gs://")
}
if (smartFlankGcsPath.count { it == '/' } <= 2 || !smartFlankGcsPath.endsWith(".xml")) {
fatalError("smartFlankGcsPath must be in the format gs://bucket/foo.xml")
}
}
}
}

Loading