diff --git a/docs/feature/1665-custom-sharding.md b/docs/feature/1665-custom-sharding.md new file mode 100644 index 0000000000..5c3b776bc0 --- /dev/null +++ b/docs/feature/1665-custom-sharding.md @@ -0,0 +1,258 @@ +# Custom sharding + +##### NOTE: Currently for android only, iOS support will be released soon + +With [#1665](https://github.com/Flank/flank/issues/1665) Flank received the new feature called `Custom Sharding`. It +enables Flank to consume predefined sharding and apply it during a test run. The feature gives flexibility and enables +manual optimization. It also allows users to set up different sharding per app-test apk pair (android only). + +## Android + +Below you can find an example flow with all features explained + +### 1. Acquire dump shard for current configuration + +Suppose you config with options: + +`flank.yml` + +```yml +gcloud: + app: ./app-debug.apk + robo-script: ./MainActivity_robo_script.json +flank: + max-test-shards: 2 + additional-app-test-apks: + - test: ./debug-1.apk + - test: ../build-dir/debug-2.apk + - test: gs://path/to/your/bucket/debug-3.apk +``` + +`flank firebase test android run -c=flank.yml` will run on 4 matrices: + +* 1x Robo run (just for example that `robo-script` will not collide with custom sharding on `additional-app-test-apks`) +* 3x instrumentation tests with 2 shards max each + +`flank firebase test android run -c=flank.yml --dump-shards` produces `android_shards.json` with sharding: + +```json +{ + "matrix-0": { + "app": "[PATH]/app-debug.apk", + "test": "[PATH]/debug-1.apk", + "shards": { + "shard-0": [ + "class com.TestClassA#test1", + "class com.TestClassA#test2", + "class com.package2.TestClassB#test4" + ], + "shard-1": [ + "class com.TestClassA#test3", + "class com.package2.TestClassB#test1", + "class com.package2.TestClassB#test2", + "class com.package2.TestClassB#test3" + ] + }, + "junit-ignored": [ + "class com.TestClassA#ignoredTest" + ] + }, + "matrix-1": { + "app": "[PATH]/app-debug.apk", + "test": "[PATH]/debug-2.apk", + "shards": { + "shard-0": [ + "class com.ParameterizedTest", + "class com.package3.TestClass3#test5" + ], + "shard-1": [ + "class com.package3.TestClass3#test1", + "class com.package3.TestClass3#test2", + "class com.package3.TestClass3#test3", + "class com.package3.TestClass3#test4" + ] + }, + "junit-ignored": [ + "class com.package.3.TestClassA#ignoredTest1", + "class com.package.3.TestClassA#ignoredTest2" + ] + }, + "matrix-2": { + "app": "[PATH]/app-debug.apk", + "test": "[PATH]/debug-3.apk", + "shards": { + "shard-0": [ + "class com.package4.TestClass4#test1", + "class com.package4.TestClass4#test2", + "class com.package4.subpackage.TestClass6#test3" + ], + "shard-1": [ + "class com.package4.TestClass4#test3", + "class com.package4.subpackage.TestClass6#test1", + "class com.package4.subpackage.TestClass6#test2" + ] + }, + "junit-ignored": [ + ] + } +} +``` + +### 2. Prepare custom sharding JSON file + +You can now make changes as you wish, flank will attempt to find corresponding app-test pair names, and then apply custom sharding. + +1. for `debug-1.apk` let's add another shard and move `TestClassB#test4` & `TestClassB#test3` into it: + +``` +{ + "matrix-0": { + "app": "[PATH]/app-debug.apk", + "test": "[PATH]/debug-1.apk", + "shards": { + "shard-0": [ + "class com.TestClassA#test1", + "class com.TestClassA#test2" + ], + "shard-1": [ + "class com.TestClassA#test3", + "class com.package2.TestClassB#test1", + "class com.package2.TestClassB#test2" + ], + "shard-2": [ + "class com.package2.TestClassB#test4", + "class com.package2.TestClassB#test3" + ] + }, + "junit-ignored": [ + "class com.TestClassA#ignoredTest" + ] + }, + "matrix-1": {...}, + "matrix-2": {...} +} +``` + +2. for `debug-2.apk` we know that parameterized test takes lots of time so we want to have it in a separate shard: + +``` +{ + "matrix-0": {...}, + "matrix-1": { + "app": "[PATH]/app-debug.apk", + "test": "[PATH]/debug-2.apk", + "shards": { + "shard-0": [ + "class com.ParameterizedTest" + ], + "shard-1": [ + "class com.package3.TestClass3#test1", + "class com.package3.TestClass3#test2", + "class com.package3.TestClass3#test3", + "class com.package3.TestClass3#test4", + "class com.package3.TestClass3#test5" + ] + }, + "junit-ignored": [ + "class com.package.3.TestClassA#ignoredTest1", + "class com.package.3.TestClassA#ignoredTest2" + ] + }, + "matrix-2": {...} +} +``` + +3. for `debug-3.apk` all tests are rather quick, so we don't care about sharding, let's move them into one shard: + +``` +{ + "matrix-0": {...}, + "matrix-1": {...}, + "matrix-2": { + "app": "[PATH]/app-debug.apk", + "test": "gs://path/to/your/bucket/debug-3.apk", + "shards": { + "shard-0": [ + "class com.package4.TestClass4#test1", + "class com.package4.TestClass4#test2", + "class com.package4.subpackage.TestClass6#test3", + "class com.package4.TestClass4#test3", + "class com.package4.subpackage.TestClass6#test1", + "class com.package4.subpackage.TestClass6#test2" + ] + }, + "junit-ignored": [ + ] + } +} +``` + +4. Let's save newly created JSON as `custom_sharding.json` + +### 3. Add custom sharding to your configuration + +Update `flank.yml` with `custom-sharding-json` option: + +```yml +gcloud: + app: ./app-debug.apk + robo-script: ./MainActivity_robo_script.json +flank: + max-test-shards: 2 + additional-app-test-apks: + - test: ./debug-1.apk + - test: ../build-dir/debug-2.apk + - test: gs://path/to/your/bucket/debug-3.apk + custom-sharding-json: ./custom_sharding.json +``` + +You can verify if shards are correctly applied by running the following command `flank firebase test android run -c=flank.yml --dump-shards`. +This command will parse your shards configuration JSON into internal structures used for executing test and will print them back to the JSON file. +The diff between the file specified in `custom-sharding-json` and the output file produced by `--dump-shards` should show that no changes were applied to custom shard configuration. + +### 4. Start Test Run + +You can now start a flank test run. With the updated config there will still be 4 matrices: + +* 1x Robo test +* 3x instrumentation tests: + * `debug-1.apk` with 3 shards + * `debug-2.apk` with 2 shards + * `debug-3.apk` with 1 shard + +## NOTE: + +* flank **DOES NOT** validate the provided custom sharding JSON -- it's your responsibility to provide a proper configuration +* flank will apply sharding by searching for test pairs by app apk and test apk paths +* custom sharding supports `gs://` paths +* custom sharding JSON is a source of truth -- no smart sharding is applied (or sharding related configurations) +* matrices ids and shard ids are not important, the only requirement is -- they should be unique +* you can provide custom sharding JSON created entirely from scratch +* custom sharding is very similar to `test-targets-for-shard`, which means you can use the same test targets when + preparing custom sharding. Below example will create 3 shards, one for each of the packages (`bar`, `foo`, `parameterized`): + + ```json + { + "matrix-0": { + "app": "./any-app.apk", + "test": "./any-debug.apk", + "shards": { + "shard-0": [ + "package com.bar" + ], + "shard-1": [ + "package com.parametrized" + ], + "shard-2": [ + "package com.similar" + ] + }, + "junit-ignored": [ + ] + } + } + ``` + +## Problems? Something missing? + +If you believe there is a problem with the custom sharding, or you would like to have some additional feature -- let us know and create an issue in flank's backlog. Any feedback is more than welcome! diff --git a/docs/index.md b/docs/index.md index 68c8fb2027..dedb7d7f4e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -344,6 +344,18 @@ flank: ## Saves output results as parsable file and optionally upload it to Gcloud.. ## Default: none # output-report: none + + ### Disable config validation (for both, yml and command line) + ## If true, Flank won't validate options provided by the user. In general, it's not a good idea but, + ## there are cases when this could be useful for a user + ## (example: project can use devices that are not commonly available, the project has higher sharding limits, etc). + ## Default: false + # skip-config-validation: false + + ### Path to the custom sharding JSON file + ## Flank will apply provided sharding to the configuration. + ## For detailed explanation please check https://github.com/Flank/flank/blob/master/docs/feature/1665-custom-sharding.md + # custom-sharding-json: ./custom_sharding.json ``` ### Android example @@ -700,6 +712,18 @@ flank: ## Saves output results as parsable file and optionally upload it to Gcloud. Possible values are [none, json]. ## Default: none # output-report: none + + ### Disable config validation (for both, yml and command line) + ## If true, Flank won't validate options provided by the user. In general, it's not a good idea but, + ## there are cases when this could be useful for a user + ## (example: project can use devices that are not commonly available, the project has higher sharding limits, etc). + ## Default: false + # skip-config-validation: false + + ### Path to the custom sharding JSON file + ## Flank will apply provided sharding to the configuration. + ## For detailed explanation please check https://github.com/Flank/flank/blob/master/docs/feature/1665-custom-sharding.md + # custom-sharding-json: ./custom_sharding.json ``` ## Android code coverage diff --git a/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt b/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt new file mode 100644 index 0000000000..63f330a257 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt @@ -0,0 +1,186 @@ +package integration + +import FlankCommand +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.google.common.truth.Truth.assertThat +import flank.common.isWindows +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import run +import utils.AndroidTestShards +import utils.CONFIGS_PATH +import utils.FLANK_JAR_PATH +import utils.androidRunCommands +import utils.asOutputReport +import utils.assertContainsUploads +import utils.assertCostMatches +import utils.assertExitCode +import utils.assertTestFail +import utils.assertTestPass +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.json +import utils.loadAsTestSuite +import utils.multipleFailedTests +import utils.multipleSuccessfulTests +import utils.removeUnicode +import utils.toJUnitXmlFile +import utils.toOutputReportFile +import java.nio.file.Paths + +class CustomShardingIT { + private val name = this::class.java.simpleName + + private val jsonMapper by lazy { JsonMapper().registerModule(KotlinModule()) } + + @get:Rule + val root = TemporaryFolder() + + @Test + fun `flank custom sharding -- android`() { + + val templateConfigPath = + "$CONFIGS_PATH/flank_android_custom_sharding.yml" + + val customShardingPath = root.newFile("custom_sharding.json").also { + it.writeText(jsonMapper.writeValueAsString(customSharding)) + }.absolutePath + + val config = root.newFile("flank.yml").also { + it.writeText( + Paths.get(templateConfigPath) + .toFile() + .readText() + .replace("{{PLACEHOLDER}}", customShardingPath) + ) + }.absolutePath + + val result = FlankCommand( + flankPath = FLANK_JAR_PATH, + ymlPath = config, + params = androidRunCommands + ).run("./", name) + + assertExitCode(result, 10) + + val resOutput = result.output.removeUnicode() + + assertContainsUploads( + resOutput, + "app-multiple-success-debug-androidTest.apk", + "app-multiple-error-debug-androidTest.apk", + "MainActivity_robo_script.json" + ) + + resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertTestResultContainsWebLinks() + assertTestPass(multipleSuccessfulTests) + assertTestFail(multipleFailedTests) + + assertThat(testSuites.size).isEqualTo(9) + assertThat(testSuites.filter { it.name == "junit-ignored" }.size).isEqualTo(1) + } + + val outputReport = resOutput.findTestDirectoryFromOutput().toOutputReportFile().json().asOutputReport() + + assertThat(outputReport.error).isEmpty() + assertThat(outputReport.cost).isNotNull() + + outputReport.assertCostMatches() + + assertThat(outputReport.testResults.count()).isEqualTo(4) + assertThat(outputReport.weblinks.count()).isEqualTo(4) + + val testsResults = outputReport.testResults + .map { it.value } + .map { it.testAxises } + .flatten() + + assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(5) + assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(41) + } +} + +val customSharding = + mapOf( + "matrix-0" to AndroidTestShards( + app = "../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.InstrumentedTest#ignoredTestWithSuppress" + ) + ), + "matrix-1" to AndroidTestShards( + app = "../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" + ), + "shard-1" to listOf( + "class com.example.test_app.ParameterizedTest", + "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner", + "class com.example.test_app.InstrumentedTest#test0" + ), + "shard-2" to listOf( + "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", + "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized" + ), + "shard-3" to listOf( + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", + "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + ) + ), + "matrix-2" to AndroidTestShards( + app = "../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" + ), + "shard-1" to listOf( + "class com.example.test_app.ParameterizedTest", + "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner" + ), + "shard-2" to listOf( + "class com.example.test_app.InstrumentedTest#test0", + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo", + "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", + "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", + "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + ) + ) + ).run { + // we need to change files paths to make tests happy when started on windows OS + if (isWindows) mapValues { (_, shards) -> + shards.copy( + app = shards.app.replace("/", "\\"), + test = shards.test.replace("/", "\\") + ) + } + else this + } diff --git a/integration_tests/src/test/resources/cases/flank_android_custom_sharding.yml b/integration_tests/src/test/resources/cases/flank_android_custom_sharding.yml new file mode 100644 index 0000000000..76721e10f3 --- /dev/null +++ b/integration_tests/src/test/resources/cases/flank_android_custom_sharding.yml @@ -0,0 +1,21 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + robo-script: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json + use-orchestrator: false + environment-variables: + coverage: true + coverageFilePath: /sdcard/ + clearPackageData: true +flank: + disable-sharding: false + max-test-shards: 2 + num-test-runs: 1 + additional-app-test-apks: + - test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + - test: ../test_projects/android/apks/invalid.apk + custom-sharding-json: {{PLACEHOLDER}} + output-report: json + disable-results-upload: true + disable-usage-statistics: true diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 98fca55c95..9db28cc44d 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -282,3 +282,9 @@ flank: ## (example: project can use devices that are not commonly available, the project has higher sharding limits, etc). ## Default: false # skip-config-validation: false + + ### Path to the custom sharding JSON file + ## Flank will apply provided sharding to the configuration. + ## For detailed explanation please check https://github.com/Flank/flank/blob/master/docs/feature/1665-custom-sharding.md + # custom-sharding-json: ./custom_sharding.json + diff --git a/test_runner/flank.yml b/test_runner/flank.yml index ed3e076823..101ca87f13 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -359,3 +359,8 @@ flank: ## (example: project can use devices that are not commonly available, the project has higher sharding limits, etc). ## Default: false # skip-config-validation: false + + ### Path to the custom sharding JSON file + ## Flank will apply provided sharding to the configuration. + ## For detailed explanation please check https://github.com/Flank/flank/blob/master/docs/feature/1665-custom-sharding.md + # custom-sharding-json: ./custom_sharding.json diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index f9dee371b9..0ee9541ed3 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -1,9 +1,11 @@ package ftl.args import ftl.analytics.AnonymizeInStatistics +import ftl.analytics.IgnoreInStatistics import ftl.args.yml.AppTestPair import ftl.args.yml.Type import ftl.run.ANDROID_SHARD_FILE +import ftl.run.model.AndroidTestShards import java.nio.file.Paths data class AndroidArgs( @@ -54,7 +56,10 @@ data class AndroidArgs( val obfuscateDumpShards: Boolean, @property:AnonymizeInStatistics - val testTargetsForShard: ShardChunks + val testTargetsForShard: ShardChunks, + + @property:IgnoreInStatistics + val customSharding: Map ) : IArgs by commonArgs { companion object : AndroidArgsCompanion() @@ -121,6 +126,7 @@ AndroidArgs disable-usage-statistics: $disableUsageStatistics output-report: $outputReportType skip-config-validation: $skipConfigValidation + custom-sharding-json: $customShardingJson """.trimIndent() } } diff --git a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt index 99ee3c8045..6367da84a1 100644 --- a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt @@ -48,4 +48,5 @@ data class CommonArgs( override val disableUsageStatistics: Boolean, override val outputReportType: OutputReportType, override val skipConfigValidation: Boolean, + override val customShardingJson: String ) : IArgs diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 451948b865..f64bea71a4 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -4,7 +4,10 @@ import ftl.args.yml.AppTestPair import ftl.config.AndroidConfig import ftl.config.android.AndroidFlankConfig import ftl.config.android.AndroidGcloudConfig +import ftl.run.common.fromJson +import ftl.run.model.AndroidTestShards import ftl.util.require +import java.nio.file.Paths fun createAndroidArgs( config: AndroidConfig? = null, @@ -42,5 +45,20 @@ fun createAndroidArgs( obbFiles = gcloud::obbfiles.require(), obbNames = gcloud::obbnames.require(), grantPermissions = gcloud.grantPermissions, - testTargetsForShard = gcloud.testTargetsForShard?.normalizeToTestTargets().orEmpty() + testTargetsForShard = gcloud.testTargetsForShard?.normalizeToTestTargets().orEmpty(), + customSharding = createCustomShards(commonArgs.customShardingJson) ) + +private fun createCustomShards(shardingJsonPath: String) = + if (shardingJsonPath.isBlank()) emptyMap() + else fromJson>( + Paths.get(shardingJsonPath).toFile().readText() + ).normalizeShardPaths() + +private fun Map.normalizeShardPaths() = + mapValues { (_, shards) -> + shards.copy( + app = shards.app.normalizeFilePath(), + test = shards.test.normalizeFilePath() + ) + } diff --git a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt index 386dcee809..e5d3a97ea7 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt @@ -55,7 +55,8 @@ fun CommonConfig.createCommonArgs( useAverageTestTimeForNewTests = flank::useAverageTestTimeForNewTests.require(), disableUsageStatistics = flank.disableUsageStatistics ?: false, outputReportType = OutputReportType.fromName(flank.outputReport), - skipConfigValidation = flank::skipConfigValidation.require() + skipConfigValidation = flank::skipConfigValidation.require(), + customShardingJson = flank::customShardingJson.require() ).apply { ArgsHelper.createJunitBucket(project, smartFlankGcsPath) } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 494d4f65c8..379e9e7fbc 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -101,6 +101,9 @@ interface IArgs { val shouldValidateConfig: Boolean get() = !skipConfigValidation + @AnonymizeInStatistics + val customShardingJson: String + fun useLocalResultDir() = localResultDir != defaultLocalResultsDir companion object { diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 2188640a13..a7b304bac0 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -100,6 +100,7 @@ IosArgs skip-test-configuration: $skipTestConfiguration output-report: $outputReportType skip-config-validation: $skipConfigValidation + custom-sharding-json: $customShardingJson """.trimIndent() } } diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt index 3476cdfa23..1bac3a1f53 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt @@ -199,6 +199,16 @@ data class CommonFlankConfig @JsonIgnore constructor( @set:JsonProperty("skip-config-validation") var skipConfigValidation: Boolean? by data + @set:CommandLine.Option( + names = ["--custom-sharding-json"], + description = [ + "Path to custom sharding JSON file. Flank will apply provided sharding to the configuration.", + "More info https://github.com/Flank/flank/blob/master/docs/feature/1665-custom-sharding.md" + ] + ) + @set:JsonProperty("custom-sharding-json") + var customShardingJson: String? by data + constructor() : this(mutableMapOf().withDefault { null }) companion object : IYmlKeys { @@ -234,6 +244,7 @@ data class CommonFlankConfig @JsonIgnore constructor( disableUsageStatistics = false outputReport = OutputReportType.NONE.name skipConfigValidation = false + customShardingJson = "" } } } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt index 95b45d7e86..67f7d5a65a 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/CreateAndroidTestContext.kt @@ -18,14 +18,18 @@ import ftl.config.FtlConstants import ftl.filter.TestFilter import ftl.filter.TestFilters import ftl.run.model.AndroidTestContext +import ftl.run.model.AndroidTestShards import ftl.run.model.InstrumentationTestContext +import ftl.shard.Chunk import ftl.shard.createShardsByTestForShards +import ftl.util.FileReference import ftl.util.FlankTestMethod import ftl.util.downloadIfNeeded import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import java.io.File +import ftl.shard.TestMethod as ShardTestMethod suspend fun AndroidArgs.createAndroidTestContexts(): List = resolveApks().setupShards(this) @@ -37,7 +41,8 @@ private suspend fun List.setupShards( async { when { testContext !is InstrumentationTestContext -> testContext - args.testTargetsForShard.isNotEmpty() -> + args.useCustomSharding -> testContext.userShards(args.customSharding) + args.useTestTargetsForShard -> testContext.downloadApks() .calculateDummyShards(args, testFilter) else -> testContext.downloadApks().calculateShards(args, testFilter) @@ -46,6 +51,26 @@ private suspend fun List.setupShards( }.awaitAll().dropEmptyInstrumentationTest() } +private fun InstrumentationTestContext.userShards(customShardingMap: Map) = customShardingMap + .values + .firstOrNull { app.hasReference(it.app) && test.hasReference(it.test) } + ?.let { customSharding -> + copy( + shards = customSharding.shards + .map { methods -> Chunk(methods.value.map(::ShardTestMethod)) }, + ignoredTestCases = customSharding.junitIgnored + ) + } + ?: this + +private fun FileReference.hasReference(path: String) = local == path || gcs == path + +private val AndroidArgs.useCustomSharding: Boolean + get() = customSharding.isNotEmpty() + +private val AndroidArgs.useTestTargetsForShard: Boolean + get() = testTargetsForShard.isNotEmpty() + private fun InstrumentationTestContext.downloadApks(): InstrumentationTestContext = copy( app = app.downloadIfNeeded(), test = test.downloadIfNeeded() diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt index 0f0456f08b..d8c6010876 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/ResolveApks.kt @@ -34,10 +34,10 @@ private fun AndroidArgs.mainApkContext() = appApk?.let { appApk -> } private fun AndroidArgs.additionalApksContexts() = additionalAppTestApks.map { + val appApk = (it.app ?: appApk) + ?: throw FlankGeneralError("Cannot create app-test apks pair for instrumentation tests, missing app apk for test ${it.test}") InstrumentationTestContext( - app = (it.app ?: appApk) - ?.asFileReference() - ?: throw FlankGeneralError("Cannot create app-test apks pair for instrumentation tests, missing app apk for test ${it.test}"), + app = appApk.asFileReference(), test = it.test.asFileReference(), environmentVariables = it.environmentVariables, testTargetsForShard = testTargetsForShard diff --git a/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt b/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt index 8b7ef96525..5e44cb37ce 100644 --- a/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt +++ b/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt @@ -5,7 +5,7 @@ import ftl.util.FlankTestMethod data class TestMethod( val name: String, - val time: Double, + val time: Double = 0.0, val isParameterized: Boolean = false ) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index ae73c67786..d218a102e6 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -381,6 +381,7 @@ AndroidArgs disable-usage-statistics: false output-report: json skip-config-validation: false + custom-sharding-json: """.trimIndent() ) } @@ -455,6 +456,7 @@ AndroidArgs disable-usage-statistics: false output-report: json skip-config-validation: false + custom-sharding-json: """.trimIndent(), args.toString() ) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index b0a5ddb97d..c6071a7739 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -287,6 +287,7 @@ IosArgs skip-test-configuration: output-report: json skip-config-validation: false + custom-sharding-json: """.trimIndent() ) } @@ -352,6 +353,7 @@ IosArgs skip-test-configuration: output-report: none skip-config-validation: false + custom-sharding-json: """.trimIndent(), args.toString() ) diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed-with-additional-apks.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed-with-additional-apks.yml new file mode 100644 index 0000000000..9ec15490f3 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed-with-additional-apks.yml @@ -0,0 +1,18 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + robo-script: ./src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json + num-flaky-test-attempts: 2 + environment-variables: + coverage: true + coverageFilePath: /sdcard/ + clearPackageData: true +flank: + disable-sharding: false + max-test-shards: 2 + num-test-runs: 1 + additional-app-test-apks: + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + - test: ../test_projects/android/apks/invalid.apk + custom-sharding-json: {{PLACEHOLDER}} diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt index 57d3f28d45..a773a08970 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/CreateAndroidTestContextKtTest.kt @@ -4,10 +4,14 @@ import com.google.common.truth.Truth.assertThat import com.linkedin.dex.parser.DexParser import com.linkedin.dex.parser.DexParser.Companion.findTestMethods import com.linkedin.dex.parser.TestMethod +import flank.common.isWindows import ftl.args.AndroidArgs +import ftl.args.normalizeFilePath import ftl.filter.TestFilter import ftl.filter.TestFilters +import ftl.run.common.prettyPrint import ftl.run.model.AndroidTestContext +import ftl.run.model.AndroidTestShards import ftl.run.model.InstrumentationTestContext import ftl.run.model.RoboTestContext import ftl.test.util.mixedConfigYaml @@ -22,10 +26,17 @@ import io.mockk.unmockkAll import kotlinx.coroutines.runBlocking import org.junit.After import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Rule import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.nio.file.Paths class CreateAndroidTestContextKtTest { + @get:Rule + val root = TemporaryFolder() + @After fun tearDown() = unmockkAll() @@ -107,7 +118,8 @@ class CreateAndroidTestContextKtTest { mockkStatic("ftl.run.platform.android.CreateAndroidTestContextKt") every { testInstrumentationContext.getParametrizedClasses() } returns listOf("foo.bar.ParamClass") - val actual = testInstrumentationContext.getFlankTestMethods(TestFilters.fromTestTargets(listOf("class foo.bar.TestClass1"))) + val actual = + testInstrumentationContext.getFlankTestMethods(TestFilters.fromTestTargets(listOf("class foo.bar.TestClass1"))) val expected = listOf( FlankTestMethod("class foo.bar.TestClass1#test1"), FlankTestMethod("class foo.bar.TestClass1#test2") @@ -115,4 +127,134 @@ class CreateAndroidTestContextKtTest { assertThat(actual).isEqualTo(expected) } } + + @Test + fun `should create contexts based of user provided sharding`() { + val customSharding = + mapOf( + "matrix-0" to AndroidTestShards( + app = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.InstrumentedTest#ignoredTestWithSuppress" + ) + ), + "matrix-1" to AndroidTestShards( + app = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" + ), + "shard-1" to listOf( + "class com.example.test_app.ParameterizedTest", + "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner", + "class com.example.test_app.InstrumentedTest#test0" + ), + "shard-2" to listOf( + "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", + "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized" + ), + "shard-3" to listOf( + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", + "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + ) + ), + "matrix-2" to AndroidTestShards( + app = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + test = "./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk", + shards = mapOf( + "shard-0" to listOf( + "class com.example.test_app.InstrumentedTest#test1", + "class com.example.test_app.InstrumentedTest#test2" + ), + "shard-1" to listOf( + "class com.example.test_app.ParameterizedTest", + "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner" + ), + "shard-2" to listOf( + "class com.example.test_app.InstrumentedTest#test0", + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo", + "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", + "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized" + ) + ), + junitIgnored = listOf( + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", + "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + ) + ) + ).run { + // we need to change files paths to make tests happy when started on windows OS + if (isWindows) mapValues { (_, shards) -> + shards.copy( + app = shards.app.normalizeFilePath(), + test = shards.test.normalizeFilePath() + ) + } + else this + } + + val templateConfigPath = + "./src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed-with-additional-apks.yml" + + val customShardingPath = root.newFile("custom_sharding.json").also { + it.writeText(prettyPrint.toJson(customSharding)) + }.absolutePath + + val config = root.newFile("flank.yml").also { + it.writeText( + Paths.get(templateConfigPath) + .toFile() + .readText() + .replace("{{PLACEHOLDER}}", customShardingPath) + ) + }.toPath() + + val actual: List = runBlocking { + AndroidArgs.load(config).createAndroidTestContexts() + } + + // total number contexts + assertEquals(4, actual.size) + + val roboContexts = actual.filterIsInstance() + assertEquals(1, roboContexts.size) + + val instrumentationContexts = actual.filterIsInstance() + assertEquals(3, instrumentationContexts.size) + + customSharding.values.forEach { customShards -> + val context = instrumentationContexts.filter { + it.app.local.contains(customShards.app.drop(1)) && + it.test.local.contains(customShards.test.drop(1)) + }.run { + // there should be only one context with app and test matching + assertEquals(1, size) + get(0) + } + + // ignored tests should be present in the context + assertEquals(customShards.junitIgnored, context.ignoredTestCases) + // all custom shards are present in the context + assertTrue(customShards.shards.values.containsAll(context.shards.map { it.testMethodNames })) + } + } } diff --git a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt index 78154e85bb..8093baa26d 100644 --- a/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/platform/android/ResolveApksKtTest.kt @@ -34,6 +34,7 @@ class ResolveApksKtTest { every { additionalApks } returns emptyList() every { additionalAppTestApks } returns emptyList() every { testTargetsForShard } returns emptyList() + every { customSharding } returns emptyMap() }.resolveApks().toTypedArray() ) } @@ -58,6 +59,7 @@ class ResolveApksKtTest { ) ) every { testTargetsForShard } returns emptyList() + every { customSharding } returns emptyMap() }.resolveApks().toTypedArray() ) }