From 6faa50e3cdddbafcf0deb1a9cd60e12fa61777ae Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:40:11 +0200 Subject: [PATCH] docs: Investigate flank options (#1131) --- docs/investigate_flank_options.md | 125 ++++++++++++++++++ .../kotlin/ftl/args/ValidateAndroidArgs.kt | 18 +++ .../kotlin/ftl/args/ValidateCommonArgs.kt | 10 +- .../src/main/kotlin/ftl/args/yml/IYmlKeys.kt | 12 ++ .../ftl/config/android/AndroidFlankConfig.kt | 10 +- .../ftl/config/android/AndroidGcloudConfig.kt | 84 ++++++------ .../ftl/config/common/CommonFlankConfig.kt | 27 +--- .../ftl/config/common/CommonGcloudConfig.kt | 15 +-- .../kotlin/ftl/config/ios/IosGcloudConfig.kt | 27 ++-- .../test/kotlin/ftl/args/ArgsHelperTest.kt | 16 ++- 10 files changed, 240 insertions(+), 104 deletions(-) create mode 100644 docs/investigate_flank_options.md diff --git a/docs/investigate_flank_options.md b/docs/investigate_flank_options.md new file mode 100644 index 0000000000..5c0363bc28 --- /dev/null +++ b/docs/investigate_flank_options.md @@ -0,0 +1,125 @@ +# Investigate flank options + +## List of options android + +### gcloud + +1. app +1. test +1. additional-apks +1. auto-google-login +1. no-auto-google-login +1. use-orchestrator +1. no-use-orchestrator +1. environment-variables +1. directories-to-pull +1. other-files +1. performance-metrics +1. no-performance-metrics +1. num-uniform-shards +1. test-runner-class +1. test-targets +1. robo-directives +1. robo-script +1. results-bucket +1. results-dir +1. record-video +1. no-record-video +1. timeout +1. async +1. client-details +1. network-profile +1. results-history-name +1. num-flaky-test-attempts +1. device + +### flank + +1. additional-app-test-apks +1. legacy-junit-result +1. max-test-shards +1. shard-time +1. num-test-runs +1. smart-flank-gcs-path +1. smart-flank-disable-upload +1. disable-sharding +1. test-targets-always-run +1. files-to-download +1. project +1. local-result-dir +1. run-timeout +1. full-junit-result +1. ignore-failed-tests +1. keep-file-path +1. output-style +1. disable-results-upload +1. default-test-time +1. default-class-test-time +1. use-average-test-time-for-new-tests + +## List of options ios + +### gcloud + +1. test +1. xctestrun-file +1. xcode-version +1. results-bucket +1. results-dir +1. record-video +1. no-record-video +1. timeout +1. async +1. client-details +1. network-profile +1. results-history-name +1. num-flaky-test-attempts +1. device + +### flank + +1. test-targets +1. max-test-shards +1. shard-time +1. num-test-runs +1. smart-flank-gcs-path +1. smart-flank-disable-upload +1. disable-sharding +1. test-targets-always-run +1. files-to-download +1. project +1. local-result-dir +1. run-timeout +1. full-junit-result +1. ignore-failed-tests +1. keep-file-path +1. output-style +1. disable-results-upload +1. default-test-time +1. default-class-test-time +1. use-average-test-time-for-new-tests + +## Investigation report + +### environment-variables (Android) + +Set the ```directories-to-pull``` variable to pull from the device directory with coverage report. +There will be no warnings or failure messages when ```environment-variables``` is set without ```directories-to-pull``` +A warning has been added about this. + +### files-to-download (Android) + +In the case where coverage reports need to be downloaded set the ```directories-to-pull``` variable. +There will be no warnings or failures when ```files-to-download``` is set without ```directories-to-pull```. +A warning is added regarding this. + +### disable-sharding (Common) + +Can be enabled by setting ```max-test-shards``` to greater than one. In this case flank will disable sharding +A warning is added regarding this. + +### num-uniform-shards (Android) + +1. When set with ```max-test-shards``` Flank will fail fast. +1. When set with ```disable-sharding```, Flank will disable sharding without any warning + - Warning added about this. diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 7a2937e114..60ffd5a49c 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -22,6 +22,9 @@ fun AndroidArgs.validate() = apply { assertTestFiles() assertOtherFiles() checkResultsDirUnique() + checkEnvironmentVariables() + checkFilesToDownload() + checkNumUniformShards() } private fun AndroidArgs.assertDevicesSupported() = devices @@ -138,3 +141,18 @@ private fun AndroidArgs.assertRoboTest() { private fun AndroidArgs.assertOtherFiles() { otherFiles.forEach { (_, path) -> ArgsHelper.assertFileExists(path, "from otherFiles") } } + +private fun AndroidArgs.checkEnvironmentVariables() { + if (environmentVariables.isNotEmpty() && directoriesToPull.isEmpty()) + println("WARNING: environment-variables set but directories-to-pull is empty, this will result in the coverage file not downloading to the bucket.") +} + +private fun AndroidArgs.checkFilesToDownload() { + if (filesToDownload.isNotEmpty() && directoriesToPull.isEmpty()) + println("WARNING: files-to-download is set but directories-to-pull is empty, the coverage file may fail to download into the bucket.") +} + +private fun AndroidArgs.checkNumUniformShards() { + if ((numUniformShards ?: 0) > 0 && disableSharding) + println("WARNING: disable-sharding is enabled with num-uniform-shards = $numUniformShards, Flank will ignore num-uniform-shards and disable sharding.") +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt index 806ec53fe0..d6115409bc 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt @@ -14,6 +14,7 @@ fun CommonArgs.validate() { assertRepeatTests() assertSmartFlankGcsPath() assertOrientationCorrectness() + checkDisableSharding() } private fun List<Device>.devicesWithMispeltOrientations(availableOrientations: List<String>) = @@ -29,8 +30,8 @@ private fun List<Device>.throwIfAnyMisspelt() = private fun CommonArgs.assertProjectId() { if (project.isBlank()) throw FlankConfigurationError( "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + - "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + - " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" + "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" ) } @@ -72,3 +73,8 @@ fun IArgs.checkResultsDirUnique() { if (useLegacyJUnitResult && GcStorage.exist(resultsBucket, resultsDir)) println("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled\n") } + +fun IArgs.checkDisableSharding() { + if (disableSharding && maxTestShards > 0) + println("WARNING: disable-sharding enabled with max-test-shards = $maxTestShards, Flank will ignore max-test-shard and disable sharding.") +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt index 8ab4e83276..ee7bf9c078 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt @@ -1,5 +1,11 @@ package ftl.args.yml +import com.fasterxml.jackson.annotation.JsonProperty +import kotlin.reflect.KClass +import kotlin.reflect.KMutableProperty +import kotlin.reflect.full.findAnnotation +import kotlin.reflect.full.memberProperties + interface IYmlKeys { val group: String val keys: List<String> @@ -10,6 +16,12 @@ interface IYmlKeys { } } +val KClass<*>.ymlKeys + get() = memberProperties + .filterIsInstance<KMutableProperty<*>>() + .mapNotNull { it.setter.findAnnotation<JsonProperty>() } + .map { it.value } + fun mergeYmlKeys( vararg keys: IYmlKeys ): Map<String, List<String>> = keys diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt index 14fc2976c5..4cadf6e7f4 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.yml.AppTestPair import ftl.args.yml.IYmlKeys +import ftl.args.yml.ymlKeys import ftl.config.Config import picocli.CommandLine @@ -19,7 +20,7 @@ data class AndroidFlankConfig @JsonIgnore constructor( names = ["--additional-app-test-apks"], split = ",", description = ["A list of app & test apks to include in the run. " + - "Useful for running multiple module tests within a single Flank run."] + "Useful for running multiple module tests within a single Flank run."] ) fun additionalAppTestApks(map: Map<String, String>?) { if (map.isNullOrEmpty()) return @@ -54,10 +55,9 @@ data class AndroidFlankConfig @JsonIgnore constructor( override val group = IYmlKeys.Group.FLANK - override val keys = listOf( - "additional-app-test-apks", - "legacy-junit-result" - ) + override val keys by lazy { + AndroidFlankConfig::class.ymlKeys + } fun default() = AndroidFlankConfig().apply { additionalAppTestApks = null diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 71412c1d0c..178b7120c1 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.yml.IYmlKeys +import ftl.args.yml.ymlKeys import ftl.config.Config import ftl.config.FlankDefaults import picocli.CommandLine @@ -23,22 +24,24 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--app"], description = ["The path to the application binary file. " + - "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] ) + @set:JsonProperty("app") var app: String? by data @set:CommandLine.Option( names = ["--test"], description = ["The path to the binary file containing instrumentation tests. " + - "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] + "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] ) + @set:JsonProperty("test") var test: String? by data @set:CommandLine.Option( names = ["--additional-apks"], split = ",", description = ["A list of up to 100 additional APKs to install, in addition to those being directly tested." + - "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "] + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "] ) @set:JsonProperty("additional-apks") var additionalApks: List<String>? by data @@ -46,7 +49,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--auto-google-login"], description = ["Automatically log into the test device using a preconfigured " + - "Google account before beginning the test. Disabled by default."] + "Google account before beginning the test. Disabled by default."] ) @set:JsonProperty("auto-google-login") var autoGoogleLogin: Boolean? by data @@ -63,10 +66,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--use-orchestrator"], description = ["Whether each test runs in its own Instrumentation instance " + - "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + - "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + - "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + - "information about Android Test Orchestrator."] + "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + + "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + + "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + + "information about Android Test Orchestrator."] ) @set:JsonProperty("use-orchestrator") var useOrchestrator: Boolean? by data @@ -83,9 +86,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor( names = ["--environment-variables"], split = ",", description = ["A comma-separated, key=value map of environment variables " + - "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + - "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + - "passed to your test runner (typically AndroidJUnitRunner)"] + "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + + "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + + "passed to your test runner (typically AndroidJUnitRunner)" + + "If you want have downloaded coverage you need also set --directories-to-pull"] ) @set:JsonProperty("environment-variables") var environmentVariables: Map<String, String>? by data @@ -94,11 +98,11 @@ data class AndroidGcloudConfig @JsonIgnore constructor( names = ["--directories-to-pull"], split = ",", description = ["A list of paths that will be copied from the device's " + - "storage to the designated results bucket after the test is complete. These must be absolute paths under " + - "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + - "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + - "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + - "storage, the system will replace it with the external storage path prefix for that device."] + "storage to the designated results bucket after the test is complete. These must be absolute paths under " + + "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + + "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + + "storage, the system will replace it with the external storage path prefix for that device."] ) @set:JsonProperty("directories-to-pull") var directoriesToPull: List<String>? by data @@ -107,8 +111,8 @@ data class AndroidGcloudConfig @JsonIgnore constructor( names = ["--other-files"], split = ",", description = ["A list of device-path=file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push." + - "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." + - "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "] + "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." + + "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "] ) @set:JsonProperty("other-files") var otherFiles: Map<String, String>? by data @@ -116,7 +120,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--performance-metrics"], description = ["Monitor and record performance metrics: CPU, memory, " + - "network usage, and FPS (game-loop only). Disabled by default."] + "network usage, and FPS (game-loop only). Disabled by default."] ) @set:JsonProperty("performance-metrics") var performanceMetrics: Boolean? by data @@ -133,12 +137,12 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--num-uniform-shards"], description = ["Specifies the number of shards into which you want to evenly distribute test cases." + - "The shards are run in parallel on separate devices. For example," + - "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." + - "The number of shards should be less than the total number of test cases." + - "The number of shards specified must be >= 1 and <= 50." + - "This option cannot be used along max-test-shards and is not compatible with smart sharding." + - "If you want to take benefits of smart sharding use max-test-shards."] + "The shards are run in parallel on separate devices. For example," + + "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." + + "The number of shards should be less than the total number of test cases." + + "The number of shards specified must be >= 1 and <= 50." + + "This option cannot be used along max-test-shards and is not compatible with smart sharding." + + "If you want to take benefits of smart sharding use max-test-shards."] ) @set:JsonProperty("num-uniform-shards") var numUniformShards: Int? by data @@ -146,7 +150,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--test-runner-class"], description = ["The fully-qualified Java class name of the instrumentation test runner (default: the last name extracted " + - "from the APK manifest)."] + "from the APK manifest)."] ) @set:JsonProperty("test-runner-class") var testRunnerClass: String? by data @@ -155,10 +159,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor( names = ["--test-targets"], split = ",", description = ["A list of one or more test target filters to apply " + - "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + - "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + - "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + - "information."] + "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + + "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + + "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + + "information."] ) @set:JsonProperty("test-targets") var testTargets: List<String?>? by data @@ -195,23 +199,9 @@ data class AndroidGcloudConfig @JsonIgnore constructor( override val group = IYmlKeys.Group.GCLOUD - override val keys = listOf( - "app", - "test", - "additional-apks", - "auto-google-login", - "use-orchestrator", - "environment-variables", - "directories-to-pull", - "other-files", - "performance-metrics", - "num-uniform-shards", - "test-runner-class", - "test-targets", - "robo-directives", - "robo-script", - "device" - ) + override val keys by lazy { + AndroidGcloudConfig::class.ymlKeys + } fun default() = AndroidGcloudConfig().apply { app = null diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt index 6f46e0df8e..e64f140441 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.ArgsHelper import ftl.args.yml.IYmlKeys +import ftl.args.yml.ymlKeys import ftl.config.Config import ftl.config.FtlConstants import ftl.shard.DEFAULT_CLASS_TEST_TIME_SEC @@ -85,6 +86,7 @@ data class CommonFlankConfig @JsonIgnore constructor( description = ["The Google Cloud Platform project name to use for this invocation. " + "If omitted, then the project from the service account credential is used"] ) + @set:JsonProperty("project") var project: String? by data @set:CommandLine.Option( @@ -165,28 +167,9 @@ data class CommonFlankConfig @JsonIgnore constructor( override val group = IYmlKeys.Group.FLANK - override val keys = listOf( - "max-test-shards", - "shard-time", - "num-test-runs", - "smart-flank-gcs-path", - "smart-flank-disable-upload", - "disable-sharding", - "test-targets-always-run", - "files-to-download", - "project", - "run-timeout", - "legacy-junit-result", - "ignore-failed-tests", - "keep-file-path", - "output-style", - "disable-results-upload", - "full-junit-result", - "local-result-dir", - "default-test-time", - "default-class-test-time", - "local-result-dir" - ) + override val keys by lazy { + CommonFlankConfig::class.ymlKeys + } const val defaultLocalResultsDir = "results" diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt index 9cfb573c51..77ee0526e5 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.ArgsHelper import ftl.args.yml.IYmlKeys +import ftl.args.yml.ymlKeys import ftl.config.Config import ftl.config.Device import ftl.config.FlankDefaults @@ -70,12 +71,14 @@ data class CommonGcloudConfig @JsonIgnore constructor( "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " + "or s. If no unit is given, seconds are assumed. "] ) + @set:JsonProperty("timeout") var timeout: String? by data @set:CommandLine.Option( names = ["--async"], description = ["Invoke a test asynchronously without waiting for test results."] ) + @set:JsonProperty("async") var async: Boolean? by data @set:CommandLine.Option( @@ -123,15 +126,9 @@ data class CommonGcloudConfig @JsonIgnore constructor( override val group = IYmlKeys.Group.GCLOUD - override val keys = listOf( - "results-bucket", - "results-dir", - "record-video", - "timeout", - "async", - "results-history-name", - "num-flaky-test-attempts" - ) + override val keys by lazy { + CommonGcloudConfig::class.ymlKeys + } fun default(android: Boolean) = CommonGcloudConfig().apply { ArgsHelper.yamlMapper.readerFor(CommonGcloudConfig::class.java) diff --git a/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt index 6272f434b0..70d1005a8d 100644 --- a/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty import ftl.args.yml.IYmlKeys +import ftl.args.yml.ymlKeys import ftl.config.Config import picocli.CommandLine @@ -22,18 +23,19 @@ data class IosGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--test"], description = ["The path to the test package (a zip file containing the iOS app " + - "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + - "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + - "is specified."] + "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + + "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + + "is specified."] ) + @set:JsonProperty("test") var test: String? by data @set:CommandLine.Option( names = ["--xctestrun-file"], description = ["The path to an .xctestrun file that will override any " + - ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + - "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + - "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] + ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + + "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + + "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] ) @set:JsonProperty("xctestrun-file") var xctestrunFile: String? by data @@ -41,8 +43,8 @@ data class IosGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--xcode-version"], description = ["The version of Xcode that should be used to run an XCTest. " + - "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + - "all iOS versions selected in the test matrix."] + "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + + "all iOS versions selected in the test matrix."] ) @set:JsonProperty("xcode-version") var xcodeVersion: String? by data @@ -53,12 +55,9 @@ data class IosGcloudConfig @JsonIgnore constructor( override val group = IYmlKeys.Group.GCLOUD - override val keys = listOf( - "test", - "xctestrun-file", - "xcode-version", - "device" - ) + override val keys by lazy { + IosGcloudConfig::class.ymlKeys + } fun default() = IosGcloudConfig().apply { test = null diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt index 0c5e688bee..814e9161ba 100644 --- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt @@ -8,8 +8,6 @@ import ftl.args.ArgsHelper.createGcsBucket import ftl.args.ArgsHelper.validateTestMethods import ftl.args.yml.mergeYmlKeys import ftl.config.FtlConstants -import ftl.config.common.CommonGcloudConfig -import ftl.config.ios.IosGcloudConfig import ftl.gc.GcStorage import ftl.gc.GcStorage.exist import ftl.shard.TestMethod @@ -20,10 +18,11 @@ import ftl.test.util.TestHelper.absolutePath import ftl.test.util.assertThrowsWithMessage import ftl.run.exception.FlankGeneralError import ftl.run.exception.FlankConfigurationError +import io.mockk.mockk +import io.mockk.unmockkAll import io.mockk.every import io.mockk.mockkObject import io.mockk.spyk -import io.mockk.unmockkAll import org.junit.After import org.junit.Assume import org.junit.Rule @@ -48,9 +47,16 @@ class ArgsHelperTest { @Test fun `mergeYmlMaps succeeds`() { - val merged = mergeYmlKeys(CommonGcloudConfig, IosGcloudConfig) + val merged = mergeYmlKeys(mockk() { + every { keys } returns listOf("devices", "test", "apk") + every { group } returns "gcloud" + }, mockk() { + every { keys } returns listOf("xcode-version", "async", "client-details") + every { group } returns "gcloud" + }) + assertThat(merged.keys.size).isEqualTo(1) - assertThat(merged["gcloud"]?.size).isEqualTo(11) + assertThat(merged["gcloud"]?.size).isEqualTo(6) } @Test