From 6faa50e3cdddbafcf0deb1a9cd60e12fa61777ae Mon Sep 17 00:00:00 2001
From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com>
Date: Fri, 25 Sep 2020 16:40:11 +0200
Subject: [PATCH] docs: Investigate flank options (#1131)

---
 docs/investigate_flank_options.md             | 125 ++++++++++++++++++
 .../kotlin/ftl/args/ValidateAndroidArgs.kt    |  18 +++
 .../kotlin/ftl/args/ValidateCommonArgs.kt     |  10 +-
 .../src/main/kotlin/ftl/args/yml/IYmlKeys.kt  |  12 ++
 .../ftl/config/android/AndroidFlankConfig.kt  |  10 +-
 .../ftl/config/android/AndroidGcloudConfig.kt |  84 ++++++------
 .../ftl/config/common/CommonFlankConfig.kt    |  27 +---
 .../ftl/config/common/CommonGcloudConfig.kt   |  15 +--
 .../kotlin/ftl/config/ios/IosGcloudConfig.kt  |  27 ++--
 .../test/kotlin/ftl/args/ArgsHelperTest.kt    |  16 ++-
 10 files changed, 240 insertions(+), 104 deletions(-)
 create mode 100644 docs/investigate_flank_options.md

diff --git a/docs/investigate_flank_options.md b/docs/investigate_flank_options.md
new file mode 100644
index 0000000000..5c0363bc28
--- /dev/null
+++ b/docs/investigate_flank_options.md
@@ -0,0 +1,125 @@
+# Investigate flank options
+
+## List of options android
+
+### gcloud
+
+1. app
+1. test
+1. additional-apks
+1. auto-google-login
+1. no-auto-google-login
+1. use-orchestrator
+1. no-use-orchestrator
+1. environment-variables
+1. directories-to-pull
+1. other-files
+1. performance-metrics
+1. no-performance-metrics
+1. num-uniform-shards
+1. test-runner-class
+1. test-targets
+1. robo-directives
+1. robo-script
+1. results-bucket
+1. results-dir
+1. record-video
+1. no-record-video
+1. timeout
+1. async
+1. client-details
+1. network-profile
+1. results-history-name
+1. num-flaky-test-attempts
+1. device
+
+### flank
+
+1. additional-app-test-apks
+1. legacy-junit-result
+1. max-test-shards
+1. shard-time
+1. num-test-runs
+1. smart-flank-gcs-path
+1. smart-flank-disable-upload
+1. disable-sharding
+1. test-targets-always-run
+1. files-to-download
+1. project
+1. local-result-dir
+1. run-timeout
+1. full-junit-result
+1. ignore-failed-tests
+1. keep-file-path
+1. output-style
+1. disable-results-upload
+1. default-test-time
+1. default-class-test-time
+1. use-average-test-time-for-new-tests
+
+## List of options ios
+
+### gcloud
+
+1. test
+1. xctestrun-file
+1. xcode-version
+1. results-bucket
+1. results-dir
+1. record-video
+1. no-record-video
+1. timeout
+1. async
+1. client-details
+1. network-profile
+1. results-history-name
+1. num-flaky-test-attempts
+1. device
+
+### flank
+
+1. test-targets
+1. max-test-shards
+1. shard-time
+1. num-test-runs
+1. smart-flank-gcs-path
+1. smart-flank-disable-upload
+1. disable-sharding
+1. test-targets-always-run
+1. files-to-download
+1. project
+1. local-result-dir
+1. run-timeout
+1. full-junit-result
+1. ignore-failed-tests
+1. keep-file-path
+1. output-style
+1. disable-results-upload
+1. default-test-time
+1. default-class-test-time
+1. use-average-test-time-for-new-tests
+
+## Investigation report
+
+### environment-variables (Android)
+
+Set the ```directories-to-pull``` variable to pull from the device directory with coverage report.
+There will be no warnings or failure messages when ```environment-variables``` is set without ```directories-to-pull```
+A warning has been added about this.
+
+### files-to-download (Android)
+
+In the case where coverage reports need to be downloaded set the ```directories-to-pull``` variable.
+There will be no warnings or failures when ```files-to-download``` is set without ```directories-to-pull```.
+A warning is added regarding this.
+
+### disable-sharding (Common)
+
+Can be enabled by setting ```max-test-shards``` to greater than one. In this case flank will disable sharding
+A warning is added regarding this.
+
+### num-uniform-shards (Android)
+
+1. When set with ```max-test-shards``` Flank will fail fast.
+1. When set with ```disable-sharding```, Flank will disable sharding without any warning
+   - Warning added about this.
diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt
index 7a2937e114..60ffd5a49c 100644
--- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt
+++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt
@@ -22,6 +22,9 @@ fun AndroidArgs.validate() = apply {
     assertTestFiles()
     assertOtherFiles()
     checkResultsDirUnique()
+    checkEnvironmentVariables()
+    checkFilesToDownload()
+    checkNumUniformShards()
 }
 
 private fun AndroidArgs.assertDevicesSupported() = devices
@@ -138,3 +141,18 @@ private fun AndroidArgs.assertRoboTest() {
 private fun AndroidArgs.assertOtherFiles() {
     otherFiles.forEach { (_, path) -> ArgsHelper.assertFileExists(path, "from otherFiles") }
 }
+
+private fun AndroidArgs.checkEnvironmentVariables() {
+    if (environmentVariables.isNotEmpty() && directoriesToPull.isEmpty())
+        println("WARNING: environment-variables set but directories-to-pull is empty, this will result in the coverage file  not downloading to the bucket.")
+}
+
+private fun AndroidArgs.checkFilesToDownload() {
+    if (filesToDownload.isNotEmpty() && directoriesToPull.isEmpty())
+        println("WARNING: files-to-download is set but directories-to-pull is empty, the coverage file may fail to download into the bucket.")
+}
+
+private fun AndroidArgs.checkNumUniformShards() {
+    if ((numUniformShards ?: 0) > 0 && disableSharding)
+        println("WARNING: disable-sharding is enabled with num-uniform-shards = $numUniformShards, Flank will ignore num-uniform-shards and disable sharding.")
+}
diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt
index 806ec53fe0..d6115409bc 100644
--- a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt
+++ b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt
@@ -14,6 +14,7 @@ fun CommonArgs.validate() {
     assertRepeatTests()
     assertSmartFlankGcsPath()
     assertOrientationCorrectness()
+    checkDisableSharding()
 }
 
 private fun List<Device>.devicesWithMispeltOrientations(availableOrientations: List<String>) =
@@ -29,8 +30,8 @@ private fun List<Device>.throwIfAnyMisspelt() =
 private fun CommonArgs.assertProjectId() {
     if (project.isBlank()) throw FlankConfigurationError(
         "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" +
-                "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" +
-                " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
+            "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" +
+            " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
     )
 }
 
@@ -72,3 +73,8 @@ fun IArgs.checkResultsDirUnique() {
     if (useLegacyJUnitResult && GcStorage.exist(resultsBucket, resultsDir))
         println("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled\n")
 }
+
+fun IArgs.checkDisableSharding() {
+    if (disableSharding && maxTestShards > 0)
+        println("WARNING: disable-sharding enabled with max-test-shards = $maxTestShards, Flank will ignore max-test-shard and disable sharding.")
+}
diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt
index 8ab4e83276..ee7bf9c078 100644
--- a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt
+++ b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt
@@ -1,5 +1,11 @@
 package ftl.args.yml
 
+import com.fasterxml.jackson.annotation.JsonProperty
+import kotlin.reflect.KClass
+import kotlin.reflect.KMutableProperty
+import kotlin.reflect.full.findAnnotation
+import kotlin.reflect.full.memberProperties
+
 interface IYmlKeys {
     val group: String
     val keys: List<String>
@@ -10,6 +16,12 @@ interface IYmlKeys {
     }
 }
 
+val KClass<*>.ymlKeys
+    get() = memberProperties
+            .filterIsInstance<KMutableProperty<*>>()
+            .mapNotNull { it.setter.findAnnotation<JsonProperty>() }
+            .map { it.value }
+
 fun mergeYmlKeys(
     vararg keys: IYmlKeys
 ): Map<String, List<String>> = keys
diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt
index 14fc2976c5..4cadf6e7f4 100644
--- a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt
+++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.yml.AppTestPair
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import picocli.CommandLine
 
@@ -19,7 +20,7 @@ data class AndroidFlankConfig @JsonIgnore constructor(
         names = ["--additional-app-test-apks"],
         split = ",",
         description = ["A list of app & test apks to include in the run. " +
-                "Useful for running multiple module tests within a single Flank run."]
+            "Useful for running multiple module tests within a single Flank run."]
     )
     fun additionalAppTestApks(map: Map<String, String>?) {
         if (map.isNullOrEmpty()) return
@@ -54,10 +55,9 @@ data class AndroidFlankConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.FLANK
 
-        override val keys = listOf(
-            "additional-app-test-apks",
-            "legacy-junit-result"
-        )
+        override val keys by lazy {
+            AndroidFlankConfig::class.ymlKeys
+        }
 
         fun default() = AndroidFlankConfig().apply {
             additionalAppTestApks = null
diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt
index 71412c1d0c..178b7120c1 100644
--- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt
+++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import ftl.config.FlankDefaults
 import picocli.CommandLine
@@ -23,22 +24,24 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--app"],
         description = ["The path to the application binary file. " +
-                "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."]
+            "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."]
     )
+    @set:JsonProperty("app")
     var app: String? by data
 
     @set:CommandLine.Option(
         names = ["--test"],
         description = ["The path to the binary file containing instrumentation tests. " +
-                "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."]
+            "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."]
     )
+    @set:JsonProperty("test")
     var test: String? by data
 
     @set:CommandLine.Option(
         names = ["--additional-apks"],
         split = ",",
         description = ["A list of up to 100 additional APKs to install, in addition to those being directly tested." +
-                "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "]
+            "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "]
     )
     @set:JsonProperty("additional-apks")
     var additionalApks: List<String>? by data
@@ -46,7 +49,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--auto-google-login"],
         description = ["Automatically log into the test device using a preconfigured " +
-                "Google account before beginning the test. Disabled by default."]
+            "Google account before beginning the test. Disabled by default."]
     )
     @set:JsonProperty("auto-google-login")
     var autoGoogleLogin: Boolean? by data
@@ -63,10 +66,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--use-orchestrator"],
         description = ["Whether each test runs in its own Instrumentation instance " +
-                "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " +
-                "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " +
-                "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " +
-                "information about Android Test Orchestrator."]
+            "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " +
+            "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " +
+            "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " +
+            "information about Android Test Orchestrator."]
     )
     @set:JsonProperty("use-orchestrator")
     var useOrchestrator: Boolean? by data
@@ -83,9 +86,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
         names = ["--environment-variables"],
         split = ",",
         description = ["A comma-separated, key=value map of environment variables " +
-                "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " +
-                "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " +
-                "passed to your test runner (typically AndroidJUnitRunner)"]
+            "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " +
+            "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " +
+            "passed to your test runner (typically AndroidJUnitRunner)" +
+            "If you want have downloaded coverage you need also set --directories-to-pull"]
     )
     @set:JsonProperty("environment-variables")
     var environmentVariables: Map<String, String>? by data
@@ -94,11 +98,11 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
         names = ["--directories-to-pull"],
         split = ",",
         description = ["A list of paths that will be copied from the device's " +
-                "storage to the designated results bucket after the test is complete. These must be absolute paths under " +
-                "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " +
-                "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " +
-                "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " +
-                "storage, the system will replace it with the external storage path prefix for that device."]
+            "storage to the designated results bucket after the test is complete. These must be absolute paths under " +
+            "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " +
+            "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " +
+            "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " +
+            "storage, the system will replace it with the external storage path prefix for that device."]
     )
     @set:JsonProperty("directories-to-pull")
     var directoriesToPull: List<String>? by data
@@ -107,8 +111,8 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
         names = ["--other-files"],
         split = ",",
         description = ["A list of device-path=file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push." +
-                "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." +
-                "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "]
+            "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." +
+            "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "]
     )
     @set:JsonProperty("other-files")
     var otherFiles: Map<String, String>? by data
@@ -116,7 +120,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--performance-metrics"],
         description = ["Monitor and record performance metrics: CPU, memory, " +
-                "network usage, and FPS (game-loop only). Disabled by default."]
+            "network usage, and FPS (game-loop only). Disabled by default."]
     )
     @set:JsonProperty("performance-metrics")
     var performanceMetrics: Boolean? by data
@@ -133,12 +137,12 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--num-uniform-shards"],
         description = ["Specifies the number of shards into which you want to evenly distribute test cases." +
-                "The shards are run in parallel on separate devices. For example," +
-                "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." +
-                "The number of shards should be less than the total number of test cases." +
-                "The number of shards specified must be >= 1 and <= 50." +
-                "This option cannot be used along max-test-shards and is not compatible with smart sharding." +
-                "If you want to take benefits of smart sharding use max-test-shards."]
+            "The shards are run in parallel on separate devices. For example," +
+            "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." +
+            "The number of shards should be less than the total number of test cases." +
+            "The number of shards specified must be >= 1 and <= 50." +
+            "This option cannot be used along max-test-shards and is not compatible with smart sharding." +
+            "If you want to take benefits of smart sharding use max-test-shards."]
     )
     @set:JsonProperty("num-uniform-shards")
     var numUniformShards: Int? by data
@@ -146,7 +150,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--test-runner-class"],
         description = ["The fully-qualified Java class name of the instrumentation test runner (default: the last name extracted " +
-                "from the APK manifest)."]
+            "from the APK manifest)."]
     )
     @set:JsonProperty("test-runner-class")
     var testRunnerClass: String? by data
@@ -155,10 +159,10 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
         names = ["--test-targets"],
         split = ",",
         description = ["A list of one or more test target filters to apply " +
-                "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " +
-                "or test annotation desired. Any test filter supported by am instrument -e … is supported. " +
-                "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " +
-                "information."]
+            "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " +
+            "or test annotation desired. Any test filter supported by am instrument -e … is supported. " +
+            "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " +
+            "information."]
     )
     @set:JsonProperty("test-targets")
     var testTargets: List<String?>? by data
@@ -195,23 +199,9 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.GCLOUD
 
-        override val keys = listOf(
-            "app",
-            "test",
-            "additional-apks",
-            "auto-google-login",
-            "use-orchestrator",
-            "environment-variables",
-            "directories-to-pull",
-            "other-files",
-            "performance-metrics",
-            "num-uniform-shards",
-            "test-runner-class",
-            "test-targets",
-            "robo-directives",
-            "robo-script",
-            "device"
-        )
+        override val keys by lazy {
+            AndroidGcloudConfig::class.ymlKeys
+        }
 
         fun default() = AndroidGcloudConfig().apply {
             app = null
diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt
index 6f46e0df8e..e64f140441 100644
--- a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt
+++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.ArgsHelper
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import ftl.config.FtlConstants
 import ftl.shard.DEFAULT_CLASS_TEST_TIME_SEC
@@ -85,6 +86,7 @@ data class CommonFlankConfig @JsonIgnore constructor(
         description = ["The Google Cloud Platform project name to use for this invocation. " +
             "If omitted, then the project from the service account credential is used"]
     )
+    @set:JsonProperty("project")
     var project: String? by data
 
     @set:CommandLine.Option(
@@ -165,28 +167,9 @@ data class CommonFlankConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.FLANK
 
-        override val keys = listOf(
-            "max-test-shards",
-            "shard-time",
-            "num-test-runs",
-            "smart-flank-gcs-path",
-            "smart-flank-disable-upload",
-            "disable-sharding",
-            "test-targets-always-run",
-            "files-to-download",
-            "project",
-            "run-timeout",
-            "legacy-junit-result",
-            "ignore-failed-tests",
-            "keep-file-path",
-            "output-style",
-            "disable-results-upload",
-            "full-junit-result",
-            "local-result-dir",
-            "default-test-time",
-            "default-class-test-time",
-            "local-result-dir"
-        )
+        override val keys by lazy {
+            CommonFlankConfig::class.ymlKeys
+        }
 
         const val defaultLocalResultsDir = "results"
 
diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt
index 9cfb573c51..77ee0526e5 100644
--- a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt
+++ b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.ArgsHelper
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import ftl.config.Device
 import ftl.config.FlankDefaults
@@ -70,12 +71,14 @@ data class CommonGcloudConfig @JsonIgnore constructor(
                 "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " +
                 "or s. If no unit is given, seconds are assumed. "]
     )
+    @set:JsonProperty("timeout")
     var timeout: String? by data
 
     @set:CommandLine.Option(
         names = ["--async"],
         description = ["Invoke a test asynchronously without waiting for test results."]
     )
+    @set:JsonProperty("async")
     var async: Boolean? by data
 
     @set:CommandLine.Option(
@@ -123,15 +126,9 @@ data class CommonGcloudConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.GCLOUD
 
-        override val keys = listOf(
-            "results-bucket",
-            "results-dir",
-            "record-video",
-            "timeout",
-            "async",
-            "results-history-name",
-            "num-flaky-test-attempts"
-        )
+        override val keys by lazy {
+            CommonGcloudConfig::class.ymlKeys
+        }
 
         fun default(android: Boolean) = CommonGcloudConfig().apply {
             ArgsHelper.yamlMapper.readerFor(CommonGcloudConfig::class.java)
diff --git a/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt
index 6272f434b0..70d1005a8d 100644
--- a/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt
+++ b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import picocli.CommandLine
 
@@ -22,18 +23,19 @@ data class IosGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--test"],
         description = ["The path to the test package (a zip file containing the iOS app " +
-                "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " +
-                "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " +
-                "is specified."]
+            "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " +
+            "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " +
+            "is specified."]
     )
+    @set:JsonProperty("test")
     var test: String? by data
 
     @set:CommandLine.Option(
         names = ["--xctestrun-file"],
         description = ["The path to an .xctestrun file that will override any " +
-                ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " +
-                "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " +
-                "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."]
+            ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " +
+            "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " +
+            "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."]
     )
     @set:JsonProperty("xctestrun-file")
     var xctestrunFile: String? by data
@@ -41,8 +43,8 @@ data class IosGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--xcode-version"],
         description = ["The version of Xcode that should be used to run an XCTest. " +
-                "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " +
-                "all iOS versions selected in the test matrix."]
+            "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " +
+            "all iOS versions selected in the test matrix."]
     )
     @set:JsonProperty("xcode-version")
     var xcodeVersion: String? by data
@@ -53,12 +55,9 @@ data class IosGcloudConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.GCLOUD
 
-        override val keys = listOf(
-            "test",
-            "xctestrun-file",
-            "xcode-version",
-            "device"
-        )
+        override val keys by lazy {
+            IosGcloudConfig::class.ymlKeys
+        }
 
         fun default() = IosGcloudConfig().apply {
             test = null
diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt
index 0c5e688bee..814e9161ba 100644
--- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt
+++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt
@@ -8,8 +8,6 @@ import ftl.args.ArgsHelper.createGcsBucket
 import ftl.args.ArgsHelper.validateTestMethods
 import ftl.args.yml.mergeYmlKeys
 import ftl.config.FtlConstants
-import ftl.config.common.CommonGcloudConfig
-import ftl.config.ios.IosGcloudConfig
 import ftl.gc.GcStorage
 import ftl.gc.GcStorage.exist
 import ftl.shard.TestMethod
@@ -20,10 +18,11 @@ import ftl.test.util.TestHelper.absolutePath
 import ftl.test.util.assertThrowsWithMessage
 import ftl.run.exception.FlankGeneralError
 import ftl.run.exception.FlankConfigurationError
+import io.mockk.mockk
+import io.mockk.unmockkAll
 import io.mockk.every
 import io.mockk.mockkObject
 import io.mockk.spyk
-import io.mockk.unmockkAll
 import org.junit.After
 import org.junit.Assume
 import org.junit.Rule
@@ -48,9 +47,16 @@ class ArgsHelperTest {
 
     @Test
     fun `mergeYmlMaps succeeds`() {
-        val merged = mergeYmlKeys(CommonGcloudConfig, IosGcloudConfig)
+        val merged = mergeYmlKeys(mockk() {
+            every { keys } returns listOf("devices", "test", "apk")
+            every { group } returns "gcloud"
+        }, mockk() {
+            every { keys } returns listOf("xcode-version", "async", "client-details")
+            every { group } returns "gcloud"
+        })
+
         assertThat(merged.keys.size).isEqualTo(1)
-        assertThat(merged["gcloud"]?.size).isEqualTo(11)
+        assertThat(merged["gcloud"]?.size).isEqualTo(6)
     }
 
     @Test