docs: Investigate flank options (#1131)

docs/investigate_flank_options.md

@@ -0,0 +1,125 @@
+# Investigate flank options
+
+## List of options android
+
+### gcloud
+
+1. app
+1. test
+1. additional-apks
+1. auto-google-login
+1. no-auto-google-login
+1. use-orchestrator
+1. no-use-orchestrator
+1. environment-variables
+1. directories-to-pull
+1. other-files
+1. performance-metrics
+1. no-performance-metrics
+1. num-uniform-shards
+1. test-runner-class
+1. test-targets
+1. robo-directives
+1. robo-script
+1. results-bucket
+1. results-dir
+1. record-video
+1. no-record-video
+1. timeout
+1. async
+1. client-details
+1. network-profile
+1. results-history-name
+1. num-flaky-test-attempts
+1. device
+
+### flank
+
+1. additional-app-test-apks
+1. legacy-junit-result
+1. max-test-shards
+1. shard-time
+1. num-test-runs
+1. smart-flank-gcs-path
+1. smart-flank-disable-upload
+1. disable-sharding
+1. test-targets-always-run
+1. files-to-download
+1. project
+1. local-result-dir
+1. run-timeout
+1. full-junit-result
+1. ignore-failed-tests
+1. keep-file-path
+1. output-style
+1. disable-results-upload
+1. default-test-time
+1. default-class-test-time
+1. use-average-test-time-for-new-tests
+
+## List of options ios
+
+### gcloud
+
+1. test
+1. xctestrun-file
+1. xcode-version
+1. results-bucket
+1. results-dir
+1. record-video
+1. no-record-video
+1. timeout
+1. async
+1. client-details
+1. network-profile
+1. results-history-name
+1. num-flaky-test-attempts
+1. device
+
+### flank
+
+1. test-targets
+1. max-test-shards
+1. shard-time
+1. num-test-runs
+1. smart-flank-gcs-path
+1. smart-flank-disable-upload
+1. disable-sharding
+1. test-targets-always-run
+1. files-to-download
+1. project
+1. local-result-dir
+1. run-timeout
+1. full-junit-result
+1. ignore-failed-tests
+1. keep-file-path
+1. output-style
+1. disable-results-upload
+1. default-test-time
+1. default-class-test-time
+1. use-average-test-time-for-new-tests
+
+## Investigation report
+
+### environment-variables (Android)
+
+Set the ```directories-to-pull``` variable to pull from the device directory with coverage report.
+There will be no warnings or failure messages when ```environment-variables``` is set without ```directories-to-pull```
+A warning has been added about this.
+
+### files-to-download (Android)
+
+In the case where coverage reports need to be downloaded set the ```directories-to-pull``` variable.
+There will be no warnings or failures when ```files-to-download``` is set without ```directories-to-pull```.
+A warning is added regarding this.
+
+### disable-sharding (Common)
+
+Can be enabled by setting ```max-test-shards``` to greater than one. In this case flank will disable sharding
+A warning is added regarding this.
+
+### num-uniform-shards (Android)
+
+1. When set with ```max-test-shards``` Flank will fail fast.
+1. When set with ```disable-sharding```, Flank will disable sharding without any warning
+   - Warning added about this.

test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt

@@ -22,6 +22,9 @@ fun AndroidArgs.validate() = apply {
     assertTestFiles()
     assertOtherFiles()
     checkResultsDirUnique()
+    checkEnvironmentVariables()
+    checkFilesToDownload()
+    checkNumUniformShards()
 }
 
 private fun AndroidArgs.assertDevicesSupported() = devices
@@ -138,3 +141,18 @@ private fun AndroidArgs.assertRoboTest() {
 private fun AndroidArgs.assertOtherFiles() {
     otherFiles.forEach { (_, path) -> ArgsHelper.assertFileExists(path, "from otherFiles") }
 }
+
+private fun AndroidArgs.checkEnvironmentVariables() {
+    if (environmentVariables.isNotEmpty() && directoriesToPull.isEmpty())
+        println("WARNING: environment-variables set but directories-to-pull is empty, this will result in the coverage file  not downloading to the bucket.")
+}
+
+private fun AndroidArgs.checkFilesToDownload() {
+    if (filesToDownload.isNotEmpty() && directoriesToPull.isEmpty())
+        println("WARNING: files-to-download is set but directories-to-pull is empty, the coverage file may fail to download into the bucket.")
+}
+
+private fun AndroidArgs.checkNumUniformShards() {
+    if ((numUniformShards ?: 0) > 0 && disableSharding)
+        println("WARNING: disable-sharding is enabled with num-uniform-shards = $numUniformShards, Flank will ignore num-uniform-shards and disable sharding.")
+}

test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt

@@ -14,6 +14,7 @@ fun CommonArgs.validate() {
     assertRepeatTests()
     assertSmartFlankGcsPath()
     assertOrientationCorrectness()
+    checkDisableSharding()
 }
 
 private fun List<Device>.devicesWithMispeltOrientations(availableOrientations: List<String>) =
@@ -29,8 +30,8 @@ private fun List<Device>.throwIfAnyMisspelt() =
 private fun CommonArgs.assertProjectId() {
     if (project.isBlank()) throw FlankConfigurationError(
         "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" +
-                "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" +
-                " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
+            "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" +
+            " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id"
     )
 }
 
@@ -72,3 +73,8 @@ fun IArgs.checkResultsDirUnique() {
     if (useLegacyJUnitResult && GcStorage.exist(resultsBucket, resultsDir))
         println("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled\n")
 }
+
+fun IArgs.checkDisableSharding() {
+    if (disableSharding && maxTestShards > 0)
+        println("WARNING: disable-sharding enabled with max-test-shards = $maxTestShards, Flank will ignore max-test-shard and disable sharding.")
+}

test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt

@@ -1,5 +1,11 @@
 package ftl.args.yml
 
+import com.fasterxml.jackson.annotation.JsonProperty
+import kotlin.reflect.KClass
+import kotlin.reflect.KMutableProperty
+import kotlin.reflect.full.findAnnotation
+import kotlin.reflect.full.memberProperties
+
 interface IYmlKeys {
     val group: String
     val keys: List<String>
@@ -10,6 +16,12 @@ interface IYmlKeys {
     }
 }
 
+val KClass<*>.ymlKeys
+    get() = memberProperties
+            .filterIsInstance<KMutableProperty<*>>()
+            .mapNotNull { it.setter.findAnnotation<JsonProperty>() }
+            .map { it.value }
+
 fun mergeYmlKeys(
     vararg keys: IYmlKeys
 ): Map<String, List<String>> = keys

test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.args.yml.AppTestPair
 import ftl.args.yml.IYmlKeys
+import ftl.args.yml.ymlKeys
 import ftl.config.Config
 import picocli.CommandLine
 
@@ -19,7 +20,7 @@ data class AndroidFlankConfig @JsonIgnore constructor(
         names = ["--additional-app-test-apks"],
         split = ",",
         description = ["A list of app & test apks to include in the run. " +
-                "Useful for running multiple module tests within a single Flank run."]
+            "Useful for running multiple module tests within a single Flank run."]
     )
     fun additionalAppTestApks(map: Map<String, String>?) {
         if (map.isNullOrEmpty()) return
@@ -54,10 +55,9 @@ data class AndroidFlankConfig @JsonIgnore constructor(
 
         override val group = IYmlKeys.Group.FLANK
 
-        override val keys = listOf(
-            "additional-app-test-apks",
-            "legacy-junit-result"
-        )
+        override val keys by lazy {
+            AndroidFlankConfig::class.ymlKeys
+        }
 
         fun default() = AndroidFlankConfig().apply {
             additionalAppTestApks = null