feat: Added scenario-numbers (#1244)

* Added scenario-numbers

* Fix merge bugs and tests

* Added tests

* Fixed invalid test

test_runner/flank.yml

@@ -96,6 +96,13 @@ gcloud:
   #   - /sdcard/dir1/file1.txt: local/file.txt
   #   - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg
 
+  ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios).
+  ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting.
+  # scenario-numbers:
+  #   - 1
+  #   - 2
+  #   - 3
+
   ## A list of game-loop scenario labels (default: None). Each game-loop scenario may be labeled in the APK manifest file with one or more arbitrary strings, creating logical groupings (e.g. GPU_COMPATIBILITY_TESTS).
   ## If --scenario-numbers and --scenario-labels are specified together, Firebase Test Lab will first execute each scenario from --scenario-numbers.
   ## It will then expand each given scenario label into a list of scenario numbers marked with that label, and execute those scenarios.

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -16,6 +16,7 @@ data class AndroidArgs(
     val directoriesToPull: List<String>,
     val grantPermissions: String?,
     val type: Type?,
+    val scenarioNumbers: List<String>,
     val otherFiles: Map<String, String>,
     val scenarioLabels: List<String>,
     val performanceMetrics: Boolean,
@@ -50,6 +51,7 @@ AndroidArgs
       grant-permissions: $grantPermissions
       type: ${type?.ymlName}
       other-files: ${ArgsToString.mapToString(otherFiles)}
+      scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)}
       scenario-labels: ${ArgsToString.listToString(scenarioLabels)}
       performance-metrics: $performanceMetrics
       num-uniform-shards: $numUniformShards

test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt

@@ -41,6 +41,7 @@ fun createAndroidArgs(
     useLegacyJUnitResult = flank.useLegacyJUnitResult!!,
     scenarioLabels = gcloud.scenarioLabels!!,
     obfuscateDumpShards = obfuscate,
+    scenarioNumbers = gcloud.scenarioNumbers!!,
     grantPermissions = gcloud.grantPermissions,
     type = gcloud.type?.let { Type.fromString(it) }
 )

test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt

@@ -33,8 +33,15 @@ fun AndroidArgs.validate() = apply {
 }
 
 private fun AndroidArgs.assertGameLoop() {
+    assertLabelContent()
+}
+
+private fun AndroidArgs.assertLabelContent() {
     if (scenarioLabels.isNotEmpty() && (type == null || type != Type.GAMELOOP))
         throw FlankConfigurationError("Scenario labels defined but Type is not Game-loop.")
+    if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP))
+        throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.")
+    scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") }
 }
 
 private fun AndroidArgs.assertType() = type?.let {

test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt

@@ -132,6 +132,16 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:JsonProperty("other-files")
     var otherFiles: Map<String, String>? by data
 
+    @set:CommandLine.Option(
+        names = ["--scenario-numbers"],
+        split = ",",
+        description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " +
+                "A maximum of 1024 scenarios may be specified in one test matrix, " +
+                "but the maximum number may also be limited by the overall test --timeout setting."]
+    )
+    @set:JsonProperty("scenario-numbers")
+    var scenarioNumbers: List<String>? by data
+
     @set:CommandLine.Option(
         names = ["--scenario-labels"],
         split = ",",
@@ -239,6 +249,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
             grantPermissions = FlankDefaults.GRANT_PERMISSIONS_ALL
             directoriesToPull = emptyList()
             otherFiles = emptyMap()
+            scenarioNumbers = emptyList()
             scenarioLabels = emptyList()
             performanceMetrics = FlankDefaults.DISABLE_PERFORMANCE_METRICS
             numUniformShards = null

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -320,6 +320,7 @@ AndroidArgs
       other-files: 
         /sdcard/dir1/file1.txt: $appApkAbsolutePath
         /sdcard/dir2/file2.jpg: $testApkAbsolutePath
+      scenario-numbers: 
       scenario-labels: 
       performance-metrics: false
       num-uniform-shards: null
@@ -398,6 +399,7 @@ AndroidArgs
       grant-permissions: all
       type: null
       other-files: 
+      scenario-numbers: 
       scenario-labels: 
       performance-metrics: false
       num-uniform-shards: null
@@ -2021,7 +2023,8 @@ AndroidArgs
         AndroidArgs.load(yaml).validate()
     }
 
-    fun `should not throw exception if game-loop required but no scenario loop provided`() {
+    @Test
+    fun `should not throw exception if game-loop and scenario labels provided`() {
         val yaml = """
         gcloud:
           app: $appApk
@@ -2038,6 +2041,104 @@ AndroidArgs
         """.trimIndent()
         AndroidArgs.load(yaml).validate()
     }
+
+    @Test
+    fun `should not throw exception if game-loop and scenario numbers provided`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: game-loop
+          scenario-numbers: 
+            - 1
+            - 2
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test(expected = FlankConfigurationError::class)
+    fun `should throw exception if game-loop not provided and scenario numbers provided`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: robo
+          scenario-numbers: 
+            - 1 
+            - 2
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test(expected = FlankConfigurationError::class)
+    fun `should throw exception if invalid scenario numbers provided`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: game-loop
+          scenario-numbers: 
+            - error
+            - 2
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should Not throw exception if valid scenario numbers provided`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: game-loop
+          scenario-numbers: 
+            - 1
+            - 2
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should Not throw exception if valid scenario numbers provided and scenario labels`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: game-loop
+          scenario-labels:
+            - label1
+            - label2
+          scenario-numbers: 
+            - 1
+            - 2
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
 }
 
 private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs =