feat: ios scenario numbers

docs/index.md

@@ -160,6 +160,13 @@ gcloud:
   #   - gs://bucket/additional.ipa
   #   - path/to/local/ipa/file.ipa
 
+  ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios).
+  ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting.
+  # scenario-numbers:
+  #   - 1
+  #   - 2
+  #   - 3
+
   ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest
   # type: xctest
 
@@ -361,6 +368,13 @@ gcloud:
   #   - local/file/path/test1.obb
   #   - local/file/path/test2.obb
 
+  ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios).
+  ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting.
+  # scenario-numbers:
+  #   - 1
+  #   - 2
+  #   - 3
+
   ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. 
   ## [main|patch].0300110.com.example.android.obb which will be installed into <shared-storage>/Android/obb/<package-name>/ on the device.
   # obb-names:

test_runner/flank.ios.yml

@@ -91,6 +91,13 @@ gcloud:
   #   - gs://bucket/additional.ipa
   #   - path/to/local/ipa/file.ipa
 
+  ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios).
+  ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting.
+  # scenario-numbers:
+  #   - 1
+  #   - 2
+  #   - 3
+
   ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest
   # type: xctest
 

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -15,7 +15,6 @@ data class AndroidArgs(
     val environmentVariables: Map<String, String>, // should not be printed, becuase could contains sensitive informations
     val directoriesToPull: List<String>,
     val grantPermissions: String?,
-    val scenarioNumbers: List<String>,
     val scenarioLabels: List<String>,
     val obbFiles: List<String>,
     val obbNames: List<String>,

test_runner/src/main/kotlin/ftl/args/CommonArgs.kt

@@ -20,6 +20,7 @@ data class CommonArgs(
     override val networkProfile: String?,
     override val otherFiles: Map<String, String>,
     override val type: Type?,
+    override val scenarioNumbers: List<String>,
 
     // flank
     override val project: String,

test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt

@@ -41,6 +41,5 @@ fun createAndroidArgs(
     obfuscateDumpShards = obfuscate,
     obbFiles = gcloud.obbfiles!!,
     obbNames = gcloud.obbnames!!,
-    scenarioNumbers = gcloud.scenarioNumbers!!,
     grantPermissions = gcloud.grantPermissions
 )

test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt

@@ -26,6 +26,7 @@ fun CommonConfig.createCommonArgs(
     networkProfile = gcloud.networkProfile,
     clientDetails = gcloud.clientDetails,
     otherFiles = gcloud.otherFiles!!.mapValues { (_, path) -> path.normalizeFilePath() },
+    scenarioNumbers = gcloud.scenarioNumbers!!,
     type = gcloud.type?.toType(),
 
     // flank

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -24,6 +24,7 @@ interface IArgs {
     val resultsHistoryName: String?
     val flakyTestAttempts: Int
     val otherFiles: Map<String, String>
+    val scenarioNumbers: List<String>
     val type: Type? get() = null
 
     // FlankYml

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -42,6 +42,7 @@ IosArgs
       num-flaky-test-attempts: $flakyTestAttempts
       other-files: ${ArgsToString.mapToString(otherFiles)}
       additional-ipas: ${ArgsToString.listToString(additionalIpas)}
+      scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)}
       type: ${type?.ymlName}
 
     flank:

test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt

@@ -15,6 +15,13 @@ fun IosArgs.validate() = apply {
     checkResultsDirUnique()
     assertAdditionalIpas()
     validType()
+    assertGameloop()
+}
+
+fun IosArgs.assertGameloop() {
+    if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP))
+        throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.")
+    scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") }
 }
 
 fun IosArgs.validateRefresh() = apply {

test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt

@@ -115,16 +115,6 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:JsonProperty("directories-to-pull")
     var directoriesToPull: List<String>? by data
 
-    @set:CommandLine.Option(
-        names = ["--scenario-numbers"],
-        split = ",",
-        description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " +
-                "A maximum of 1024 scenarios may be specified in one test matrix, " +
-                "but the maximum number may also be limited by the overall test --timeout setting."]
-    )
-    @set:JsonProperty("scenario-numbers")
-    var scenarioNumbers: List<String>? by data
-
     @set:CommandLine.Option(
         names = ["--scenario-labels"],
         split = ",",
@@ -250,7 +240,6 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
             environmentVariables = emptyMap()
             grantPermissions = FlankDefaults.GRANT_PERMISSIONS_ALL
             directoriesToPull = emptyList()
-            scenarioNumbers = emptyList()
             scenarioLabels = emptyList()
             obbfiles = emptyList()
             obbnames = emptyList()

test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt

@@ -130,6 +130,16 @@ data class CommonGcloudConfig @JsonIgnore constructor(
     @set:JsonProperty("other-files")
     var otherFiles: Map<String, String>? by data
 
+    @set:CommandLine.Option(
+        names = ["--scenario-numbers"],
+        split = ",",
+        description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " +
+                "A maximum of 1024 scenarios may be specified in one test matrix, " +
+                "but the maximum number may also be limited by the overall test --timeout setting."]
+    )
+    @set:JsonProperty("scenario-numbers")
+    var scenarioNumbers: List<String>? by data
+
     @set:CommandLine.Option(
         names = ["--type"],
         description = ["The type of test to run. TYPE must be one of: instrumentation, robo, xctest, game-loop."]
@@ -162,6 +172,7 @@ data class CommonGcloudConfig @JsonIgnore constructor(
             devices = listOf(defaultDevice(android))
             otherFiles = emptyMap()
             type = null
+            scenarioNumbers = emptyList()
         }
     }
 }

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -241,6 +241,7 @@ IosArgs
       additional-ipas: 
         - $testIpa1
         - $testIpa2
+      scenario-numbers: 
       type: xctest
 
     flank:
@@ -301,6 +302,7 @@ IosArgs
       num-flaky-test-attempts: 0
       other-files: 
       additional-ipas: 
+      scenario-numbers: 
       type: xctest
 
     flank:
@@ -1127,6 +1129,62 @@ IosArgs
             assertFalse(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled"))
         }
     }
+
+    @Test
+    fun `should not throw exception if game-loop is provided and nothing else`() {
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+          results-dir: test
+          type: game-loop
+        """.trimIndent()
+        IosArgs.load(yaml).validate()
+    }
+
+    @Test(expected = FlankConfigurationError::class)
+    fun `should throw exception if game-loop is not provided and scenario numbers are`() {
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+          results-dir: test
+          scenario-numbers:
+             - 1
+             - 2
+        """.trimIndent()
+        IosArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should not throw exception if game-loop is provided and scenario numbers are`() {
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+          results-dir: test
+          type: game-loop
+          scenario-numbers:
+             - 1
+             - 2
+        """.trimIndent()
+        IosArgs.load(yaml).validate()
+    }
+
+    @Test(expected = FlankConfigurationError::class)
+    fun `should throw exception if invalid scenario numbers are provided`() {
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+          results-dir: test
+          type: game-loop
+          scenario-numbers:
+             - error1
+             - error2
+        """.trimIndent()
+        IosArgs.load(yaml).validate()
+    }
 }
 
 private fun IosArgs.Companion.load(yamlData: String, cli: IosRunCommand? = null): IosArgs =