From 7b0dd1cb6536216fb1ed5cee88601ad935f69dca Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Thu, 29 Oct 2020 09:53:38 +0100 Subject: [PATCH 1/4] added missing documentation --- docs/index.md | 14 ++++++++++++++ test_runner/flank.ios.yml | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/docs/index.md b/docs/index.md index 1651a146bc..8e1c1371c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -160,6 +160,13 @@ gcloud: # - gs://bucket/additional.ipa # - path/to/local/ipa/file.ipa + ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). + ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. + # scenario-numbers: + # - 1 + # - 2 + # - 3 + flank: # -- FlankYml -- @@ -358,6 +365,13 @@ gcloud: # - local/file/path/test1.obb # - local/file/path/test2.obb + ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). + ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. + # scenario-numbers: + # - 1 + # - 2 + # - 3 + ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. ## [main|patch].0300110.com.example.android.obb which will be installed into /Android/obb// on the device. # obb-names: diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 5ef01e11bf..c9b51d4030 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -91,6 +91,13 @@ gcloud: # - gs://bucket/additional.ipa # - path/to/local/ipa/file.ipa + ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). + ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. + # scenario-numbers: + # - 1 + # - 2 + # - 3 + flank: # -- FlankYml -- From d8603d038e5c8259fe2b495cc43e56522c7ead11 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Thu, 29 Oct 2020 10:27:00 +0100 Subject: [PATCH 2/4] added scenario-numbers to common flank config --- test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 1 - test_runner/src/main/kotlin/ftl/args/CommonArgs.kt | 1 + .../src/main/kotlin/ftl/args/CreateAndroidArgs.kt | 1 - .../src/main/kotlin/ftl/args/CreateCommonArgs.kt | 1 + test_runner/src/main/kotlin/ftl/args/IArgs.kt | 1 + test_runner/src/main/kotlin/ftl/args/IosArgs.kt | 1 + .../src/main/kotlin/ftl/args/ValidateIosArgs.kt | 7 +++++++ .../kotlin/ftl/config/android/AndroidGcloudConfig.kt | 11 ----------- .../kotlin/ftl/config/common/CommonGcloudConfig.kt | 11 +++++++++++ test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt | 2 ++ 10 files changed, 24 insertions(+), 13 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index e2251596ec..b6588afa15 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -16,7 +16,6 @@ data class AndroidArgs( val directoriesToPull: List, val grantPermissions: String?, val type: Type?, - val scenarioNumbers: List, val scenarioLabels: List, val obbFiles: List, val obbNames: List, diff --git a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt index e18699d1cd..25559e912d 100644 --- a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt @@ -18,6 +18,7 @@ data class CommonArgs( override val clientDetails: Map?, override val networkProfile: String?, override val otherFiles: Map, + override val scenarioNumbers: List, // flank override val project: String, diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 9d9d0d23b6..27dbbbf7ed 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -42,7 +42,6 @@ fun createAndroidArgs( obfuscateDumpShards = obfuscate, obbFiles = gcloud.obbfiles!!, obbNames = gcloud.obbnames!!, - scenarioNumbers = gcloud.scenarioNumbers!!, grantPermissions = gcloud.grantPermissions, type = gcloud.type?.let { Type.fromString(it) } ) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt index 725717a980..ce0fff42b8 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt @@ -25,6 +25,7 @@ fun CommonConfig.createCommonArgs( networkProfile = gcloud.networkProfile, clientDetails = gcloud.clientDetails, otherFiles = gcloud.otherFiles!!.mapValues { (_, path) -> path.normalizeFilePath() }, + scenarioNumbers = gcloud.scenarioNumbers!!, // flank maxTestShards = flank.maxTestShards!!, diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 8f872613e5..71b8d60cf3 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -23,6 +23,7 @@ interface IArgs { val resultsHistoryName: String? val flakyTestAttempts: Int val otherFiles: Map + val scenarioNumbers: List // FlankYml val maxTestShards: Int diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 19eb42a5ef..5bdcf39b88 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -43,6 +43,7 @@ IosArgs num-flaky-test-attempts: $flakyTestAttempts other-files: ${ArgsToString.mapToString(otherFiles)} additional-ipas: ${ArgsToString.listToString(additionalIpas)} + scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)} flank: max-test-shards: $maxTestShards diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt index 6eabf1a4ea..aa4cc6c905 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -13,6 +13,13 @@ fun IosArgs.validate() = apply { assertTestFiles() checkResultsDirUnique() assertAdditionalIpas() + assertGameloop() +} + +fun IosArgs.assertGameloop() { + /*if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP)) + throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.")*/ + scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") } } fun IosArgs.validateRefresh() = apply { diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 122bab2199..87e5615ff5 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -122,16 +122,6 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:JsonProperty("directories-to-pull") var directoriesToPull: List? by data - @set:CommandLine.Option( - names = ["--scenario-numbers"], - split = ",", - description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " + - "A maximum of 1024 scenarios may be specified in one test matrix, " + - "but the maximum number may also be limited by the overall test --timeout setting."] - ) - @set:JsonProperty("scenario-numbers") - var scenarioNumbers: List? by data - @set:CommandLine.Option( names = ["--scenario-labels"], split = ",", @@ -257,7 +247,6 @@ data class AndroidGcloudConfig @JsonIgnore constructor( environmentVariables = emptyMap() grantPermissions = FlankDefaults.GRANT_PERMISSIONS_ALL directoriesToPull = emptyList() - scenarioNumbers = emptyList() scenarioLabels = emptyList() obbfiles = emptyList() obbnames = emptyList() diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt index cdfee248e5..2e1c91ebf0 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt @@ -130,6 +130,16 @@ data class CommonGcloudConfig @JsonIgnore constructor( @set:JsonProperty("other-files") var otherFiles: Map? by data + @set:CommandLine.Option( + names = ["--scenario-numbers"], + split = ",", + description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " + + "A maximum of 1024 scenarios may be specified in one test matrix, " + + "but the maximum number may also be limited by the overall test --timeout setting."] + ) + @set:JsonProperty("scenario-numbers") + var scenarioNumbers: List? by data + constructor() : this(mutableMapOf().withDefault { null }) companion object : IYmlKeys { @@ -154,6 +164,7 @@ data class CommonGcloudConfig @JsonIgnore constructor( networkProfile = null devices = listOf(defaultDevice(android)) otherFiles = emptyMap() + scenarioNumbers = emptyList() } } } diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 75cb47352b..28564a267c 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -240,6 +240,7 @@ IosArgs additional-ipas: - $testIpa1 - $testIpa2 + scenario-numbers: flank: max-test-shards: 7 @@ -299,6 +300,7 @@ IosArgs num-flaky-test-attempts: 0 other-files: additional-ipas: + scenario-numbers: flank: max-test-shards: 1 From e6bd10de8976ebf2a6a454988585135740929420 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Fri, 30 Oct 2020 09:37:54 +0100 Subject: [PATCH 3/4] Added and fixed tests --- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 2 - .../src/test/kotlin/ftl/args/IosArgsTest.kt | 58 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index e2251596ec..42c76f6c70 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -15,8 +15,6 @@ data class AndroidArgs( val environmentVariables: Map, // should not be printed, becuase could contains sensitive informations val directoriesToPull: List, val grantPermissions: String?, - val type: Type?, - val scenarioNumbers: List, val scenarioLabels: List, val obbFiles: List, val obbNames: List, diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 0f9a8b283d..e9460a3ec3 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -241,8 +241,8 @@ IosArgs additional-ipas: - $testIpa1 - $testIpa2 - type: xctest scenario-numbers: + type: xctest flank: max-test-shards: 7 @@ -1129,6 +1129,62 @@ IosArgs assertFalse(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) } } + + @Test + fun `should not throw exception if game-loop is provided and nothing else`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + type: game-loop + """.trimIndent() + IosArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if game-loop is not provided and scenario numbers are`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + scenario-numbers: + - 1 + - 2 + """.trimIndent() + IosArgs.load(yaml).validate() + } + + @Test + fun `should not throw exception if game-loop is provided and scenario numbers are`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + type: game-loop + scenario-numbers: + - 1 + - 2 + """.trimIndent() + IosArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if invalid scenario numbers are provided`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + type: game-loop + scenario-numbers: + - error1 + - error2 + """.trimIndent() + IosArgs.load(yaml).validate() + } } private fun IosArgs.Companion.load(yamlData: String, cli: IosRunCommand? = null): IosArgs = From 6a78749ce444175e8196099c09776385c534fde1 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Fri, 30 Oct 2020 13:21:16 +0100 Subject: [PATCH 4/4] Fix PR comments and check if scenario numbers is less than 1024 --- test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 15e4f05d4a..ca974c8b22 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -56,9 +56,10 @@ private fun AndroidArgs.assertLabelContent() { else -> obbFiles.forEach { ArgsHelper.assertFileExists(it, " (obb file)") } } - if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP)) + if (scenarioNumbers.isNotEmpty() && (type != Type.GAMELOOP)) throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.") scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") } + if (scenarioNumbers.size > 1024) throw FlankConfigurationError("There cannot be more than 1024 Scenario numbers") } private const val MAX_OBB_FILES = 2 diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt index 4d577e3e11..bc0bb848a0 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -19,9 +19,10 @@ fun IosArgs.validate() = apply { } fun IosArgs.assertGameloop() { - if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP)) + if (scenarioNumbers.isNotEmpty() && (type != Type.GAMELOOP)) throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.") scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") } + if (scenarioNumbers.size > 1024) throw FlankConfigurationError("There cannot be more than 1024 Scenario numbers") } fun IosArgs.validateRefresh() = apply {