From 53bbc44b7bf831c9f27f6918a39b7b71e989fa2d Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Fri, 16 Oct 2020 10:12:29 +0200 Subject: [PATCH 1/4] Added scenario-numbers --- README.md | 7 +++++++ test_runner/flank.yml | 7 +++++++ test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 2 ++ .../src/main/kotlin/ftl/args/CreateAndroidArgs.kt | 1 + .../kotlin/ftl/config/android/AndroidGcloudConfig.kt | 11 +++++++++++ .../src/test/kotlin/ftl/args/AndroidArgsTest.kt | 2 ++ 6 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 693cf4de56..bfb0648b9a 100644 --- a/README.md +++ b/README.md @@ -337,6 +337,13 @@ gcloud: # other-files # - /sdcard/dir1/file1.txt: local/file.txt # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + + ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). + ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. + # scenario-numbers: + # - 1 + # - 2 + # - 3 ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Disabled by default. Use --performance-metrics to enable. diff --git a/test_runner/flank.yml b/test_runner/flank.yml index a493e54154..621ebdc213 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -96,6 +96,13 @@ gcloud: # - /sdcard/dir1/file1.txt: local/file.txt # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). + ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. + # scenario-numbers: + # - 1 + # - 2 + # - 3 + ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Disabled by default. Use --performance-metrics to enable. # performance-metrics: true diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 3ae8af0843..4ecae254af 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -16,6 +16,7 @@ data class AndroidArgs( val directoriesToPull: List, val grantPermissions: String?, val type: Type?, + val scenarioNumbers: List, val otherFiles: Map, val performanceMetrics: Boolean, val numUniformShards: Int?, @@ -49,6 +50,7 @@ AndroidArgs grant-permissions: $grantPermissions type: ${type?.ymlName} other-files: ${ArgsToString.mapToString(otherFiles)} + scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)} performance-metrics: $performanceMetrics num-uniform-shards: $numUniformShards test-runner-class: $testRunnerClass diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 67a1cdd7f7..649ba53e31 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -40,6 +40,7 @@ fun createAndroidArgs( } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!!, obfuscateDumpShards = obfuscate, + scenarioNumbers = gcloud.scenarioNumbers!!, grantPermissions = gcloud.grantPermissions, type = gcloud.type?.let { Type.fromString(it) } ) diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 4359b1b03b..464221a93d 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -132,6 +132,16 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:JsonProperty("other-files") var otherFiles: Map? by data + @set:CommandLine.Option( + names = ["--scenario-numbers"], + split = ",", + description = ["A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). " + + "A maximum of 1024 scenarios may be specified in one test matrix, " + + "but the maximum number may also be limited by the overall test --timeout setting."] + ) + @set:JsonProperty("scenario-labels") + var scenarioNumbers: List? by data + @set:CommandLine.Option( names = ["--performance-metrics"], description = ["Monitor and record performance metrics: CPU, memory, " + @@ -228,6 +238,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( grantPermissions = FlankDefaults.GRANT_PERMISSIONS_ALL directoriesToPull = emptyList() otherFiles = emptyMap() + scenarioNumbers = emptyList() performanceMetrics = FlankDefaults.DISABLE_PERFORMANCE_METRICS numUniformShards = null testRunnerClass = null diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 249545ce28..95477beb38 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -320,6 +320,7 @@ AndroidArgs other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath + scenario-numbers: performance-metrics: false num-uniform-shards: null test-runner-class: com.foo.TestRunner @@ -397,6 +398,7 @@ AndroidArgs grant-permissions: all type: null other-files: + scenario-numbers: performance-metrics: false num-uniform-shards: null test-runner-class: null From fbd45520a8dbff3e76bd3b2d2c0ca39652885efc Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 19 Oct 2020 10:02:12 +0200 Subject: [PATCH 2/4] Fix merge bugs and tests --- .../src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt | 2 +- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 507ee50f85..21ab8cb532 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -139,7 +139,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( "A maximum of 1024 scenarios may be specified in one test matrix, " + "but the maximum number may also be limited by the overall test --timeout setting."] ) - @set:JsonProperty("scenario-labels") + @set:JsonProperty("scenario-numbers") var scenarioNumbers: List? by data @set:CommandLine.Option( diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 5aa75ba15a..ab4bd06078 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -320,8 +320,8 @@ AndroidArgs other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath - scenario-labels: scenario-numbers: + scenario-labels: performance-metrics: false num-uniform-shards: null test-runner-class: com.foo.TestRunner From 8e6c1076a6c70c37e3f6d5de48899762c78e7633 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 19 Oct 2020 11:43:35 +0200 Subject: [PATCH 3/4] Added tests --- .../kotlin/ftl/args/ValidateAndroidArgs.kt | 7 ++ .../test/kotlin/ftl/args/AndroidArgsTest.kt | 101 +++++++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 7cf2c790c5..30f1c2c724 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -33,8 +33,15 @@ fun AndroidArgs.validate() = apply { } private fun AndroidArgs.assertGameLoop() { + assertLabelContent() +} + +private fun AndroidArgs.assertLabelContent() { if (scenarioLabels.isNotEmpty() && (type == null || type != Type.GAMELOOP)) throw FlankConfigurationError("Scenario labels defined but Type is not Game-loop.") + if (scenarioNumbers.isNotEmpty() && (type == null || type != Type.GAMELOOP)) + throw FlankConfigurationError("Scenario numbers defined but Type is not Game-loop.") + scenarioNumbers.forEach { it.toIntOrNull() ?: throw FlankConfigurationError("Invalid scenario number provided - $it") } } private fun AndroidArgs.assertType() = type?.let { diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index ab4bd06078..ebffb43b66 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -2023,7 +2023,8 @@ AndroidArgs AndroidArgs.load(yaml).validate() } - fun `should not throw exception if game-loop required but no scenario loop provided`() { + @Test + fun `should not throw exception if game-loop and scenario labels provided`() { val yaml = """ gcloud: app: $appApk @@ -2040,6 +2041,104 @@ AndroidArgs """.trimIndent() AndroidArgs.load(yaml).validate() } + + @Test + fun `should not throw exception if game-loop and scenario numbers provided`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: game-loop + scenario-numbers: + - 1 + - 2 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if game-loop not provided and scenario numbers provided`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: robo + scenario-numbers: + - 1 + - 2 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if invalid scenario numbers provided`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: robo + scenario-numbers: + - error + - 2 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if valid scenario numbers provided`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: game-loop + scenario-numbers: + - 1 + - 2 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if valid scenario numbers provided and scenario labels`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: game-loop + scenario-labels: + - label1 + - label2 + scenario-numbers: + - 1 + - 2 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = From 7be223f03ad32be373612f1548345b825a275907 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 19 Oct 2020 11:45:10 +0200 Subject: [PATCH 4/4] Fixed invalid test --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index ebffb43b66..16fbaae6ba 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -2091,7 +2091,7 @@ AndroidArgs version: 25 locale: en orientation: portrait - type: robo + type: game-loop scenario-numbers: - error - 2