From f568b05d433be42f83ce24760285507362f5cdb8 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Mon, 12 Oct 2020 10:52:40 +0200 Subject: [PATCH 1/7] Add type --- README.md | 3 +++ test_runner/flank.yml | 3 +++ test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 4 ++++ .../src/main/kotlin/ftl/args/CreateAndroidArgs.kt | 4 +++- test_runner/src/main/kotlin/ftl/args/yml/Type.kt | 5 +++++ .../kotlin/ftl/config/android/AndroidGcloudConfig.kt | 10 +++++++++- .../src/test/kotlin/ftl/args/AndroidArgsTest.kt | 3 +++ 7 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/Type.kt diff --git a/README.md b/README.md index 1d7c9ed063..94397d6f4d 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,9 @@ gcloud: ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all + ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. + # type: instrumentation + ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 74adc2ac6e..37efe043f5 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -82,6 +82,9 @@ gcloud: # directories-to-pull: # - /sdcard/ + ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. + # type: instrumentation + ## Whether to grant runtime permissions on the device before the test begins. ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 1e877711d4..7474b62d31 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -1,6 +1,7 @@ package ftl.args import ftl.args.yml.AppTestPair +import ftl.args.yml.Type data class AndroidArgs( val commonArgs: CommonArgs, @@ -14,6 +15,7 @@ data class AndroidArgs( val environmentVariables: Map, // should not be printed, becuase could contains sensitive informations val directoriesToPull: List, val grantPermissions: String?, + val type: Type?, val otherFiles: Map, val performanceMetrics: Boolean, val numUniformShards: Int?, @@ -44,6 +46,7 @@ AndroidArgs use-orchestrator: $useOrchestrator directories-to-pull: ${ArgsToString.listToString(directoriesToPull)} grant-permissions: $grantPermissions + type: ${type?.ymlName} other-files: ${ArgsToString.mapToString(otherFiles)} performance-metrics: $performanceMetrics num-uniform-shards: $numUniformShards @@ -83,6 +86,7 @@ AndroidArgs val AndroidArgs.isDontAutograntPermissions get() = !(grantPermissions.isNotNull() && (grantPermissions.equals("all"))) + val AndroidArgs.isInstrumentationTest get() = appApk.isNotNull() && testApk.isNotNull() || diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 664ac2742f..42de879017 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -1,6 +1,7 @@ package ftl.args import ftl.args.yml.AppTestPair +import ftl.args.yml.Type import ftl.config.AndroidConfig import ftl.config.android.AndroidFlankConfig import ftl.config.android.AndroidGcloudConfig @@ -37,5 +38,6 @@ fun createAndroidArgs( ) } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!!, - grantPermissions = gcloud.grantPermissions + grantPermissions = gcloud.grantPermissions, + type = gcloud.type?.let { Type.valueOf(it) } ) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt new file mode 100644 index 0000000000..5ca269818b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt @@ -0,0 +1,5 @@ +package ftl.args.yml + +enum class Type(val ymlName: String) { + INSTURMENTATION("instrumentation"), ROBO("robo"), GAMELOOP("game-loop") +} diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 91394fda69..4359b1b03b 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -24,7 +24,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--app"], description = ["The path to the application binary file. " + - "The path may be in the local filesystem or .setDontAutograntPermissions(true)in Google Cloud Storage using gs:// notation."] + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] ) @set:JsonProperty("app") var app: String? by data @@ -102,6 +102,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:JsonProperty("grant-permissions") var grantPermissions: String? by data + @set:CommandLine.Option( + names = ["--type"], + description = ["The type of test to run. TYPE must be one of: instrumentation, robo, game-loop."] + ) + @set:JsonProperty("type") + var type: String? by data + @set:CommandLine.Option( names = ["--directories-to-pull"], split = ",", @@ -227,6 +234,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( testTargets = emptyList() roboDirectives = emptyMap() roboScript = null + type = null } } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 027875f6d5..eaf220e8fa 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -98,6 +98,7 @@ class AndroidArgsTest { - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all + type: null other-files: /sdcard/dir1/file1.txt: $appApk /sdcard/dir2/file2.jpg: $testApk @@ -312,6 +313,7 @@ AndroidArgs - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all + type: null other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath @@ -390,6 +392,7 @@ AndroidArgs use-orchestrator: true directories-to-pull: grant-permissions: all + type: null other-files: performance-metrics: false num-uniform-shards: null From 81ddefd46c17fcc3997b625509f847cf9c6101ed Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Tue, 13 Oct 2020 10:28:24 +0200 Subject: [PATCH 2/7] Finialize the type parameter --- .../main/kotlin/ftl/args/CreateAndroidArgs.kt | 2 +- .../src/main/kotlin/ftl/args/yml/Type.kt | 14 ++++- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 63 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 42de879017..68a2ad20ae 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -39,5 +39,5 @@ fun createAndroidArgs( } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!!, grantPermissions = gcloud.grantPermissions, - type = gcloud.type?.let { Type.valueOf(it) } + type = gcloud.type?.let { Type.fromString(it) } ) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt index 5ca269818b..969e802ac2 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt @@ -1,5 +1,17 @@ package ftl.args.yml +import ftl.run.exception.FlankGeneralError + enum class Type(val ymlName: String) { - INSTURMENTATION("instrumentation"), ROBO("robo"), GAMELOOP("game-loop") + INSTRUMENTATION("instrumentation"), ROBO("robo"), GAMELOOP("game-loop"); + + companion object { + fun fromString(stringVal: String): Type { + val filtered = values().filter { it.ymlName == stringVal } + if (filtered.isEmpty()) { + throw FlankGeneralError("Unsupported Type given `$stringVal` only [${values().joinToString(",")}] supported.") + } + return filtered.first() + } + } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index eaf220e8fa..3883dc74d5 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -1915,6 +1915,69 @@ AndroidArgs """.trimIndent() AndroidArgs.load(yaml).validate() } + @Test(expected = FlankGeneralError::class) + fun `should throw exception if incorrect type requested`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + grant-permissions: error + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested game-loop`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: game-loop + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested instrumental`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: instrumentation + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested robo`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: robo + """.trimIndent() + AndroidArgs.load(yaml).validate() + } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = From 2b553386cddc651ccb609cc91af474462a430e74 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Tue, 13 Oct 2020 12:36:38 +0200 Subject: [PATCH 3/7] Better unit tests --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 3883dc74d5..320cc1a409 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -5,6 +5,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE import ftl.args.IArgs.Companion.AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE import ftl.args.yml.AppTestPair +import ftl.args.yml.Type import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FtlConstants.defaultAndroidModel @@ -98,7 +99,7 @@ class AndroidArgsTest { - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all - type: null + type: instrumentation other-files: /sdcard/dir1/file1.txt: $appApk /sdcard/dir2/file2.jpg: $testApk @@ -239,6 +240,7 @@ class AndroidArgsTest { assert(useOrchestrator, false) assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false")) assert(directoriesToPull, listOf("/sdcard/screenshots", "/sdcard/screenshots2")) + assert(grantPermissions, "all") assert( otherFiles, mapOf( @@ -246,6 +248,7 @@ class AndroidArgsTest { "/sdcard/dir2/file2.jpg" to testApkAbsolutePath ) ) + assert(type, Type.INSTRUMENTATION) assert(performanceMetrics, false) assert(testRunnerClass, "com.foo.TestRunner") assert( @@ -313,7 +316,7 @@ AndroidArgs - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all - type: null + type: instrumentation other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath From 332c4276985951e09c9ba80be380c468442b16b5 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Wed, 14 Oct 2020 10:22:47 +0200 Subject: [PATCH 4/7] Added more assertiveness for the Type --- .../main/kotlin/ftl/args/ValidateAndroidArgs.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 1e22f27f9b..24962dd749 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -5,6 +5,7 @@ import ftl.android.IncompatibleModelVersion import ftl.android.SupportedDeviceConfig import ftl.android.UnsupportedModelId import ftl.android.UnsupportedVersionId +import ftl.args.yml.Type import ftl.config.containsPhysicalDevices import ftl.config.containsVirtualDevices import ftl.run.exception.FlankConfigurationError @@ -23,12 +24,26 @@ fun AndroidArgs.validate() = apply { assertTestFiles() assertOtherFiles() assertGrantPermissions() + assertType() checkResultsDirUnique() checkEnvironmentVariables() checkFilesToDownload() checkNumUniformShards() } + +private fun AndroidArgs.assertType() = type?.let { + if (appApk == null) throw FlankGeneralError("A valid AppApk must be defined if Type parameter is used.") + when (it) { + Type.INSTRUMENTATION -> { + if (testApk == null) throw FlankGeneralError("Instrumentation tests require a valid testApk defined.") + if (testRunnerClass == null) throw FlankGeneralError("Instrumentation tests require a valid test-runner-class defined.") + } + else -> {/*noop*/} + } +} + + private fun AndroidArgs.assertGrantPermissions() = grantPermissions?.let { if (it !in listOf("all", "none")) throw FlankGeneralError("Unsupported permission '$grantPermissions'\nOnly 'all' or 'none' supported.") } From 27c73e981c67dae88ac34285973582f138d85e75 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Wed, 14 Oct 2020 10:30:03 +0200 Subject: [PATCH 5/7] detekt fixes --- .../src/main/kotlin/ftl/args/ValidateAndroidArgs.kt | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 24962dd749..89cd10a671 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -31,19 +31,14 @@ fun AndroidArgs.validate() = apply { checkNumUniformShards() } - private fun AndroidArgs.assertType() = type?.let { if (appApk == null) throw FlankGeneralError("A valid AppApk must be defined if Type parameter is used.") - when (it) { - Type.INSTRUMENTATION -> { - if (testApk == null) throw FlankGeneralError("Instrumentation tests require a valid testApk defined.") - if (testRunnerClass == null) throw FlankGeneralError("Instrumentation tests require a valid test-runner-class defined.") - } - else -> {/*noop*/} + if (it == Type.INSTRUMENTATION) { + if (testApk == null) throw FlankGeneralError("Instrumentation tests require a valid testApk defined.") + if (testRunnerClass == null) throw FlankGeneralError("Instrumentation tests require a valid test-runner-class defined.") } } - private fun AndroidArgs.assertGrantPermissions() = grantPermissions?.let { if (it !in listOf("all", "none")) throw FlankGeneralError("Unsupported permission '$grantPermissions'\nOnly 'all' or 'none' supported.") } From f465b6833c8afb498206658d27810e6d0240a5de Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Wed, 14 Oct 2020 12:51:53 +0200 Subject: [PATCH 6/7] Added tests and fixed erronous ones --- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 320cc1a409..249545ce28 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -1962,6 +1962,24 @@ AndroidArgs locale: en orientation: portrait type: instrumentation + test-runner-class: com.foo.TestRunner + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if correct type requested instrumental but no test runner set`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + test + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: instrumentation """.trimIndent() AndroidArgs.load(yaml).validate() } From 2c7c9a7f378434ade7b18d86ea7b4a709cba5d81 Mon Sep 17 00:00:00 2001 From: Michael Wright Date: Thu, 15 Oct 2020 09:35:21 +0200 Subject: [PATCH 7/7] Nicer name outputs --- test_runner/src/main/kotlin/ftl/args/yml/Type.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt index 969e802ac2..7c7d163fa9 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt @@ -9,7 +9,7 @@ enum class Type(val ymlName: String) { fun fromString(stringVal: String): Type { val filtered = values().filter { it.ymlName == stringVal } if (filtered.isEmpty()) { - throw FlankGeneralError("Unsupported Type given `$stringVal` only [${values().joinToString(",")}] supported.") + throw FlankGeneralError("Unsupported Type given `$stringVal` only [${values().joinToString(","){it.ymlName}}] supported.") } return filtered.first() }