diff --git a/README.md b/README.md index 191eabfe6a..693cf4de56 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,9 @@ gcloud: ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all + ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. + # type: instrumentation + ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 8f95a63adb..a493e54154 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -82,6 +82,9 @@ gcloud: # directories-to-pull: # - /sdcard/ + ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. + # type: instrumentation + ## Whether to grant runtime permissions on the device before the test begins. ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 1956d3916c..3ae8af0843 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -1,6 +1,7 @@ package ftl.args import ftl.args.yml.AppTestPair +import ftl.args.yml.Type data class AndroidArgs( val commonArgs: CommonArgs, @@ -14,6 +15,7 @@ data class AndroidArgs( val environmentVariables: Map, // should not be printed, becuase could contains sensitive informations val directoriesToPull: List, val grantPermissions: String?, + val type: Type?, val otherFiles: Map, val performanceMetrics: Boolean, val numUniformShards: Int?, @@ -45,6 +47,7 @@ AndroidArgs use-orchestrator: $useOrchestrator directories-to-pull: ${ArgsToString.listToString(directoriesToPull)} grant-permissions: $grantPermissions + type: ${type?.ymlName} other-files: ${ArgsToString.mapToString(otherFiles)} performance-metrics: $performanceMetrics num-uniform-shards: $numUniformShards @@ -84,6 +87,7 @@ AndroidArgs val AndroidArgs.isDontAutograntPermissions get() = !(grantPermissions.isNotNull() && (grantPermissions.equals("all"))) + val AndroidArgs.isInstrumentationTest get() = appApk.isNotNull() && testApk.isNotNull() || diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 90936d1a9a..67a1cdd7f7 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -1,6 +1,7 @@ package ftl.args import ftl.args.yml.AppTestPair +import ftl.args.yml.Type import ftl.config.AndroidConfig import ftl.config.android.AndroidFlankConfig import ftl.config.android.AndroidGcloudConfig @@ -39,5 +40,6 @@ fun createAndroidArgs( } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!!, obfuscateDumpShards = obfuscate, - grantPermissions = gcloud.grantPermissions + grantPermissions = gcloud.grantPermissions, + type = gcloud.type?.let { Type.fromString(it) } ) diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 1e22f27f9b..89cd10a671 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -5,6 +5,7 @@ import ftl.android.IncompatibleModelVersion import ftl.android.SupportedDeviceConfig import ftl.android.UnsupportedModelId import ftl.android.UnsupportedVersionId +import ftl.args.yml.Type import ftl.config.containsPhysicalDevices import ftl.config.containsVirtualDevices import ftl.run.exception.FlankConfigurationError @@ -23,12 +24,21 @@ fun AndroidArgs.validate() = apply { assertTestFiles() assertOtherFiles() assertGrantPermissions() + assertType() checkResultsDirUnique() checkEnvironmentVariables() checkFilesToDownload() checkNumUniformShards() } +private fun AndroidArgs.assertType() = type?.let { + if (appApk == null) throw FlankGeneralError("A valid AppApk must be defined if Type parameter is used.") + if (it == Type.INSTRUMENTATION) { + if (testApk == null) throw FlankGeneralError("Instrumentation tests require a valid testApk defined.") + if (testRunnerClass == null) throw FlankGeneralError("Instrumentation tests require a valid test-runner-class defined.") + } +} + private fun AndroidArgs.assertGrantPermissions() = grantPermissions?.let { if (it !in listOf("all", "none")) throw FlankGeneralError("Unsupported permission '$grantPermissions'\nOnly 'all' or 'none' supported.") } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/Type.kt b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt new file mode 100644 index 0000000000..7c7d163fa9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/Type.kt @@ -0,0 +1,17 @@ +package ftl.args.yml + +import ftl.run.exception.FlankGeneralError + +enum class Type(val ymlName: String) { + INSTRUMENTATION("instrumentation"), ROBO("robo"), GAMELOOP("game-loop"); + + companion object { + fun fromString(stringVal: String): Type { + val filtered = values().filter { it.ymlName == stringVal } + if (filtered.isEmpty()) { + throw FlankGeneralError("Unsupported Type given `$stringVal` only [${values().joinToString(","){it.ymlName}}] supported.") + } + return filtered.first() + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt index 91394fda69..4359b1b03b 100644 --- a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -24,7 +24,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:CommandLine.Option( names = ["--app"], description = ["The path to the application binary file. " + - "The path may be in the local filesystem or .setDontAutograntPermissions(true)in Google Cloud Storage using gs:// notation."] + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] ) @set:JsonProperty("app") var app: String? by data @@ -102,6 +102,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor( @set:JsonProperty("grant-permissions") var grantPermissions: String? by data + @set:CommandLine.Option( + names = ["--type"], + description = ["The type of test to run. TYPE must be one of: instrumentation, robo, game-loop."] + ) + @set:JsonProperty("type") + var type: String? by data + @set:CommandLine.Option( names = ["--directories-to-pull"], split = ",", @@ -227,6 +234,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor( testTargets = emptyList() roboDirectives = emptyMap() roboScript = null + type = null } } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 027875f6d5..249545ce28 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -5,6 +5,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE import ftl.args.IArgs.Companion.AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE import ftl.args.yml.AppTestPair +import ftl.args.yml.Type import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FtlConstants.defaultAndroidModel @@ -98,6 +99,7 @@ class AndroidArgsTest { - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all + type: instrumentation other-files: /sdcard/dir1/file1.txt: $appApk /sdcard/dir2/file2.jpg: $testApk @@ -238,6 +240,7 @@ class AndroidArgsTest { assert(useOrchestrator, false) assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false")) assert(directoriesToPull, listOf("/sdcard/screenshots", "/sdcard/screenshots2")) + assert(grantPermissions, "all") assert( otherFiles, mapOf( @@ -245,6 +248,7 @@ class AndroidArgsTest { "/sdcard/dir2/file2.jpg" to testApkAbsolutePath ) ) + assert(type, Type.INSTRUMENTATION) assert(performanceMetrics, false) assert(testRunnerClass, "com.foo.TestRunner") assert( @@ -312,6 +316,7 @@ AndroidArgs - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all + type: instrumentation other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath @@ -390,6 +395,7 @@ AndroidArgs use-orchestrator: true directories-to-pull: grant-permissions: all + type: null other-files: performance-metrics: false num-uniform-shards: null @@ -1912,6 +1918,87 @@ AndroidArgs """.trimIndent() AndroidArgs.load(yaml).validate() } + @Test(expected = FlankGeneralError::class) + fun `should throw exception if incorrect type requested`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + grant-permissions: error + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested game-loop`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: game-loop + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested instrumental`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: instrumentation + test-runner-class: com.foo.TestRunner + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw exception if correct type requested instrumental but no test runner set`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + test + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: instrumentation + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test + fun `should Not throw exception if correct type requested robo`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + device: + - model: Nexus6 + version: 25 + locale: en + orientation: portrait + type: robo + """.trimIndent() + AndroidArgs.load(yaml).validate() + } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs =