feat: Add type (#1230)

* Add type

* Finialize the type parameter

* Better unit tests

* Added more assertiveness for the Type

* detekt fixes

* Added tests and fixed erronous ones

* Nicer name outputs

README.md

@@ -328,6 +328,9 @@ gcloud:
   ## By default, all permissions are granted. PERMISSIONS must be one of: all, none
   # grant-permissions: all
 
+  ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop.
+  # type: instrumentation
+
   ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push.
   ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp).
   ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…).

test_runner/flank.yml

@@ -82,6 +82,9 @@ gcloud:
   # directories-to-pull:
   #   - /sdcard/
 
+  ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop.
+  # type: instrumentation
+
   ## Whether to grant runtime permissions on the device before the test begins.
   ## By default, all permissions are granted. PERMISSIONS must be one of: all, none
   # grant-permissions: all

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -1,6 +1,7 @@
 package ftl.args
 
 import ftl.args.yml.AppTestPair
+import ftl.args.yml.Type
 
 data class AndroidArgs(
     val commonArgs: CommonArgs,
@@ -14,6 +15,7 @@ data class AndroidArgs(
     val environmentVariables: Map<String, String>, // should not be printed, becuase could contains sensitive informations
     val directoriesToPull: List<String>,
     val grantPermissions: String?,
+    val type: Type?,
     val otherFiles: Map<String, String>,
     val performanceMetrics: Boolean,
     val numUniformShards: Int?,
@@ -45,6 +47,7 @@ AndroidArgs
       use-orchestrator: $useOrchestrator
       directories-to-pull: ${ArgsToString.listToString(directoriesToPull)}
       grant-permissions: $grantPermissions
+      type: ${type?.ymlName}
       other-files: ${ArgsToString.mapToString(otherFiles)}
       performance-metrics: $performanceMetrics
       num-uniform-shards: $numUniformShards
@@ -84,6 +87,7 @@ AndroidArgs
 
 val AndroidArgs.isDontAutograntPermissions
     get() = !(grantPermissions.isNotNull() && (grantPermissions.equals("all")))
+
 val AndroidArgs.isInstrumentationTest
     get() = appApk.isNotNull() &&
             testApk.isNotNull() ||

test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt

@@ -1,6 +1,7 @@
 package ftl.args
 
 import ftl.args.yml.AppTestPair
+import ftl.args.yml.Type
 import ftl.config.AndroidConfig
 import ftl.config.android.AndroidFlankConfig
 import ftl.config.android.AndroidGcloudConfig
@@ -39,5 +40,6 @@ fun createAndroidArgs(
     } ?: emptyList(),
     useLegacyJUnitResult = flank.useLegacyJUnitResult!!,
     obfuscateDumpShards = obfuscate,
-    grantPermissions = gcloud.grantPermissions
+    grantPermissions = gcloud.grantPermissions,
+    type = gcloud.type?.let { Type.fromString(it) }
 )

test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt

@@ -5,6 +5,7 @@ import ftl.android.IncompatibleModelVersion
 import ftl.android.SupportedDeviceConfig
 import ftl.android.UnsupportedModelId
 import ftl.android.UnsupportedVersionId
+import ftl.args.yml.Type
 import ftl.config.containsPhysicalDevices
 import ftl.config.containsVirtualDevices
 import ftl.run.exception.FlankConfigurationError
@@ -23,12 +24,21 @@ fun AndroidArgs.validate() = apply {
     assertTestFiles()
     assertOtherFiles()
     assertGrantPermissions()
+    assertType()
     checkResultsDirUnique()
     checkEnvironmentVariables()
     checkFilesToDownload()
     checkNumUniformShards()
 }
 
+private fun AndroidArgs.assertType() = type?.let {
+    if (appApk == null) throw FlankGeneralError("A valid AppApk must be defined if Type parameter is used.")
+    if (it == Type.INSTRUMENTATION) {
+        if (testApk == null) throw FlankGeneralError("Instrumentation tests require a valid testApk defined.")
+        if (testRunnerClass == null) throw FlankGeneralError("Instrumentation tests require a valid test-runner-class defined.")
+    }
+}
+
 private fun AndroidArgs.assertGrantPermissions() = grantPermissions?.let {
     if (it !in listOf("all", "none")) throw FlankGeneralError("Unsupported permission '$grantPermissions'\nOnly 'all' or 'none' supported.")
 }

test_runner/src/main/kotlin/ftl/args/yml/Type.kt

@@ -0,0 +1,17 @@
+package ftl.args.yml
+
+import ftl.run.exception.FlankGeneralError
+
+enum class Type(val ymlName: String) {
+    INSTRUMENTATION("instrumentation"), ROBO("robo"), GAMELOOP("game-loop");
+
+    companion object {
+        fun fromString(stringVal: String): Type {
+            val filtered = values().filter { it.ymlName == stringVal }
+            if (filtered.isEmpty()) {
+                throw FlankGeneralError("Unsupported Type given `$stringVal` only [${values().joinToString(","){it.ymlName}}] supported.")
+            }
+            return filtered.first()
+        }
+    }
+}

test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt

@@ -24,7 +24,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:CommandLine.Option(
         names = ["--app"],
         description = ["The path to the application binary file. " +
-            "The path may be in the local filesystem or .setDontAutograntPermissions(true)in Google Cloud Storage using gs:// notation."]
+            "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."]
     )
     @set:JsonProperty("app")
     var app: String? by data
@@ -102,6 +102,13 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
     @set:JsonProperty("grant-permissions")
     var grantPermissions: String? by data
 
+    @set:CommandLine.Option(
+        names = ["--type"],
+        description = ["The type of test to run. TYPE must be one of: instrumentation, robo, game-loop."]
+    )
+    @set:JsonProperty("type")
+    var type: String? by data
+
     @set:CommandLine.Option(
         names = ["--directories-to-pull"],
         split = ",",
@@ -227,6 +234,7 @@ data class AndroidGcloudConfig @JsonIgnore constructor(
             testTargets = emptyList()
             roboDirectives = emptyMap()
             roboScript = null
+            type = null
         }
     }
 }

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -5,6 +5,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE
 import ftl.args.IArgs.Companion.AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE
 import ftl.args.yml.AppTestPair
+import ftl.args.yml.Type
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FtlConstants.defaultAndroidModel
@@ -98,6 +99,7 @@ class AndroidArgsTest {
           - /sdcard/screenshots
           - /sdcard/screenshots2
           grant-permissions: all
+          type: instrumentation
           other-files:
             /sdcard/dir1/file1.txt: $appApk
             /sdcard/dir2/file2.jpg: $testApk
@@ -238,13 +240,15 @@ class AndroidArgsTest {
             assert(useOrchestrator, false)
             assert(environmentVariables, linkedMapOf("clearPackageData" to "true", "randomEnvVar" to "false"))
             assert(directoriesToPull, listOf("/sdcard/screenshots", "/sdcard/screenshots2"))
+            assert(grantPermissions, "all")
             assert(
                 otherFiles,
                 mapOf(
                     "/sdcard/dir1/file1.txt" to appApkAbsolutePath,
                     "/sdcard/dir2/file2.jpg" to testApkAbsolutePath
                 )
             )
+            assert(type, Type.INSTRUMENTATION)
             assert(performanceMetrics, false)
             assert(testRunnerClass, "com.foo.TestRunner")
             assert(
@@ -312,6 +316,7 @@ AndroidArgs
         - /sdcard/screenshots
         - /sdcard/screenshots2
       grant-permissions: all
+      type: instrumentation
       other-files: 
         /sdcard/dir1/file1.txt: $appApkAbsolutePath
         /sdcard/dir2/file2.jpg: $testApkAbsolutePath
@@ -390,6 +395,7 @@ AndroidArgs
       use-orchestrator: true
       directories-to-pull: 
       grant-permissions: all
+      type: null
       other-files: 
       performance-metrics: false
       num-uniform-shards: null
@@ -1912,6 +1918,87 @@ AndroidArgs
         """.trimIndent()
         AndroidArgs.load(yaml).validate()
     }
+    @Test(expected = FlankGeneralError::class)
+    fun `should throw exception if incorrect type requested`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          grant-permissions: error
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should Not throw exception if correct type requested game-loop`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: game-loop
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should Not throw exception if correct type requested instrumental`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: instrumentation
+          test-runner-class: com.foo.TestRunner
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test(expected = FlankConfigurationError::class)
+    fun `should throw exception if correct type requested instrumental but no test runner set`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          test
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: instrumentation
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
+
+    @Test
+    fun `should Not throw exception if correct type requested robo`() {
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          device:
+            - model: Nexus6
+              version: 25
+              locale: en
+              orientation: portrait
+          type: robo
+        """.trimIndent()
+        AndroidArgs.load(yaml).validate()
+    }
 }
 
 private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs =