Add robo for robo-directives & robo-script options

test_runner/flank.yml

@@ -101,6 +101,24 @@ gcloud:
   # test-targets:
   #  - class com.example.app.ExampleUiTest#testPasses
 
+  ## A map of robo_directives that you can use to customize the behavior of Robo test.
+  ## The type specifies the action type of the directive, which may take on values click, text or ignore.
+  ## If no type is provided, text will be used by default.
+  ## Each key should be the Android resource name of a target UI element and each value should be the text input for that element.
+  ## Values are only permitted for text type elements, so no value should be specified for click and ignore type elements.
+  # robo-directives:
+  # - type: text
+  #   name: input_resource_name
+  #   input: message
+  # - type: click
+  #   name: button_resource_name
+
+  ## The path to a Robo Script JSON file.
+  ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+  ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.
+  ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting.
+  # robo-script: path_to_robo_script
+
   ## A list of DIMENSION=VALUE pairs which specify a target device to test against.
   ## This flag may be repeated to specify multiple devices.
   ## The four device dimensions are: model, version, locale, and orientation.

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -14,9 +14,9 @@ import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.apksToString
-import ftl.args.ArgsToString.devicesToString
 import ftl.args.ArgsToString.listToString
 import ftl.args.ArgsToString.mapToString
+import ftl.args.ArgsToString.objectsToString
 import ftl.args.yml.AndroidFlankYml
 import ftl.args.yml.AndroidGcloudYml
 import ftl.args.yml.AndroidGcloudYmlParams
@@ -27,6 +27,7 @@ import ftl.args.yml.YamlDeprecated
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FtlConstants
+import ftl.config.parseRoboDirectives
 import ftl.util.FlankFatalError
 import java.nio.file.Files
 import java.nio.file.Path
@@ -57,6 +58,8 @@ class AndroidArgs(
 
     // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false
     val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator
+    val roboDirectives = cli?.roboDirectives?.parseRoboDirectives() ?: androidGcloud.roboDirectives
+    val roboScript = (cli?.roboScript ?: androidGcloud.roboScript)?.processFilePath("from roboScript")
     val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables
     val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull
     val otherFiles = (cli?.otherFiles ?: androidGcloud.otherFiles).map { (devicePath, filePath) ->
@@ -136,7 +139,9 @@ AndroidArgs
       performance-metrics: $performanceMetrics
       test-runner-class: $testRunnerClass
       test-targets:${listToString(testTargets)}
-      device:${devicesToString(devices)}
+      robo-directives:${objectsToString(roboDirectives)}
+      robo-script: $roboScript
+      device:${objectsToString(devices)}
       num-flaky-test-attempts: $flakyTestAttempts
 
     flank:

test_runner/src/main/kotlin/ftl/args/ArgsToString.kt

@@ -13,7 +13,7 @@ object ArgsToString {
             .joinToString("\n")
     }
 
-    fun listToString(list: List<String?>?): String {
+    fun listToString(list: List<Any?>?): String {
         if (list.isNullOrEmpty()) return ""
         return NEW_LINE + list.filterNotNull()
             .joinToString("\n") { dir -> "        - $dir" }
@@ -25,6 +25,12 @@ object ArgsToString {
             .joinToString("\n") { "$it" }
     }
 
+    fun objectsToString(objects: List<Any?>?): String {
+        if (objects.isNullOrEmpty()) return ""
+        return NEW_LINE + objects.filterNotNull()
+            .joinToString("\n") { "$it" }
+    }
+
     fun apksToString(devices: List<AppTestPair>): String {
         if (devices.isNullOrEmpty()) return ""
         return NEW_LINE + devices.joinToString("\n") { (app, test) -> "        - app: $app\n          test: $test" }

test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt

@@ -6,6 +6,7 @@ import ftl.config.Device
 import ftl.config.FlankDefaults
 import ftl.config.FtlConstants.defaultAndroidModel
 import ftl.config.FtlConstants.defaultAndroidVersion
+import ftl.config.FlankRoboDirective
 
 /**
  * Android specific gcloud parameters
@@ -44,6 +45,12 @@ class AndroidGcloudYmlParams(
     @field:JsonProperty("test-targets")
     val testTargets: List<String?> = emptyList(),
 
+    @field:JsonProperty("robo-directives")
+    val roboDirectives: List<FlankRoboDirective> = emptyList(),
+
+    @field:JsonProperty("robo-script")
+    val roboScript: String? = null,
+
     val device: List<Device> = listOf(Device(defaultAndroidModel, defaultAndroidVersion))
 ) {
     companion object : IYmlKeys {
@@ -59,6 +66,8 @@ class AndroidGcloudYmlParams(
             "performance-metrics",
             "test-runner-class",
             "test-targets",
+            "robo-directives",
+            "robo-script",
             "device"
         )
     }

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -129,6 +129,30 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {
     )
     var noUseOrchestrator: Boolean? = null
 
+    @Option(
+        names = ["--robo-directives"],
+        split = ",",
+        description = [
+            "A comma-separated (<type>:<key>=<value>) map of robo_directives that you can use to customize the behavior of Robo test.",
+            "The type specifies the action type of the directive, which may take on values click, text or ignore.",
+            "If no type is provided, text will be used by default.",
+            "Each key should be the Android resource name of a target UI element and each value should be the text input for that element.",
+            "Values are only permitted for text type elements, so no value should be specified for click and ignore type elements."
+        ]
+    )
+    var roboDirectives: List<String>? = null
+
+    @Option(
+        names = ["--robo-script"],
+        description = [
+            "The path to a Robo Script JSON file.",
+            "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.",
+            "You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.",
+            "Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. "
+        ]
+    )
+    var roboScript: String? = null
+
     @Option(
         names = ["--environment-variables"],
         split = ",",

test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt

@@ -0,0 +1,29 @@
+package ftl.config
+
+import ftl.util.trimStartLine
+
+data class FlankRoboDirective(
+    val type: String,
+    val name: String,
+    val input: String? = null // Input is only permitted for text type elements
+) {
+    override fun toString() = """
+        - type: $type
+          name: $name
+          input: $input""".trimStartLine()
+}
+
+fun List<String>.parseRoboDirectives() = map(String::parseRoboDirective)
+
+fun String.parseRoboDirective(): FlankRoboDirective = split(
+    Regex("([:=])")
+).let { chunks ->
+    require(chunks.size == 3) {
+        "Cannot parse robo directive `$this`, use following format `\$TYPE:\$RESOURCE_NAME=\$INPUT`"
+    }
+    FlankRoboDirective(
+        type = chunks[0],
+        name = chunks[1],
+        input = chunks[2]
+    )
+}

test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt

@@ -4,6 +4,7 @@ import com.google.api.services.testing.Testing
 import com.google.api.services.testing.model.Account
 import com.google.api.services.testing.model.AndroidDeviceList
 import com.google.api.services.testing.model.AndroidInstrumentationTest
+import com.google.api.services.testing.model.AndroidRoboTest
 import com.google.api.services.testing.model.Apk
 import com.google.api.services.testing.model.ClientInfo
 import com.google.api.services.testing.model.DeviceFile
@@ -15,6 +16,7 @@ import com.google.api.services.testing.model.GoogleCloudStorage
 import com.google.api.services.testing.model.ManualSharding
 import com.google.api.services.testing.model.RegularFile
 import com.google.api.services.testing.model.ResultStorage
+import com.google.api.services.testing.model.RoboDirective
 import com.google.api.services.testing.model.ShardingOption
 import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.testing.model.TestSetup
@@ -23,6 +25,7 @@ import com.google.api.services.testing.model.TestTargetsForShard
 import com.google.api.services.testing.model.ToolResultsHistory
 import ftl.args.AndroidArgs
 import ftl.args.ShardChunks
+import ftl.config.FlankRoboDirective
 import ftl.util.join
 import ftl.util.timeoutToSeconds
 
@@ -33,6 +36,7 @@ object GcAndroidTestMatrix {
         value = this@toEnvironmentVariable.value
     }
 
+    @Suppress("LongParameterList")
     fun build(
         appApkGcsPath: String,
         testApkGcsPath: String,
@@ -42,7 +46,8 @@ object GcAndroidTestMatrix {
         testShards: ShardChunks,
         args: AndroidArgs,
         toolResultsHistory: ToolResultsHistory,
-        additionalApkGcsPaths: List<String>
+        additionalApkGcsPaths: List<String>,
+        roboScriptGcsPath: String?
     ): Testing.Projects.TestMatrices.Create {
 
         // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120
@@ -101,6 +106,11 @@ object GcAndroidTestMatrix {
             .setDisableVideoRecording(!args.recordVideo)
             .setTestTimeout("${testTimeoutSeconds}s")
             .setTestSetup(testSetup)
+            .setAndroidRoboTest(
+                AndroidRoboTest()
+                    .setRoboDirectives(args.roboDirectives.mapToApiRoboDirectives())
+                    .setRoboScript(FileReference().setGcsPath(roboScriptGcsPath))
+            )
 
         val resultsStorage = ResultStorage()
             .setGoogleCloudStorage(GoogleCloudStorage().setGcsPath(matrixGcsPath))
@@ -134,3 +144,11 @@ private fun Map<String, String>.mapToDeviceFiles() = map { (devicePath: String,
             .setContent(FileReference().setGcsPath(gcsFilePath))
     )
 }
+
+private fun List<FlankRoboDirective>.mapToApiRoboDirectives() = map {
+    RoboDirective().apply {
+        actionType = it.type
+        resourceName = it.name
+        inputText = it.input
+    }
+}

test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt

@@ -1,8 +1,7 @@
 package ftl.run.platform
 
-import com.google.api.services.testing.model.AndroidDeviceList
+import com.google.api.services.testing.Testing
 import com.google.api.services.testing.model.TestMatrix
-import com.google.api.services.testing.model.ToolResultsHistory
 import ftl.args.AndroidArgs
 import ftl.args.AndroidTestShard
 import ftl.args.ShardChunks
@@ -35,25 +34,31 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS
     val history = GcToolResults.createToolResultsHistory(args)
     val resolvedTestApks = args.getResolvedTestApks()
     val otherGcsFiles = args.otherFiles.uploadOtherFiles(args.resultsBucket, runGcsPath)
+    val roboScriptGcsPath = args.roboScript?.let { GcStorage.upload(it, args.resultsBucket, runGcsPath) }
 
     val allTestShardChunks: ShardChunks = resolvedTestApks.map { apks: ResolvedTestApks ->
         // Ensure we only shard tests that are part of the test apk. Use the resolved test apk path to make sure
         // we don't re-download an apk it is on the local file system.
         AndroidTestShard.getTestShardChunks(args, apks.test).also { testShards ->
-            testMatrices += executeAndroidTestMatrix(
-                uploadedTestApks = uploadTestApks(
-                    apks = apks,
-                    args = args,
-                    runGcsPath = runGcsPath
-                ),
-                otherFiles = otherGcsFiles,
-                runGcsPath = runGcsPath,
-                androidDeviceList = androidDeviceList,
-                testShards = testShards,
+            val uploadedTestApks = uploadTestApks(
+                apks = apks,
                 args = args,
-                history = history,
-                runCount = runCount
+                runGcsPath = runGcsPath
             )
+            testMatrices += executeAndroidTestMatrix(runCount) {
+                GcAndroidTestMatrix.build(
+                    appApkGcsPath = uploadedTestApks.app,
+                    testApkGcsPath = uploadedTestApks.test,
+                    runGcsPath = runGcsPath,
+                    additionalApkGcsPaths = uploadedTestApks.additionalApks,
+                    roboScriptGcsPath = roboScriptGcsPath,
+                    androidDeviceList = androidDeviceList,
+                    testShards = testShards,
+                    args = args,
+                    otherFiles = otherGcsFiles,
+                    toolResultsHistory = history
+                )
+            }
         }
     }.flatten()
 
@@ -79,28 +84,12 @@ private fun AndroidArgs.getResolvedTestApks() = listOf(
 )
 
 private suspend fun executeAndroidTestMatrix(
-    runGcsPath: String,
-    args: AndroidArgs,
-    testShards: ShardChunks,
-    uploadedTestApks: UploadedTestApks,
-    otherFiles: Map<String, String>,
-    androidDeviceList: AndroidDeviceList,
-    history: ToolResultsHistory,
-    runCount: Int
+    runCount: Int,
+    createTestMatrix: () -> Testing.Projects.TestMatrices.Create
 ): List<Deferred<TestMatrix>> = coroutineScope {
     (0 until runCount).map {
         async(Dispatchers.IO) {
-            GcAndroidTestMatrix.build(
-                appApkGcsPath = uploadedTestApks.app,
-                testApkGcsPath = uploadedTestApks.test,
-                runGcsPath = runGcsPath,
-                androidDeviceList = androidDeviceList,
-                testShards = testShards,
-                args = args,
-                otherFiles = otherFiles,
-                toolResultsHistory = history,
-                additionalApkGcsPaths = uploadedTestApks.additionalApks
-            ).executeWithRetry()
+            createTestMatrix().executeWithRetry()
         }
     }
 }

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -4,6 +4,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.args.yml.AppTestPair
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
+import ftl.config.FlankRoboDirective
 import ftl.config.FtlConstants.defaultAndroidModel
 import ftl.config.FtlConstants.defaultAndroidVersion
 import ftl.run.platform.runAndroidTests
@@ -69,6 +70,13 @@ class AndroidArgsTest {
           test-targets:
           - class com.example.app.ExampleUiTest#testPasses
           - class com.example.app.ExampleUiTest#testFails
+          robo-directives:
+            - type: text
+              name: resource_name_1
+              input: some_text
+            - type: click
+              name: resource_name_2
+          robo-script: $appApk
           device:
           - model: NexusLowRes
             version: 23
@@ -210,6 +218,13 @@ class AndroidArgsTest {
                     "class com.example.app.ExampleUiTest#testFails"
                 )
             )
+            assert(
+                roboDirectives, listOf(
+                    FlankRoboDirective(type = "text", name = "resource_name_1", input = "some_text"),
+                    FlankRoboDirective(type = "click", name = "resource_name_2")
+                )
+            )
+            assert(roboScript, appApkAbsolutePath)
             assert(
                 devices, listOf(
                     Device("NexusLowRes", "23", "en", "portrait"),
@@ -270,6 +285,14 @@ AndroidArgs
       test-targets:
         - class com.example.app.ExampleUiTest#testPasses
         - class com.example.app.ExampleUiTest#testFails
+      robo-directives:
+        - type: text
+          name: resource_name_1
+          input: some_text
+        - type: click
+          name: resource_name_2
+          input: null
+      robo-script: $appApkAbsolutePath
       device:
         - model: NexusLowRes
           version: 23
@@ -333,6 +356,8 @@ AndroidArgs
       performance-metrics: false
       test-runner-class: null
       test-targets:
+      robo-directives:
+      robo-script: null
       device:
         - model: NexusLowRes
           version: 28