Add support for robo-directives & robo-script options

README.md

@@ -215,6 +215,20 @@ gcloud:
   ## Invoke a test asynchronously without waiting for test results.
   # async: false
 
+  ## A key-value map of additional details to attach to the test matrix.
+  ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run.
+  ## When consuming the test results, such as in Cloud Functions or a CI system,
+  ## these details can add additional context such as a link to the corresponding pull request.
+  # client-details
+  #   key1: value1
+  #   key2: value2
+
+  ## The name of the network traffic profile, for example LTE, HSPA, etc,
+  ## which consists of a set of parameters to emulate network conditions when running the test
+  ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command).
+  ## This feature only works on physical devices.
+  # network-profile: LTE
+
   ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest).
   ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list.
   # results-history-name: android-history
@@ -257,10 +271,27 @@ gcloud:
   # directories-to-pull:
   #   - /sdcard/
 
+  ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push.
+  ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp).
+  ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…).
+  # other-files
+  #   - /sdcard/dir1/file1.txt: local/file.txt
+  #   - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg
+
   ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only).
   ## Disabled by default. Use --performance-metrics to enable.
   # performance-metrics: true
 
+  ## Specifies the number of shards into which you want to evenly distribute test cases.
+  ## The shards are run in parallel on separate devices. For example,
+  ## if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases.
+  ## The number of shards should be less than the total number of test cases.
+  ## The number of shards specified must be >= 1 and <= 50.
+  ## This option cannot be used along max-test-shards and is not compatible with smart sharding.
+  ## If you want to take benefits of smart sharding use max-test-shards instead.
+  ## default: null
+  # num-uniform-shards: 50
+
   ## The fully-qualified Java class name of the instrumentation test runner
   ## (default: the last name extracted from the APK manifest).
   # test-runner-class: com.foo.TestRunner
@@ -273,6 +304,21 @@ gcloud:
   # test-targets:
   #  - class com.example.app.ExampleUiTest#testPasses
 
+  ## A map of robo_directives that you can use to customize the behavior of Robo test.
+  ## The type specifies the action type of the directive, which may take on values click, text or ignore.
+  ## If no type is provided, text will be used by default.
+  ## Each key should be the Android resource name of a target UI element and each value should be the text input for that element.
+  ## Values are only permitted for text type elements, so no value should be specified for click and ignore type elements.
+  # robo-directives:
+  #   "text:input_resource_name": message
+  #   "click:button_resource_name": ""
+
+  ## The path to a Robo Script JSON file.
+  ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+  ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.
+  ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting.
+  # robo-script: path_to_robo_script
+
   ## A list of DIMENSION=VALUE pairs which specify a target device to test against.
   ## This flag may be repeated to specify multiple devices.
   ## The four device dimensions are: model, version, locale, and orientation.
@@ -297,7 +343,7 @@ flank:
   ## default: -1 (unlimited)
   # shard-time: -1
 
-  ## repeat tests - the amount of times to run the tests.
+  ## The amount of times to run the tests.
   ## 1 runs the tests once. 10 runs all the tests 10x
   # num-test-runs: 1
 
@@ -344,6 +390,12 @@ flank:
   ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code.
   ## The JUnit XML is used to determine failure. (default: false)
   # ignore-failed-tests: true
+
+  ## Flank provides two ways for parsing junit xml results.
+  ## New way uses google api instead of merging xml files, but can generate slightly different output format.
+  ## This flag allows fallback for legacy xml junit results parsing
+  ## Currently available for android, iOS still uses only legacy way.
+  # legacy-junit-result: false
 ```
 
 ### Android code coverage

release_notes.md

@@ -3,6 +3,7 @@
 - [#712](https://github.com/Flank/flank/pull/712) Add keep file path for ios. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#711](https://github.com/Flank/flank/pull/711) Remove hardcoded height. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#708](https://github.com/Flank/flank/pull/708) Add ignore failed tests option to Flank. ([pawelpasterz](https://github.com/pawelpasterz))
+- [#704](https://github.com/Flank/flank/pull/709) Add robo for robo-directives & robo-script options. ([jan-gogo](https://github.com/jan-gogo))
 - [#704](https://github.com/Flank/flank/pull/704) Fix shards calculation when there are ignored tests and shardTime is -1. ([jan-gogo](https://github.com/jan-gogo))
 - [#692](https://github.com/Flank/flank/pull/698) Add support for other-files option. ([jan-gogo](https://github.com/jan-gogo))
 - [#695](https://github.com/Flank/flank/pull/695) Add support for additional-apks option. ([jan-gogo](https://github.com/jan-gogo))

test_app/app/src/androidTest/java/com/example/test_app/BaseInstrumentedTest.kt

@@ -1,12 +1,12 @@
 package com.example.test_app
 
 import android.Manifest
-import android.os.SystemClock
 import androidx.test.espresso.intent.rule.IntentsTestRule
 import androidx.test.rule.GrantPermissionRule
 import com.example.test_app.screenshot.ScreenshotTestRule
 import org.junit.Assert.assertTrue
 import org.junit.Rule
+import kotlin.random.Random
 
 abstract class BaseInstrumentedTest {
 
@@ -29,7 +29,7 @@ abstract class BaseInstrumentedTest {
         val result: Boolean = when (BuildConfig.FLAVOR_type) {
             "success" -> true
             "error" -> false
-            "flaky" -> SystemClock.uptimeMillis() % 2 == 0L
+            "flaky" -> Random.nextBoolean()
             else -> throw Error("Invalid flavour type")
         }
         assertTrue(result)

test_app/app/src/main/java/com/example/test_app/MainActivity.kt

@@ -1,16 +1,31 @@
 package com.example.test_app
 
+import android.content.DialogInterface
 import androidx.appcompat.app.AppCompatActivity
 import android.os.Bundle
+import android.widget.Button
+import android.widget.Toast
+import androidx.appcompat.app.AlertDialog
 
 class MainActivity : AppCompatActivity() {
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
         setContentView(R.layout.activity_main)
-    }
 
-    fun foo() {
-        println("foo!")
+        findViewById<Button>(R.id.button1).setOnClickListener {
+            Toast.makeText(this, "toast", Toast.LENGTH_SHORT).show()
+        }
+
+        findViewById<Button>(R.id.button2).setOnClickListener {
+            AlertDialog.Builder(this)
+                .setTitle("Alert Dialog")
+                .setPositiveButton("OK") { dialog: DialogInterface, _: Int -> dialog.dismiss() }
+                .show()
+        }
+
+        findViewById<Button>(R.id.button3).setOnClickListener {
+            throw Exception()
+        }
     }
 }

test_app/app/src/main/res/layout/activity_main.xml

@@ -1,19 +1,33 @@
 <?xml version="1.0" encoding="utf-8"?>
-<androidx.constraintlayout.widget.ConstraintLayout
-        xmlns:android="http://schemas.android.com/apk/res/android"
-        xmlns:tools="http://schemas.android.com/tools"
-        xmlns:app="http://schemas.android.com/apk/res-auto"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        tools:context=".MainActivity">
+<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:gravity="center"
+    android:orientation="vertical"
+    tools:context=".MainActivity">
 
     <TextView
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content"
-            android:text="Hello World!"
-            app:layout_constraintBottom_toBottomOf="parent"
-            app:layout_constraintLeft_toLeftOf="parent"
-            app:layout_constraintRight_toRightOf="parent"
-            app:layout_constraintTop_toTopOf="parent"/>
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="Hello World!" />
 
-</androidx.constraintlayout.widget.ConstraintLayout>
+    <Button
+        android:id="@+id/button1"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="toast" />
+
+    <Button
+        android:id="@+id/button2"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="alert" />
+
+    <Button
+        android:id="@+id/button3"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="exception" />
+
+</LinearLayout>

test_runner/flank.yml

@@ -10,7 +10,7 @@ gcloud:
   ## (default: a timestamp with a random suffix).
   # results-dir: tmp
 
-  ## Enable video recording during the test. Enabled by default, use --no-record-video to disable.
+  ## Enable video recording during the test. Disabled by default. Use --record-video to enable.
   # record-video: true
 
   ## The max time this test execution can run before it is cancelled (default: 15m).
@@ -57,7 +57,7 @@ gcloud:
   test: ../test_app/apks/app-debug-androidTest.apk
 
   ## Automatically log into the test device using a preconfigured Google account before beginning the test.
-  ## Enabled by default, use --no-auto-google-login to disable.
+  ## Disabled by default. Use --auto-google-login to enable.
   # auto-google-login: true
 
   ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator
@@ -86,7 +86,7 @@ gcloud:
   #   - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg
 
   ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only).
-  ## Enabled by default, use --no-performance-metrics to disable.
+  ## Disabled by default. Use --performance-metrics to enable.
   # performance-metrics: true
 
   ## Specifies the number of shards into which you want to evenly distribute test cases.
@@ -111,6 +111,21 @@ gcloud:
   # test-targets:
   #  - class com.example.app.ExampleUiTest#testPasses
 
+  ## A map of robo_directives that you can use to customize the behavior of Robo test.
+  ## The type specifies the action type of the directive, which may take on values click, text or ignore.
+  ## If no type is provided, text will be used by default.
+  ## Each key should be the Android resource name of a target UI element and each value should be the text input for that element.
+  ## Values are only permitted for text type elements, so no value should be specified for click and ignore type elements.
+  # robo-directives:
+  #   "text:input_resource_name": message
+  #   "click:button_resource_name": ""
+
+  ## The path to a Robo Script JSON file.
+  ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+  ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.
+  ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting.
+  # robo-script: path_to_robo_script
+
   ## A list of DIMENSION=VALUE pairs which specify a target device to test against.
   ## This flag may be repeated to specify multiple devices.
   ## The four device dimensions are: model, version, locale, and orientation.
@@ -130,7 +145,7 @@ flank:
   # max-test-shards: 1
 
   ## shard time - the amount of time tests within a shard should take
-  ## when set to > 0, the shard count is dynamically set based on time up to the maxmimum limit defined by max-test-shards
+  ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards
   ## 2 minutes (120) is recommended.
   ## default: -1 (unlimited)
   # shard-time: -1
@@ -164,19 +179,25 @@ flank:
   ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
   # local-result-dir: flank
 
-  ## Downloaded files preserves the original path of file. Required when file names are not unique.
+  ## Keeps the full path of downloaded files. Required when file names are not unique.
   ## Default: false
   # keep-file-path: false
 
-  ## Include additional app/test apk pairs in the run. If app is omitted, then the top level app is used for that pair.
+  ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path!
+  ## If app is omitted, then the top level app is used for that pair.
   # additional-app-test-apks:
   #  - app: ../test_app/apks/app-debug.apk
-  #    test: ../test_app/apks/app-debug-androidTest.apk
-  #  - test: ../test_app/apks/app-debug-androidTest.apk
+  #    test: ../test_app/apks/app1-debug-androidTest.apk
+  #  - test: ../test_app/apks/app2-debug-androidTest.apk
 
   ## The max time this test run can execute before it is cancelled (default: unlimited).
   # run-timeout: 60m
 
+  ## Terminate with exit code 0 when there are failed tests.
+  ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code.
+  ## The JUnit XML is used to determine failure. (default: false)
+  # ignore-failed-tests: true
+
   ## Flank provides two ways for parsing junit xml results.
   ## New way uses google api instead of merging xml files, but can generate slightly different output format.
   ## This flag allows fallback for legacy xml junit results parsing

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -14,9 +14,9 @@ import ftl.args.ArgsHelper.evaluateFilePath
 import ftl.args.ArgsHelper.mergeYmlMaps
 import ftl.args.ArgsHelper.yamlMapper
 import ftl.args.ArgsToString.apksToString
-import ftl.args.ArgsToString.devicesToString
 import ftl.args.ArgsToString.listToString
 import ftl.args.ArgsToString.mapToString
+import ftl.args.ArgsToString.objectsToString
 import ftl.args.yml.AndroidFlankYml
 import ftl.args.yml.AndroidGcloudYml
 import ftl.args.yml.AndroidGcloudYmlParams
@@ -27,6 +27,7 @@ import ftl.args.yml.YamlDeprecated
 import ftl.cli.firebase.test.android.AndroidRunCommand
 import ftl.config.Device
 import ftl.config.FtlConstants
+import ftl.config.parseRoboDirectives
 import ftl.util.FlankFatalError
 import java.nio.file.Files
 import java.nio.file.Path
@@ -51,12 +52,14 @@ class AndroidArgs(
 
     private val androidGcloud = androidGcloudYml.gcloud
     var appApk = (cli?.app ?: androidGcloud.app ?: throw FlankFatalError("app is not set")).processFilePath("from app")
-    var testApk = (cli?.test ?: androidGcloud.test ?: throw FlankFatalError("test is not set")).processFilePath("from test")
+    var testApk = (cli?.test ?: androidGcloud.test)?.processFilePath("from test")
     val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processFilePath("from additional-apks") }
     val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin
 
     // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false
     val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator
+    val roboDirectives = cli?.roboDirectives?.parseRoboDirectives() ?: androidGcloud.roboDirectives.parseRoboDirectives()
+    val roboScript = (cli?.roboScript ?: androidGcloud.roboScript)?.processFilePath("from roboScript")
     val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables
     val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull
     val otherFiles = (cli?.otherFiles ?: androidGcloud.otherFiles).map { (devicePath, filePath) ->
@@ -104,9 +107,21 @@ class AndroidArgs(
             "Option num-uniform-shards cannot be specified along with max-test-shards. Use only one of them"
         )
 
+        if (!(isRoboTest xor isInstrumentationTest)) throw FlankFatalError(
+            "One of following options must be specified [test, robo-directives, robo-script]."
+        )
+
+        // Using both roboDirectives and roboScript may hang test execution on FTL
+        if (roboDirectives.isNotEmpty() && roboScript != null) throw FlankFatalError(
+            "Options robo-directives and robo-script are mutually exclusive, use only one of them."
+        )
+
         assertCommonProps(this)
     }
 
+    val isInstrumentationTest get() = testApk != null
+    val isRoboTest get() = roboDirectives.isNotEmpty() || roboScript != null
+
     private fun assertDeviceSupported(device: Device) {
         when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) {
             SupportedDeviceConfig -> {
@@ -142,7 +157,9 @@ AndroidArgs
       num-uniform-shards: $numUniformShards
       test-runner-class: $testRunnerClass
       test-targets:${listToString(testTargets)}
-      device:${devicesToString(devices)}
+      robo-directives:${objectsToString(roboDirectives)}
+      robo-script: $roboScript
+      device:${objectsToString(devices)}
       num-flaky-test-attempts: $flakyTestAttempts
 
     flank:

test_runner/src/main/kotlin/ftl/args/ArgsToString.kt

@@ -1,7 +1,6 @@
 package ftl.args
 
 import ftl.args.yml.AppTestPair
-import ftl.config.Device
 
 private const val NEW_LINE = '\n'
 
@@ -19,9 +18,9 @@ object ArgsToString {
             .joinToString("\n") { dir -> "        - $dir" }
     }
 
-    fun devicesToString(devices: List<Device?>?): String {
-        if (devices.isNullOrEmpty()) return ""
-        return NEW_LINE + devices.filterNotNull()
+    fun objectsToString(objects: List<Any?>?): String {
+        if (objects.isNullOrEmpty()) return ""
+        return NEW_LINE + objects.filterNotNull()
             .joinToString("\n") { "$it" }
     }