test: Add 3dmark instrumented test (#1783)

Fixes #1771 

The instrumented test based on ui-automator should run a benchmark on the tested device.
The benchmark must be installed on the tested device.

## Goal

Running the benchmark on FTL device and obtaining the benchmark results.

## Config

https://github.com/Flank/flank/blob/1771_ftl_benchmarks/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml

## Details

* The benchmark `apk` file is specified using `additional-apks` option
* The 3dmark benchmark results will be pulled from device dir `/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/`

## Test Plan
> How do we know the code works?

copy
* `Geekbench 5_v5.3.2.apk`
* `3dmarkandroid-v2-1-4726.apk`

to `test_artifacts/1771_ftl_benchmarks/apk/benchmark/`

```bash
. .env
buildFlankScripts
flankScripts assemble app --artifact benchmark
cd test_runner
flank android run -c="./src/test/kotlin/ftl/fixtures/benchmark/*.yml"
```

# Devices

* Pixel 5e (physical) - API 30
* NexusLowRes (virtual) - API 30
* NexusLowResEmulator (emulator) - API 30

# Results

## 3dmarkandroid-v2-1-4726.apk

* Pixel 5e (physical) - API 30 - [passed](https://gist.github.com/jan-gogo/794a01631645f7a1549f82334eb2fba4) 43dfc01 [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/4704820152614748057/executions/bs.458fb96e63f99c51/videos)
* NexusLowRes (virtual) - API 30 - [failed](https://gist.github.com/jan-gogo/e3e4e7d80b1b65d550300c1ecb75498b) 180e154
* NexusLowResEmulator (emulator) - API 30 - [freezed](https://gist.github.com/jan-gogo/683f1477b876649416d4366f6d1b005f) bc1a2f6

## Geekbench - [all passed](https://gist.github.com/jan-gogo/64700f34a5f7511aeecffe6da44a7d07) 836e622 [firebase](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803)

* Pixel 5e (physical) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.e9ea758006f16bee/videos)
* NexusLowRes (virtual) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.7ee7ba87833f43ab/videos)
* NexusLowResEmulator (emulator) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.2d604e5ac73159aa/videos)

## PassMark

* NexusLowRes (virtual) - API 30
* NexusLowResEmulator (emulator) - API 30 - starting but failing during the benchmark - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5356711449268917714/executions/bs.a612c1ef9c1df07/videos) b239766

## Checklist

- [ ] Documented

common/src/main/kotlin/flank/common/PathHelper.kt

@@ -19,9 +19,8 @@ fun Path.isRoot() = Files.exists(Paths.get(toString(), "settings.gradle.kts"))
 val testProjectsPath = Paths.get(rootDirectoryPathString, "test_projects").toString()
 val androidTestProjectsPath = Paths.get(testProjectsPath, "android").toString()
 val iOSTestProjectsPath = Paths.get(testProjectsPath, "ios").toString()
-val flankFixturesTmpPath =
-    Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures", "tmp").toString()
-val flankFixturesIosTmpPath =
-    Paths.get(flankFixturesTmpPath, "ios").toString()
+val flankFixturesPath = Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures").toString()
+val flankFixturesTmpPath = Paths.get(flankFixturesPath, "tmp").toString()
+val flankFixturesIosTmpPath = Paths.get(flankFixturesTmpPath, "ios").toString()
 
 val flankCommonRootPathString = Paths.get(rootDirectoryPathString, "common").toString()

flank-scripts/build.gradle.kts

@@ -26,7 +26,7 @@ shadowJar.apply {
     }
 }
 // <breaking change>.<feature added>.<fix/minor change>
-version = "1.9.10"
+version = "1.9.11"
 group = "com.github.flank"
 
 application {

flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt

@@ -0,0 +1,30 @@
+package flank.scripts.ops.assemble.android
+
+import flank.common.androidTestProjectsPath
+import flank.common.flankFixturesTmpPath
+import flank.scripts.utils.createGradleCommand
+import flank.scripts.utils.runCommand
+import java.nio.file.Paths
+
+// TODO Design common abstraction for building apks and coping artifacts, add java doc.
+
+private const val BENCHMARK = "benchmark"
+
+fun AndroidBuildConfiguration.buildBenchmark() {
+    if (artifacts.canExecute(BENCHMARK).not()) return
+    createGradleCommand(
+        workingDir = androidTestProjectsPath,
+        options = listOf(
+            "-p",
+            androidTestProjectsPath,
+            "$BENCHMARK:assembleDebugAndroidTest"
+        )
+    ).runCommand()
+
+    if (copy) copyApks()
+}
+
+private fun copyApks() {
+    val outputDir = Paths.get(flankFixturesTmpPath, "apk", BENCHMARK).toString()
+    Paths.get(androidTestProjectsPath, BENCHMARK).toFile().findApks().copyApksToPath(outputDir)
+}

flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt

@@ -7,4 +7,5 @@ fun AndroidBuildConfiguration.runAndroidBuild() = takeIf { generate }?.let {
     buildMultiModulesApks()
     buildCucumberSampleApp()
     buildManyTestsApk()
+    buildBenchmark()
 }

test_projects/android/benchmark/README.md

@@ -0,0 +1,25 @@
+# Automated benchmark
+
+Instrumented test based on ui-automator responsible for running benchmark on tested device
+
+## Benchmarks
+
+* `3dmarkandroid-v2-1-4726.apk`
+* `Geekbench 5_v5.3.2.apk` - TODO
+
+## Devices
+
+* Pixel 5e (physical) - API 30
+* NexusLowRes (virtual) - API 30
+* NexusLowResEmulator (emulator) - API 30
+
+## Table
+
+```
+┌─────────────────────┬────────────────────┬─────────────────────────────────────────┬──────────┬─────────────┬────────────────────────────────────────────┬─────────────────────┐
+│      MODEL_ID       │        MAKE        │               MODEL_NAME                │   FORM   │ RESOLUTION  │               OS_VERSION_IDS               │        TAGS         │
+├─────────────────────┼────────────────────┼─────────────────────────────────────────┼──────────┼─────────────┼────────────────────────────────────────────┼─────────────────────┤
+│ NexusLowRes         │ Generic            │ Low-resolution MDPI phone               │ VIRTUAL  │ 640 x 360   │ 23, 24, 25, 26, 27, 28, 29, 30             │ beta=30             │
+│ NexusLowResEmulator │ Generic            │ Low-resolution MDPI phone               │ EMULATOR │ 640 x 360   │ 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 │ private, alpha      │
+│ redfin              │ Google             │ Pixel 5e                                │ PHYSICAL │ 2340 x 1080 │                     30                     │                     │
+```

test_projects/android/benchmark/build.gradle

@@ -0,0 +1,45 @@
+apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlin-android-extensions'
+
+android {
+    compileSdkVersion 30
+    buildToolsVersion "30.0.1"
+
+    defaultConfig {
+        minSdkVersion 22
+        targetSdkVersion 30
+        versionCode 1
+        versionName "1.0"
+
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+    kotlinOptions {
+        jvmTarget = '1.8'
+    }
+}
+
+dependencies {
+
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'androidx.core:core-ktx:1.2.0'
+    implementation 'androidx.appcompat:appcompat:1.1.0'
+    implementation 'com.google.android.material:material:1.1.0'
+    implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+    testImplementation 'junit:junit:4.+'
+    androidTestUtil 'androidx.test:orchestrator:1.2.0'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.1'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
+    androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0'
+}

test_projects/android/benchmark/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt

@@ -0,0 +1,130 @@
+package com.example.test.benchmark
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.uiautomator.By
+import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.UiSelector
+import androidx.test.uiautomator.Until
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+
+
+private const val TIMEOUT = 10_000L
+
+private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L
+
+private const val DOWNLOAD_TIMEOUT = 5 * 60 * 1000L
+
+
+@RunWith(AndroidJUnit4::class)
+class Run3DMark {
+
+    private object Text {
+        const val search = "Search"
+        const val ok = "OK"
+        const val allow = "Allow"
+        const val permTitle = "Before we start…"
+        const val appName = "3DMark"
+        const val benchmarkType = "SLING SHOT"
+    }
+
+    private object Res {
+        const val centerLayout = "com.futuremark.dmandroid.application:id/flm_pager_benchmarks"
+        const val centerLayoutChild = "com.futuremark.dmandroid.application:id/flm_cl_root"
+        const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip"
+        const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark"
+        const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings"
+        const val scoreDetails =
+            "com.futuremark.dmandroid.application:id/flm_ll_score_details_container"
+    }
+
+    @Test
+    fun run() {
+
+        UiDevice.getInstance(
+            InstrumentationRegistry.getInstrumentation()
+        ).run {
+            // Start from the home screen
+            pressHome()
+
+            // Open apps menu on pixel launcher
+            findObject(UiSelector().descriptionContains(Text.search)).apply {
+                swipe(0, visibleBounds.centerY(), 0, 0, 10)
+            }
+
+            // Wait for 3d mark launcher icon
+            wait(Until.hasObject(By.text(Text.appName)), TIMEOUT)
+
+            waitForIdle(5000)
+
+            // Click 3d mark launcher icon
+            findObject(UiSelector().text(Text.appName)).click()
+
+            waitForIdle(5000)
+
+            // Check permissions dialog
+            if (findObject(UiSelector().text(Text.permTitle)).exists()) {
+                findObject(UiSelector().text(Text.ok)).click()
+                findObject(UiSelector().text(Text.allow)).click()
+            }
+
+            waitForIdle(5000)
+            Thread.sleep(4000)
+
+            // Skip tutorial if needed
+            findObject(UiSelector().resourceId(Res.btnSkip)).apply {
+                if (exists()) click()
+            }
+
+            waitForIdle(5000)
+            Thread.sleep(2000)
+
+            // Swipe to the sling shot benchmark if needed
+            var swipeCount = 0
+            while (findObject(UiSelector().text(Text.benchmarkType)).exists().not()) {
+                findObject(UiSelector().resourceId(Res.centerLayoutChild)).apply {
+                    with(visibleBounds) { swipe(right - 80, centerY(), left + 10, centerY(), 10) }
+                }
+                waitForIdle(2000)
+                Thread.sleep(1500)
+                if (swipeCount++ >= 4) Assert.fail("Max swipe count exceeded")
+            }
+            waitForIdle(2000)
+
+            // Choose proper benchmark screen
+            findObject(UiSelector().text(Text.benchmarkType)).click()
+
+            waitForIdle(5000)
+
+            // Settings fab is not visible if the additional software is not installed
+            if (findObject(UiSelector().resourceId(Res.fabSettings)).exists().not()) {
+
+                // Install additional software
+                findObject(UiSelector().resourceId(Res.fabBenchmark)).click()
+
+                // Wait until download finish
+                wait(
+                    Until.hasObject(By.res(Res.fabSettings)),
+                    DOWNLOAD_TIMEOUT
+                )
+            }
+
+            // Run benchmark
+            findObject(UiSelector().resourceId(Res.fabBenchmark)).click()
+
+
+            waitForIdle(5000)
+
+            // Wait until benchmark finish
+            wait(Until.hasObject(By.res(Res.scoreDetails)), BENCHMARK_TIMEOUT)
+
+            // Assert that benchmark results screen is visible
+            Assert.assertTrue(findObject(UiSelector().resourceId(Res.scoreDetails)).exists())
+
+            // Make sure that results was recorded
+            Thread.sleep(5000)
+        }
+    }
+}

test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt

@@ -0,0 +1,82 @@
+package com.example.test.benchmark
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.uiautomator.By
+import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.UiSelector
+import androidx.test.uiautomator.Until
+import org.junit.Assert
+import org.junit.Test
+import org.junit.runner.RunWith
+
+
+private const val TIMEOUT = 10_000L
+
+private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L
+
+
+@RunWith(AndroidJUnit4::class)
+class RunGeekbench {
+
+    private object Text {
+        const val search = "Search"
+        const val appName = "Geek"
+        const val accept = "ACCEPT"
+        const val benchmarkType = "RUN CPU BENCHMARK"
+        const val runningDialog = "Geekbench 5"
+        const val results = "Benchmark Results"
+    }
+
+    @Test
+    fun run() {
+
+        UiDevice.getInstance(
+            InstrumentationRegistry.getInstrumentation()
+        ).run {
+            // Start from the home screen
+            pressHome()
+
+            // Open apps menu on pixel launcher
+            findObject(UiSelector().descriptionContains(Text.search)).apply {
+                swipe(0, visibleBounds.centerY(), 0, 0, 10)
+            }
+
+            // Wait for 3d mark launcher icon
+            wait(Until.hasObject(By.textContains(Text.appName)), TIMEOUT)
+
+            waitForIdle(5000)
+
+            // Click 3d mark launcher icon
+            findObject(UiSelector().textContains(Text.appName)).click()
+
+            waitForIdle(5000)
+
+            Thread.sleep(1000)
+
+            // Check permissions dialog
+            findObject(UiSelector().text(Text.accept)).run {
+                if (exists()) click()
+            }
+
+            waitForIdle(5000)
+            Thread.sleep(2000)
+
+            // Choose proper benchmark screen
+            findObject(UiSelector().text(Text.benchmarkType)).click()
+
+            waitForIdle(5000)
+            Thread.sleep(2000)
+
+            // Wait until benchmark finish
+            wait(Until.hasObject(By.text(Text.runningDialog)), TIMEOUT)
+            wait(Until.gone(By.text(Text.runningDialog)), BENCHMARK_TIMEOUT)
+
+            // Assert that benchmark results screen is visible
+            wait(Until.hasObject(By.text(Text.results)), TIMEOUT)
+
+            // Make sure that results was recorded
+            Thread.sleep(5000)
+        }
+    }
+}