From 6dbf8be56b275d77e5c88f41bf7df0667fb547a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Thu, 15 Apr 2021 09:59:34 +0200 Subject: [PATCH] test: Add 3dmark instrumented test (#1783) Fixes #1771 The instrumented test based on ui-automator should run a benchmark on the tested device. The benchmark must be installed on the tested device. ## Goal Running the benchmark on FTL device and obtaining the benchmark results. ## Config https://github.com/Flank/flank/blob/1771_ftl_benchmarks/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml ## Details * The benchmark `apk` file is specified using `additional-apks` option * The 3dmark benchmark results will be pulled from device dir `/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/` ## Test Plan > How do we know the code works? copy * `Geekbench 5_v5.3.2.apk` * `3dmarkandroid-v2-1-4726.apk` to `test_artifacts/1771_ftl_benchmarks/apk/benchmark/` ```bash . .env buildFlankScripts flankScripts assemble app --artifact benchmark cd test_runner flank android run -c="./src/test/kotlin/ftl/fixtures/benchmark/*.yml" ``` # Devices * Pixel 5e (physical) - API 30 * NexusLowRes (virtual) - API 30 * NexusLowResEmulator (emulator) - API 30 # Results ## 3dmarkandroid-v2-1-4726.apk * Pixel 5e (physical) - API 30 - [passed](https://gist.github.com/jan-gogo/794a01631645f7a1549f82334eb2fba4) https://github.com/Flank/flank/pull/1783/commits/43dfc014fc3f23c309aa81de8ac0af104b104a0c [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/4704820152614748057/executions/bs.458fb96e63f99c51/videos) * NexusLowRes (virtual) - API 30 - [failed](https://gist.github.com/jan-gogo/e3e4e7d80b1b65d550300c1ecb75498b) https://github.com/Flank/flank/pull/1783/commits/180e154e43546958d007417b4ab3bdb879880786 * NexusLowResEmulator (emulator) - API 30 - [freezed](https://gist.github.com/jan-gogo/683f1477b876649416d4366f6d1b005f) https://github.com/Flank/flank/pull/1783/commits/bc1a2f6a13b55e2cb6e3d0e6e234311b6eb2c942 ## Geekbench - [all passed](https://gist.github.com/jan-gogo/64700f34a5f7511aeecffe6da44a7d07) https://github.com/Flank/flank/pull/1783/commits/836e6221df89f996887a1fa6bcdf21d31c2e4a52 [firebase](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803) * Pixel 5e (physical) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.e9ea758006f16bee/videos) * NexusLowRes (virtual) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.7ee7ba87833f43ab/videos) * NexusLowResEmulator (emulator) - API 30 - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/7399025084907015803/executions/bs.2d604e5ac73159aa/videos) ## PassMark * NexusLowRes (virtual) - API 30 * NexusLowResEmulator (emulator) - API 30 - starting but failing during the benchmark - [video](https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5356711449268917714/executions/bs.a612c1ef9c1df07/videos) https://github.com/Flank/flank/pull/1783/commits/b239766ffa471ab4055522aa992227e44302def9 ## Checklist - [ ] Documented --- .../main/kotlin/flank/common/PathHelper.kt | 7 +- flank-scripts/build.gradle.kts | 2 +- .../ops/assemble/android/BuildBenchmark.kt | 30 ++++ .../ops/assemble/android/RunAndroidOps.kt | 1 + test_projects/android/benchmark/README.md | 25 ++++ test_projects/android/benchmark/build.gradle | 45 ++++++ .../android/benchmark/proguard-rules.pro | 21 +++ .../com/example/test/benchmark/Run3DMark.kt | 130 ++++++++++++++++++ .../example/test/benchmark/RunGeekbench.kt | 82 +++++++++++ .../com/example/test/benchmark/RunPassMark.kt | 117 ++++++++++++++++ .../benchmark/src/main/AndroidManifest.xml | 3 + test_projects/android/settings.gradle | 7 +- .../fixtures/benchmark/3dmark-NexusLowRes.yml | 16 +++ .../benchmark/3dmark-NexusLowResEmulator.yml | 16 +++ .../ftl/fixtures/benchmark/3dmark-all.yml | 24 ++++ .../ftl/fixtures/benchmark/3dmark-redfin.yml | 16 +++ .../benchmark/geekbench-NexusLowRes.yml | 16 +++ .../geekbench-NexusLowResEmulator.yml | 16 +++ .../ftl/fixtures/benchmark/geekbench-all.yml | 24 ++++ .../fixtures/benchmark/geekbench-redfin.yml | 16 +++ .../ftl/fixtures/benchmark/passmark-all.yml | 24 ++++ 21 files changed, 631 insertions(+), 7 deletions(-) create mode 100644 flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt create mode 100644 test_projects/android/benchmark/README.md create mode 100644 test_projects/android/benchmark/build.gradle create mode 100644 test_projects/android/benchmark/proguard-rules.pro create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt create mode 100644 test_projects/android/benchmark/src/main/AndroidManifest.xml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml diff --git a/common/src/main/kotlin/flank/common/PathHelper.kt b/common/src/main/kotlin/flank/common/PathHelper.kt index 00fc869d1b..6b3b81a617 100644 --- a/common/src/main/kotlin/flank/common/PathHelper.kt +++ b/common/src/main/kotlin/flank/common/PathHelper.kt @@ -19,9 +19,8 @@ fun Path.isRoot() = Files.exists(Paths.get(toString(), "settings.gradle.kts")) val testProjectsPath = Paths.get(rootDirectoryPathString, "test_projects").toString() val androidTestProjectsPath = Paths.get(testProjectsPath, "android").toString() val iOSTestProjectsPath = Paths.get(testProjectsPath, "ios").toString() -val flankFixturesTmpPath = - Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures", "tmp").toString() -val flankFixturesIosTmpPath = - Paths.get(flankFixturesTmpPath, "ios").toString() +val flankFixturesPath = Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures").toString() +val flankFixturesTmpPath = Paths.get(flankFixturesPath, "tmp").toString() +val flankFixturesIosTmpPath = Paths.get(flankFixturesTmpPath, "ios").toString() val flankCommonRootPathString = Paths.get(rootDirectoryPathString, "common").toString() diff --git a/flank-scripts/build.gradle.kts b/flank-scripts/build.gradle.kts index c99f5bd703..53b650ec59 100644 --- a/flank-scripts/build.gradle.kts +++ b/flank-scripts/build.gradle.kts @@ -26,7 +26,7 @@ shadowJar.apply { } } // .. -version = "1.9.10" +version = "1.9.11" group = "com.github.flank" application { diff --git a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt new file mode 100644 index 0000000000..439a82c923 --- /dev/null +++ b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt @@ -0,0 +1,30 @@ +package flank.scripts.ops.assemble.android + +import flank.common.androidTestProjectsPath +import flank.common.flankFixturesTmpPath +import flank.scripts.utils.createGradleCommand +import flank.scripts.utils.runCommand +import java.nio.file.Paths + +// TODO Design common abstraction for building apks and coping artifacts, add java doc. + +private const val BENCHMARK = "benchmark" + +fun AndroidBuildConfiguration.buildBenchmark() { + if (artifacts.canExecute(BENCHMARK).not()) return + createGradleCommand( + workingDir = androidTestProjectsPath, + options = listOf( + "-p", + androidTestProjectsPath, + "$BENCHMARK:assembleDebugAndroidTest" + ) + ).runCommand() + + if (copy) copyApks() +} + +private fun copyApks() { + val outputDir = Paths.get(flankFixturesTmpPath, "apk", BENCHMARK).toString() + Paths.get(androidTestProjectsPath, BENCHMARK).toFile().findApks().copyApksToPath(outputDir) +} diff --git a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt index deb1f48d9b..1e7f533b31 100644 --- a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt +++ b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt @@ -7,4 +7,5 @@ fun AndroidBuildConfiguration.runAndroidBuild() = takeIf { generate }?.let { buildMultiModulesApks() buildCucumberSampleApp() buildManyTestsApk() + buildBenchmark() } diff --git a/test_projects/android/benchmark/README.md b/test_projects/android/benchmark/README.md new file mode 100644 index 0000000000..8636559ae7 --- /dev/null +++ b/test_projects/android/benchmark/README.md @@ -0,0 +1,25 @@ +# Automated benchmark + +Instrumented test based on ui-automator responsible for running benchmark on tested device + +## Benchmarks + +* `3dmarkandroid-v2-1-4726.apk` +* `Geekbench 5_v5.3.2.apk` - TODO + +## Devices + +* Pixel 5e (physical) - API 30 +* NexusLowRes (virtual) - API 30 +* NexusLowResEmulator (emulator) - API 30 + +## Table + +``` +┌─────────────────────┬────────────────────┬─────────────────────────────────────────┬──────────┬─────────────┬────────────────────────────────────────────┬─────────────────────┐ +│ MODEL_ID │ MAKE │ MODEL_NAME │ FORM │ RESOLUTION │ OS_VERSION_IDS │ TAGS │ +├─────────────────────┼────────────────────┼─────────────────────────────────────────┼──────────┼─────────────┼────────────────────────────────────────────┼─────────────────────┤ +│ NexusLowRes │ Generic │ Low-resolution MDPI phone │ VIRTUAL │ 640 x 360 │ 23, 24, 25, 26, 27, 28, 29, 30 │ beta=30 │ +│ NexusLowResEmulator │ Generic │ Low-resolution MDPI phone │ EMULATOR │ 640 x 360 │ 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 │ private, alpha │ +│ redfin │ Google │ Pixel 5e │ PHYSICAL │ 2340 x 1080 │ 30 │ │ +``` diff --git a/test_projects/android/benchmark/build.gradle b/test_projects/android/benchmark/build.gradle new file mode 100644 index 0000000000..ef47451dff --- /dev/null +++ b/test_projects/android/benchmark/build.gradle @@ -0,0 +1,45 @@ +apply plugin: 'com.android.library' +apply plugin: 'kotlin-android' +apply plugin: 'kotlin-android-extensions' + +android { + compileSdkVersion 30 + buildToolsVersion "30.0.1" + + defaultConfig { + minSdkVersion 22 + targetSdkVersion 30 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation 'androidx.core:core-ktx:1.2.0' + implementation 'androidx.appcompat:appcompat:1.1.0' + implementation 'com.google.android.material:material:1.1.0' + implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + testImplementation 'junit:junit:4.+' + androidTestUtil 'androidx.test:orchestrator:1.2.0' + androidTestImplementation 'androidx.test.ext:junit:1.1.1' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0' +} diff --git a/test_projects/android/benchmark/proguard-rules.pro b/test_projects/android/benchmark/proguard-rules.pro new file mode 100644 index 0000000000..481bb43481 --- /dev/null +++ b/test_projects/android/benchmark/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt new file mode 100644 index 0000000000..c724ba892a --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -0,0 +1,130 @@ +package com.example.test.benchmark + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + +private const val DOWNLOAD_TIMEOUT = 5 * 60 * 1000L + + +@RunWith(AndroidJUnit4::class) +class Run3DMark { + + private object Text { + const val search = "Search" + const val ok = "OK" + const val allow = "Allow" + const val permTitle = "Before we start…" + const val appName = "3DMark" + const val benchmarkType = "SLING SHOT" + } + + private object Res { + const val centerLayout = "com.futuremark.dmandroid.application:id/flm_pager_benchmarks" + const val centerLayoutChild = "com.futuremark.dmandroid.application:id/flm_cl_root" + const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" + const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" + const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" + const val scoreDetails = + "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" + } + + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for 3d mark launcher icon + wait(Until.hasObject(By.text(Text.appName)), TIMEOUT) + + waitForIdle(5000) + + // Click 3d mark launcher icon + findObject(UiSelector().text(Text.appName)).click() + + waitForIdle(5000) + + // Check permissions dialog + if (findObject(UiSelector().text(Text.permTitle)).exists()) { + findObject(UiSelector().text(Text.ok)).click() + findObject(UiSelector().text(Text.allow)).click() + } + + waitForIdle(5000) + Thread.sleep(4000) + + // Skip tutorial if needed + findObject(UiSelector().resourceId(Res.btnSkip)).apply { + if (exists()) click() + } + + waitForIdle(5000) + Thread.sleep(2000) + + // Swipe to the sling shot benchmark if needed + var swipeCount = 0 + while (findObject(UiSelector().text(Text.benchmarkType)).exists().not()) { + findObject(UiSelector().resourceId(Res.centerLayoutChild)).apply { + with(visibleBounds) { swipe(right - 80, centerY(), left + 10, centerY(), 10) } + } + waitForIdle(2000) + Thread.sleep(1500) + if (swipeCount++ >= 4) Assert.fail("Max swipe count exceeded") + } + waitForIdle(2000) + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + waitForIdle(5000) + + // Settings fab is not visible if the additional software is not installed + if (findObject(UiSelector().resourceId(Res.fabSettings)).exists().not()) { + + // Install additional software + findObject(UiSelector().resourceId(Res.fabBenchmark)).click() + + // Wait until download finish + wait( + Until.hasObject(By.res(Res.fabSettings)), + DOWNLOAD_TIMEOUT + ) + } + + // Run benchmark + findObject(UiSelector().resourceId(Res.fabBenchmark)).click() + + + waitForIdle(5000) + + // Wait until benchmark finish + wait(Until.hasObject(By.res(Res.scoreDetails)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + Assert.assertTrue(findObject(UiSelector().resourceId(Res.scoreDetails)).exists()) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt new file mode 100644 index 0000000000..c768ea89b8 --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt @@ -0,0 +1,82 @@ +package com.example.test.benchmark + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + + +@RunWith(AndroidJUnit4::class) +class RunGeekbench { + + private object Text { + const val search = "Search" + const val appName = "Geek" + const val accept = "ACCEPT" + const val benchmarkType = "RUN CPU BENCHMARK" + const val runningDialog = "Geekbench 5" + const val results = "Benchmark Results" + } + + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for 3d mark launcher icon + wait(Until.hasObject(By.textContains(Text.appName)), TIMEOUT) + + waitForIdle(5000) + + // Click 3d mark launcher icon + findObject(UiSelector().textContains(Text.appName)).click() + + waitForIdle(5000) + + Thread.sleep(1000) + + // Check permissions dialog + findObject(UiSelector().text(Text.accept)).run { + if (exists()) click() + } + + waitForIdle(5000) + Thread.sleep(2000) + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + waitForIdle(5000) + Thread.sleep(2000) + + // Wait until benchmark finish + wait(Until.hasObject(By.text(Text.runningDialog)), TIMEOUT) + wait(Until.gone(By.text(Text.runningDialog)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + wait(Until.hasObject(By.text(Text.results)), TIMEOUT) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt new file mode 100644 index 0000000000..424ccc0f69 --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt @@ -0,0 +1,117 @@ +package com.example.test.benchmark + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + + +@RunWith(AndroidJUnit4::class) +class RunPassMark { + + private object Res { + const val splash = "android:id/content" + } + + private object Text { + const val search = "Search" + const val appName = "Perform" + const val accept = "Continue" + const val ok = "OK" + const val next = "NEXT" + const val gotIt = "GOT IT!" + const val cancel = "CANCEL" + const val benchmarkType = "RUN BENCHMARK" + const val results = "Benchmark Results" + } + + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for app launcher icon + wait(Until.hasObject(By.textContains(Text.appName)), TIMEOUT) + + waitForIdle(5000) + + // Click launcher icon + findObject(UiSelector().textContains(Text.appName)).click() + + waitForIdle(5000) + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + Thread.sleep(1000) + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + // Check permissions dialog + findObject(UiSelector().text(Text.accept)).run { + if (exists()) click() + } + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + // Touch to continue... + findObject(UiSelector().resourceId(Res.splash)).click() + + // Check tooltip dialog + findObject(UiSelector().text(Text.next)).run { + if (exists()) click() + } + + // Check tooltip dialog + findObject(UiSelector().text(Text.gotIt)).run { + if (exists()) click() + } + + waitForIdle(5000) + Thread.sleep(2000) + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + waitForIdle(5000) + Thread.sleep(2000) + + // Wait until benchmark finish + wait(Until.hasObject(By.text(Text.cancel)), TIMEOUT) + wait(Until.gone(By.text(Text.cancel)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + wait(Until.hasObject(By.text(Text.results)), BENCHMARK_TIMEOUT) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} diff --git a/test_projects/android/benchmark/src/main/AndroidManifest.xml b/test_projects/android/benchmark/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..4252803134 --- /dev/null +++ b/test_projects/android/benchmark/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/test_projects/android/settings.gradle b/test_projects/android/settings.gradle index d312a45194..ab522c1a45 100644 --- a/test_projects/android/settings.gradle +++ b/test_projects/android/settings.gradle @@ -30,5 +30,8 @@ include ':app', ':dir3:testModule' -include ':cucumber_sample_app' -include ':cucumber_sample_app:cucumber-android', ':cucumber_sample_app:cukeulator' +include ':cucumber_sample_app', + ':cucumber_sample_app:cucumber-android', + ':cucumber_sample_app:cukeulator' + +include ':benchmark' diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml new file mode 100644 index 0000000000..f307031531 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml new file mode 100644 index 0000000000..1459c7ac80 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml new file mode 100644 index 0000000000..9b357fd25c --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml @@ -0,0 +1,24 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml new file mode 100644 index 0000000000..ca226eb396 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml new file mode 100644 index 0000000000..203222fed9 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml new file mode 100644 index 0000000000..fea5c2cb92 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml new file mode 100644 index 0000000000..013c994051 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml @@ -0,0 +1,24 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml new file mode 100644 index 0000000000..6cfd916346 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml @@ -0,0 +1,16 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml new file mode 100644 index 0000000000..96fbae5e8a --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml @@ -0,0 +1,24 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/PassMark_PerformanceTest_10.0.1001-fileplanet.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait +# - model: redfin +# version: 30 +# locale: en +# orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunPassMark"