From 82c2a6ca85bf98e7c9023dd800d961d60fb5755d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 7 Apr 2021 09:39:57 +0200 Subject: [PATCH 01/10] Add 3dmark instrumented test --- .../main/kotlin/flank/common/PathHelper.kt | 7 +- .../ops/assemble/android/BuildBenchmark.kt | 31 ++++++ .../ops/assemble/android/RunAndroidOps.kt | 1 + test_projects/android/benchmark/README.md | 14 +++ test_projects/android/benchmark/build.gradle | 46 ++++++++ .../android/benchmark/proguard-rules.pro | 21 ++++ .../example/benchmarkroboscript/Run3DMark.kt | 104 ++++++++++++++++++ .../benchmark/src/main/AndroidManifest.xml | 3 + test_projects/android/settings.gradle | 7 +- .../test/kotlin/ftl/fixtures/benchmark.yml | 12 ++ 10 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt create mode 100644 test_projects/android/benchmark/README.md create mode 100644 test_projects/android/benchmark/build.gradle create mode 100644 test_projects/android/benchmark/proguard-rules.pro create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt create mode 100644 test_projects/android/benchmark/src/main/AndroidManifest.xml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml diff --git a/common/src/main/kotlin/flank/common/PathHelper.kt b/common/src/main/kotlin/flank/common/PathHelper.kt index 00fc869d1b..6b3b81a617 100644 --- a/common/src/main/kotlin/flank/common/PathHelper.kt +++ b/common/src/main/kotlin/flank/common/PathHelper.kt @@ -19,9 +19,8 @@ fun Path.isRoot() = Files.exists(Paths.get(toString(), "settings.gradle.kts")) val testProjectsPath = Paths.get(rootDirectoryPathString, "test_projects").toString() val androidTestProjectsPath = Paths.get(testProjectsPath, "android").toString() val iOSTestProjectsPath = Paths.get(testProjectsPath, "ios").toString() -val flankFixturesTmpPath = - Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures", "tmp").toString() -val flankFixturesIosTmpPath = - Paths.get(flankFixturesTmpPath, "ios").toString() +val flankFixturesPath = Paths.get(rootDirectoryPathString, "test_runner", "src", "test", "kotlin", "ftl", "fixtures").toString() +val flankFixturesTmpPath = Paths.get(flankFixturesPath, "tmp").toString() +val flankFixturesIosTmpPath = Paths.get(flankFixturesTmpPath, "ios").toString() val flankCommonRootPathString = Paths.get(rootDirectoryPathString, "common").toString() diff --git a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt new file mode 100644 index 0000000000..2e6771fabb --- /dev/null +++ b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt @@ -0,0 +1,31 @@ +package flank.scripts.ops.assemble.android + +import flank.common.androidTestProjectsPath +import flank.common.flankFixturesTmpPath +import flank.scripts.utils.createGradleCommand +import flank.scripts.utils.runCommand +import java.nio.file.Paths + +// TODO Design common abstraction for building apks and coping artifacts, add java doc. + +private const val BENCHMARK = "benchmark" + +fun AndroidBuildConfiguration.buildBenchmark() { + if (artifacts.canExecute(BENCHMARK).not()) return + createGradleCommand( + workingDir = androidTestProjectsPath, + options = listOf( + "-p", + androidTestProjectsPath, + "$BENCHMARK:assembleDebugAndroidTest" + ) + ).runCommand() + + if (copy) copyApks() +} + +private fun copyApks() { + val outputDir = Paths.get(flankFixturesTmpPath, "apk", BENCHMARK).toString() + Paths.get(androidTestProjectsPath, BENCHMARK).toFile().findApks().copyApksToPath(outputDir) +} + diff --git a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt index deb1f48d9b..1e7f533b31 100644 --- a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt +++ b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/RunAndroidOps.kt @@ -7,4 +7,5 @@ fun AndroidBuildConfiguration.runAndroidBuild() = takeIf { generate }?.let { buildMultiModulesApks() buildCucumberSampleApp() buildManyTestsApk() + buildBenchmark() } diff --git a/test_projects/android/benchmark/README.md b/test_projects/android/benchmark/README.md new file mode 100644 index 0000000000..38ee6e2564 --- /dev/null +++ b/test_projects/android/benchmark/README.md @@ -0,0 +1,14 @@ +# Automated benchmark + +Instrumented test based on ui-automator responsible for running benchmark on tested device + +## Benchmarks + +* `3dmarkandroid-v2-1-4726.apk` +* `Geekbench 5_v5.3.2.apk` - TODO + +## Devices + +* Pixel 5e (physical) - API 30 +* NexusLowRes (virtual) - API 30 +* NexusLowResEmulator (emulator) - API 30 diff --git a/test_projects/android/benchmark/build.gradle b/test_projects/android/benchmark/build.gradle new file mode 100644 index 0000000000..736cc4d618 --- /dev/null +++ b/test_projects/android/benchmark/build.gradle @@ -0,0 +1,46 @@ +apply plugin: 'com.android.library' +apply plugin: 'kotlin-android' +apply plugin: 'kotlin-android-extensions' + +android { + compileSdkVersion 30 + buildToolsVersion "30.0.1" + + defaultConfig { +// applicationId "com.example.benchmarkroboscript" + minSdkVersion 22 + targetSdkVersion 30 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation 'androidx.core:core-ktx:1.2.0' + implementation 'androidx.appcompat:appcompat:1.1.0' + implementation 'com.google.android.material:material:1.1.0' + implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + testImplementation 'junit:junit:4.+' + androidTestUtil 'androidx.test:orchestrator:1.2.0' + androidTestImplementation 'androidx.test.ext:junit:1.1.1' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0' +} diff --git a/test_projects/android/benchmark/proguard-rules.pro b/test_projects/android/benchmark/proguard-rules.pro new file mode 100644 index 0000000000..481bb43481 --- /dev/null +++ b/test_projects/android/benchmark/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt new file mode 100644 index 0000000000..edb13379bb --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt @@ -0,0 +1,104 @@ +package com.example.benchmarkroboscript + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + +private const val DOWNLOAD_TIMEOUT = 5 * 60 * 1000L + +private object Text { + const val search = "Search" + const val ok = "OK" + const val allow = "Allow" + const val permTitle = "Before we start…" + const val appName = "3DMark" + const val benchmarkType = "SLING SHOT" +} + +private object Res { + const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" + const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" + const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" + const val scoreDetails = "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" +} + + +@RunWith(AndroidJUnit4::class) +class Run3DMark { + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for 3d mark launcher icon + wait(Until.hasObject(By.text(Text.appName)), TIMEOUT) + + // Click 3d mark launcher icon + findObject(UiSelector().text(Text.appName)).click() + + // Check permissions dialog + if (findObject(UiSelector().text(Text.permTitle)).exists()) { + findObject(UiSelector().text(Text.ok)).click() + findObject(UiSelector().text(Text.allow)).click() + } + + Thread.sleep(3000) + + // Skip tutorial if needed + findObject(UiSelector().resourceId(Res.btnSkip)).apply { + if (exists()) click() + } + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + // Settings fab is not visible if the additional software is not installed + if (findObject(UiSelector().resourceId(Res.fabSettings)).exists().not()) { + + // Install additional software + findObject(UiSelector().resourceId(Res.fabBenchmark)).click() + + // Wait until download finish + wait( + Until.hasObject(By.res(Res.fabSettings)), + DOWNLOAD_TIMEOUT + ) + } + + // Run benchmark + findObject(UiSelector().resourceId(Res.fabBenchmark)).click() + + + Thread.sleep(3000) + + // Wait until benchmark finish + wait(Until.hasObject(By.res(Res.scoreDetails)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + Assert.assertTrue(findObject(UiSelector().resourceId(Res.scoreDetails)).exists()) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} \ No newline at end of file diff --git a/test_projects/android/benchmark/src/main/AndroidManifest.xml b/test_projects/android/benchmark/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..4252803134 --- /dev/null +++ b/test_projects/android/benchmark/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/test_projects/android/settings.gradle b/test_projects/android/settings.gradle index d312a45194..ab522c1a45 100644 --- a/test_projects/android/settings.gradle +++ b/test_projects/android/settings.gradle @@ -30,5 +30,8 @@ include ':app', ':dir3:testModule' -include ':cucumber_sample_app' -include ':cucumber_sample_app:cucumber-android', ':cucumber_sample_app:cukeulator' +include ':cucumber_sample_app', + ':cucumber_sample_app:cucumber-android', + ':cucumber_sample_app:cukeulator' + +include ':benchmark' diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml new file mode 100644 index 0000000000..3bad0b3962 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml @@ -0,0 +1,12 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait From 8eec0029ef9c9920b2c5112c6c784eebae5ca5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 7 Apr 2021 11:45:06 +0200 Subject: [PATCH 02/10] Update --- test_projects/android/benchmark/README.md | 11 +++++++++++ test_projects/android/benchmark/build.gradle | 1 - .../benchmark}/Run3DMark.kt | 2 +- .../src/test/kotlin/ftl/fixtures/benchmark.yml | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) rename test_projects/android/benchmark/src/androidTest/java/com/example/{benchmarkroboscript => test/benchmark}/Run3DMark.kt (98%) diff --git a/test_projects/android/benchmark/README.md b/test_projects/android/benchmark/README.md index 38ee6e2564..8636559ae7 100644 --- a/test_projects/android/benchmark/README.md +++ b/test_projects/android/benchmark/README.md @@ -12,3 +12,14 @@ Instrumented test based on ui-automator responsible for running benchmark on tes * Pixel 5e (physical) - API 30 * NexusLowRes (virtual) - API 30 * NexusLowResEmulator (emulator) - API 30 + +## Table + +``` +┌─────────────────────┬────────────────────┬─────────────────────────────────────────┬──────────┬─────────────┬────────────────────────────────────────────┬─────────────────────┐ +│ MODEL_ID │ MAKE │ MODEL_NAME │ FORM │ RESOLUTION │ OS_VERSION_IDS │ TAGS │ +├─────────────────────┼────────────────────┼─────────────────────────────────────────┼──────────┼─────────────┼────────────────────────────────────────────┼─────────────────────┤ +│ NexusLowRes │ Generic │ Low-resolution MDPI phone │ VIRTUAL │ 640 x 360 │ 23, 24, 25, 26, 27, 28, 29, 30 │ beta=30 │ +│ NexusLowResEmulator │ Generic │ Low-resolution MDPI phone │ EMULATOR │ 640 x 360 │ 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 │ private, alpha │ +│ redfin │ Google │ Pixel 5e │ PHYSICAL │ 2340 x 1080 │ 30 │ │ +``` diff --git a/test_projects/android/benchmark/build.gradle b/test_projects/android/benchmark/build.gradle index 736cc4d618..ef47451dff 100644 --- a/test_projects/android/benchmark/build.gradle +++ b/test_projects/android/benchmark/build.gradle @@ -7,7 +7,6 @@ android { buildToolsVersion "30.0.1" defaultConfig { -// applicationId "com.example.benchmarkroboscript" minSdkVersion 22 targetSdkVersion 30 versionCode 1 diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt similarity index 98% rename from test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt rename to test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt index edb13379bb..0119489ad3 100644 --- a/test_projects/android/benchmark/src/androidTest/java/com/example/benchmarkroboscript/Run3DMark.kt +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -1,4 +1,4 @@ -package com.example.benchmarkroboscript +package com.example.test.benchmark import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml index 3bad0b3962..777c50b162 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml @@ -1,6 +1,7 @@ gcloud: app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false directories-to-pull: - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" additional-apks: From 9626f3b3344f382187538bacb20cf43a08ea607b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 7 Apr 2021 13:09:02 +0200 Subject: [PATCH 03/10] Fix video recording --- test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml index 777c50b162..0cd66d25c0 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml @@ -11,3 +11,4 @@ gcloud: version: 30 locale: en orientation: portrait + record-video: true From dd7b2601e6dc6acb28938384952af1f11fb944ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 7 Apr 2021 18:00:57 +0200 Subject: [PATCH 04/10] fixes --- .../com/example/test/benchmark/Run3DMark.kt | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt index 0119489ad3..81a338dfff 100644 --- a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -27,6 +27,7 @@ private object Text { } private object Res { + const val tabLayout = "com.futuremark.dmandroid.application:id/flm_tab_layout_benchmarks" const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" @@ -53,24 +54,39 @@ class Run3DMark { // Wait for 3d mark launcher icon wait(Until.hasObject(By.text(Text.appName)), TIMEOUT) + waitForIdle(5000) + // Click 3d mark launcher icon findObject(UiSelector().text(Text.appName)).click() + waitForIdle(5000) + // Check permissions dialog if (findObject(UiSelector().text(Text.permTitle)).exists()) { findObject(UiSelector().text(Text.ok)).click() findObject(UiSelector().text(Text.allow)).click() } - Thread.sleep(3000) + waitForIdle(5000) + Thread.sleep(2000) // Skip tutorial if needed findObject(UiSelector().resourceId(Res.btnSkip)).apply { if (exists()) click() } + waitForIdle(5000) + + if (findObject(UiSelector().text(Text.benchmarkType)).exists().not()) { + findObject(UiSelector().resourceId(Res.tabLayout)).swipeLeft(10) + waitForIdle(1000) + } + // Choose proper benchmark screen findObject(UiSelector().text(Text.benchmarkType)).click() +// click(318, 104) + + waitForIdle(5000) // Settings fab is not visible if the additional software is not installed if (findObject(UiSelector().resourceId(Res.fabSettings)).exists().not()) { @@ -89,7 +105,7 @@ class Run3DMark { findObject(UiSelector().resourceId(Res.fabBenchmark)).click() - Thread.sleep(3000) + waitForIdle(5000) // Wait until benchmark finish wait(Until.hasObject(By.res(Res.scoreDetails)), BENCHMARK_TIMEOUT) @@ -101,4 +117,4 @@ class Run3DMark { Thread.sleep(5000) } } -} \ No newline at end of file +} From cc3db880fde6645e6b9329df0f1ab2a971c542ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 8 Apr 2021 08:42:18 +0200 Subject: [PATCH 05/10] fixes --- .../java/com/example/test/benchmark/Run3DMark.kt | 2 +- .../{benchmark.yml => benchmark-NexusLowRes.yml} | 0 .../ftl/fixtures/benchmark-NexusLowResEmulator.yml | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) rename test_runner/src/test/kotlin/ftl/fixtures/{benchmark.yml => benchmark-NexusLowRes.yml} (100%) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt index 81a338dfff..2ea3257459 100644 --- a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -68,7 +68,7 @@ class Run3DMark { } waitForIdle(5000) - Thread.sleep(2000) + Thread.sleep(4000) // Skip tutorial if needed findObject(UiSelector().resourceId(Res.btnSkip)).apply { diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowRes.yml similarity index 100% rename from test_runner/src/test/kotlin/ftl/fixtures/benchmark.yml rename to test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowRes.yml diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml new file mode 100644 index 0000000000..7092a21bdd --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml @@ -0,0 +1,14 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + record-video: true From 4d12ffd91dbd7952187d19df2507ecefe1a0fa5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 8 Apr 2021 11:42:16 +0200 Subject: [PATCH 06/10] fixes for redfin Pixel 5e --- .../com/example/test/benchmark/Run3DMark.kt | 23 +++++++++++++------ .../kotlin/ftl/fixtures/benchmark-redfin.yml | 14 +++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt index 2ea3257459..44cd48e1aa 100644 --- a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -27,11 +27,13 @@ private object Text { } private object Res { - const val tabLayout = "com.futuremark.dmandroid.application:id/flm_tab_layout_benchmarks" + const val centerLayout = "com.futuremark.dmandroid.application:id/flm_pager_benchmarks" + const val centerLayoutChild = "com.futuremark.dmandroid.application:id/flm_cl_root" const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" - const val scoreDetails = "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" + const val scoreDetails = + "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" } @@ -76,15 +78,22 @@ class Run3DMark { } waitForIdle(5000) - - if (findObject(UiSelector().text(Text.benchmarkType)).exists().not()) { - findObject(UiSelector().resourceId(Res.tabLayout)).swipeLeft(10) - waitForIdle(1000) + Thread.sleep(2000) + + // Swipe to the sling shot benchmark if needed + var swipeCount = 0 + while (findObject(UiSelector().text(Text.benchmarkType)).exists().not()) { + findObject(UiSelector().resourceId(Res.centerLayoutChild)).apply { + with(visibleBounds) { swipe(right - 80, centerY(), left + 10, centerY(), 10) } + } + waitForIdle(2000) + Thread.sleep(1500) + if (swipeCount++ >= 4) Assert.fail("Max swipe count exceeded") } + waitForIdle(2000) // Choose proper benchmark screen findObject(UiSelector().text(Text.benchmarkType)).click() -// click(318, 104) waitForIdle(5000) diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml new file mode 100644 index 0000000000..40609b23b4 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml @@ -0,0 +1,14 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true From 42485d40843969f74427d78a77ef3bab2bdaf1ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 8 Apr 2021 13:20:21 +0200 Subject: [PATCH 07/10] fixes for redfin Pixel 5e --- .../com/example/test/benchmark/Run3DMark.kt | 39 ++++----- .../example/test/benchmark/RunGeekbench.kt | 82 +++++++++++++++++++ .../3dmark-NexusLowRes.yml} | 3 + .../3dmark-NexusLowResEmulator.yml} | 3 + .../3dmark-redfin.yml} | 3 + .../benchmark/geekbench-NexusLowRes.yml | 17 ++++ .../geekbench-NexusLowResEmulator.yml | 17 ++++ .../ftl/fixtures/benchmark/geekbench-all.yml | 25 ++++++ .../fixtures/benchmark/geekbench-redfin.yml | 17 ++++ 9 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt rename test_runner/src/test/kotlin/ftl/fixtures/{benchmark-NexusLowRes.yml => benchmark/3dmark-NexusLowRes.yml} (87%) rename test_runner/src/test/kotlin/ftl/fixtures/{benchmark-NexusLowResEmulator.yml => benchmark/3dmark-NexusLowResEmulator.yml} (87%) rename test_runner/src/test/kotlin/ftl/fixtures/{benchmark-redfin.yml => benchmark/3dmark-redfin.yml} (87%) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt index 44cd48e1aa..c724ba892a 100644 --- a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/Run3DMark.kt @@ -17,28 +17,29 @@ private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L private const val DOWNLOAD_TIMEOUT = 5 * 60 * 1000L -private object Text { - const val search = "Search" - const val ok = "OK" - const val allow = "Allow" - const val permTitle = "Before we start…" - const val appName = "3DMark" - const val benchmarkType = "SLING SHOT" -} - -private object Res { - const val centerLayout = "com.futuremark.dmandroid.application:id/flm_pager_benchmarks" - const val centerLayoutChild = "com.futuremark.dmandroid.application:id/flm_cl_root" - const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" - const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" - const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" - const val scoreDetails = - "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" -} - @RunWith(AndroidJUnit4::class) class Run3DMark { + + private object Text { + const val search = "Search" + const val ok = "OK" + const val allow = "Allow" + const val permTitle = "Before we start…" + const val appName = "3DMark" + const val benchmarkType = "SLING SHOT" + } + + private object Res { + const val centerLayout = "com.futuremark.dmandroid.application:id/flm_pager_benchmarks" + const val centerLayoutChild = "com.futuremark.dmandroid.application:id/flm_cl_root" + const val btnSkip = "com.futuremark.dmandroid.application:id/flm_bt_tutorial_skip" + const val fabBenchmark = "com.futuremark.dmandroid.application:id/flm_fab_benchmark" + const val fabSettings = "com.futuremark.dmandroid.application:id/flm_fab_settings" + const val scoreDetails = + "com.futuremark.dmandroid.application:id/flm_ll_score_details_container" + } + @Test fun run() { diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt new file mode 100644 index 0000000000..c768ea89b8 --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunGeekbench.kt @@ -0,0 +1,82 @@ +package com.example.test.benchmark + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + + +@RunWith(AndroidJUnit4::class) +class RunGeekbench { + + private object Text { + const val search = "Search" + const val appName = "Geek" + const val accept = "ACCEPT" + const val benchmarkType = "RUN CPU BENCHMARK" + const val runningDialog = "Geekbench 5" + const val results = "Benchmark Results" + } + + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for 3d mark launcher icon + wait(Until.hasObject(By.textContains(Text.appName)), TIMEOUT) + + waitForIdle(5000) + + // Click 3d mark launcher icon + findObject(UiSelector().textContains(Text.appName)).click() + + waitForIdle(5000) + + Thread.sleep(1000) + + // Check permissions dialog + findObject(UiSelector().text(Text.accept)).run { + if (exists()) click() + } + + waitForIdle(5000) + Thread.sleep(2000) + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + waitForIdle(5000) + Thread.sleep(2000) + + // Wait until benchmark finish + wait(Until.hasObject(By.text(Text.runningDialog)), TIMEOUT) + wait(Until.gone(By.text(Text.runningDialog)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + wait(Until.hasObject(By.text(Text.results)), TIMEOUT) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml similarity index 87% rename from test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowRes.yml rename to test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml index 0cd66d25c0..2ba0e9702c 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowRes.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml @@ -12,3 +12,6 @@ gcloud: locale: en orientation: portrait record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml similarity index 87% rename from test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml rename to test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml index 7092a21bdd..d7ec2fba40 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-NexusLowResEmulator.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml @@ -12,3 +12,6 @@ gcloud: locale: en orientation: portrait record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml similarity index 87% rename from test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml rename to test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml index 40609b23b4..256202714e 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark-redfin.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml @@ -12,3 +12,6 @@ gcloud: locale: en orientation: portrait record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml new file mode 100644 index 0000000000..05872f836d --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml @@ -0,0 +1,17 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml new file mode 100644 index 0000000000..d54fa223e4 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml @@ -0,0 +1,17 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml new file mode 100644 index 0000000000..4be435c33b --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml @@ -0,0 +1,25 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml new file mode 100644 index 0000000000..e8f7a75337 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml @@ -0,0 +1,17 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/Geekbench 5_v5.3.2.apk" + device: + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true +flank: + test-targets: + - "class com.example.test.benchmark.RunGeekbench" From 08256793eba8749e4f309c88df927c6781ba5b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 8 Apr 2021 13:35:00 +0200 Subject: [PATCH 08/10] fix configs --- .../fixtures/benchmark/3dmark-NexusLowRes.yml | 1 - .../benchmark/3dmark-NexusLowResEmulator.yml | 1 - .../ftl/fixtures/benchmark/3dmark-all.yml | 24 +++++++++++++++++++ .../ftl/fixtures/benchmark/3dmark-redfin.yml | 1 - .../benchmark/geekbench-NexusLowRes.yml | 1 - .../geekbench-NexusLowResEmulator.yml | 1 - .../ftl/fixtures/benchmark/geekbench-all.yml | 1 - .../fixtures/benchmark/geekbench-redfin.yml | 1 - 8 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml index 2ba0e9702c..f307031531 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowRes.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml index d7ec2fba40..1459c7ac80 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-NexusLowResEmulator.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml new file mode 100644 index 0000000000..9b357fd25c --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-all.yml @@ -0,0 +1,24 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/3dmarkandroid-v2-1-4726.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait + - model: redfin + version: 30 + locale: en + orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml index 256202714e..ca226eb396 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/3dmark-redfin.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.Run3dMark" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml index 05872f836d..203222fed9 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowRes.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml index d54fa223e4..fea5c2cb92 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-NexusLowResEmulator.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml index 4be435c33b..013c994051 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-all.yml @@ -20,6 +20,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.RunGeekbench" diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml index e8f7a75337..6cfd916346 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/geekbench-redfin.yml @@ -12,6 +12,5 @@ gcloud: locale: en orientation: portrait record-video: true -flank: test-targets: - "class com.example.test.benchmark.RunGeekbench" From e7289738a79c8af70bc28a089f5b886b00fb8048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 9 Apr 2021 12:13:56 +0200 Subject: [PATCH 09/10] Add passmark test --- .../com/example/test/benchmark/RunPassMark.kt | 117 ++++++++++++++++++ .../ftl/fixtures/benchmark/passmark-all.yml | 24 ++++ 2 files changed, 141 insertions(+) create mode 100644 test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml diff --git a/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt new file mode 100644 index 0000000000..424ccc0f69 --- /dev/null +++ b/test_projects/android/benchmark/src/androidTest/java/com/example/test/benchmark/RunPassMark.kt @@ -0,0 +1,117 @@ +package com.example.test.benchmark + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiSelector +import androidx.test.uiautomator.Until +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + + +private const val TIMEOUT = 10_000L + +private const val BENCHMARK_TIMEOUT = 15 * 60 * 1000L + + +@RunWith(AndroidJUnit4::class) +class RunPassMark { + + private object Res { + const val splash = "android:id/content" + } + + private object Text { + const val search = "Search" + const val appName = "Perform" + const val accept = "Continue" + const val ok = "OK" + const val next = "NEXT" + const val gotIt = "GOT IT!" + const val cancel = "CANCEL" + const val benchmarkType = "RUN BENCHMARK" + const val results = "Benchmark Results" + } + + @Test + fun run() { + + UiDevice.getInstance( + InstrumentationRegistry.getInstrumentation() + ).run { + // Start from the home screen + pressHome() + + // Open apps menu on pixel launcher + findObject(UiSelector().descriptionContains(Text.search)).apply { + swipe(0, visibleBounds.centerY(), 0, 0, 10) + } + + // Wait for app launcher icon + wait(Until.hasObject(By.textContains(Text.appName)), TIMEOUT) + + waitForIdle(5000) + + // Click launcher icon + findObject(UiSelector().textContains(Text.appName)).click() + + waitForIdle(5000) + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + Thread.sleep(1000) + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + // Check permissions dialog + findObject(UiSelector().text(Text.accept)).run { + if (exists()) click() + } + + // Check version dialog + findObject(UiSelector().text(Text.ok)).run { + if (exists()) click() + } + + // Touch to continue... + findObject(UiSelector().resourceId(Res.splash)).click() + + // Check tooltip dialog + findObject(UiSelector().text(Text.next)).run { + if (exists()) click() + } + + // Check tooltip dialog + findObject(UiSelector().text(Text.gotIt)).run { + if (exists()) click() + } + + waitForIdle(5000) + Thread.sleep(2000) + + // Choose proper benchmark screen + findObject(UiSelector().text(Text.benchmarkType)).click() + + waitForIdle(5000) + Thread.sleep(2000) + + // Wait until benchmark finish + wait(Until.hasObject(By.text(Text.cancel)), TIMEOUT) + wait(Until.gone(By.text(Text.cancel)), BENCHMARK_TIMEOUT) + + // Assert that benchmark results screen is visible + wait(Until.hasObject(By.text(Text.results)), BENCHMARK_TIMEOUT) + + // Make sure that results was recorded + Thread.sleep(5000) + } + } +} diff --git a/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml new file mode 100644 index 0000000000..96fbae5e8a --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/benchmark/passmark-all.yml @@ -0,0 +1,24 @@ +gcloud: + app: "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk" + test: "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/benchmark-debug-androidTest.apk" + use-orchestrator: false + directories-to-pull: + - "/sdcard/Android/data/com.futuremark.dmandroid.application/files/3DMarkAndroid/" + additional-apks: + - "./src/test/kotlin/ftl/fixtures/tmp/apk/benchmark/PassMark_PerformanceTest_10.0.1001-fileplanet.apk" + device: + - model: NexusLowRes + version: 30 + locale: en + orientation: portrait + - model: NexusLowResEmulator + version: 30 + locale: en + orientation: portrait +# - model: redfin +# version: 30 +# locale: en +# orientation: portrait + record-video: true + test-targets: + - "class com.example.test.benchmark.RunPassMark" From 90785eda33bac847d29b0499181744032611da7d Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 14 Apr 2021 22:46:24 +0200 Subject: [PATCH 10/10] Linting --- flank-scripts/build.gradle.kts | 2 +- .../kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flank-scripts/build.gradle.kts b/flank-scripts/build.gradle.kts index c99f5bd703..53b650ec59 100644 --- a/flank-scripts/build.gradle.kts +++ b/flank-scripts/build.gradle.kts @@ -26,7 +26,7 @@ shadowJar.apply { } } // .. -version = "1.9.10" +version = "1.9.11" group = "com.github.flank" application { diff --git a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt index 2e6771fabb..439a82c923 100644 --- a/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt +++ b/flank-scripts/src/main/kotlin/flank/scripts/ops/assemble/android/BuildBenchmark.kt @@ -28,4 +28,3 @@ private fun copyApks() { val outputDir = Paths.get(flankFixturesTmpPath, "apk", BENCHMARK).toString() Paths.get(androidTestProjectsPath, BENCHMARK).toFile().findApks().copyApksToPath(outputDir) } -