From 003d082c71f99c34045df39ffd75fbb328a33e35 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Sat, 7 Nov 2020 20:54:53 +0100 Subject: [PATCH 01/32] Improve integration tests --- integration_tests/build.gradle.kts | 12 + .../integration/IntergrationTestsUtils.kt | 153 ++++++++++ .../test/kotlin/integration/SanityRoboIT.kt | 281 ++++++++++++++++++ .../resources/all_test_filtered_android.yml | 8 + .../test/resources/all_test_filtered_ios.yml | 5 + .../src/test/resources/flank_android.yml | 1 + .../src/test/resources/flank_android_full.yml | 12 + .../src/test/resources/flank_ios.yml | 2 + .../src/test/resources/sanity_robo.yml | 7 + 9 files changed, 481 insertions(+) create mode 100644 integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt create mode 100644 integration_tests/src/test/kotlin/integration/SanityRoboIT.kt create mode 100644 integration_tests/src/test/resources/all_test_filtered_android.yml create mode 100644 integration_tests/src/test/resources/all_test_filtered_ios.yml create mode 100644 integration_tests/src/test/resources/flank_android_full.yml create mode 100644 integration_tests/src/test/resources/sanity_robo.yml diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index c33e3ba5b5..394fc12a09 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -31,6 +31,7 @@ detekt { dependencies { implementation(kotlin("stdlib")) testImplementation(Dependencies.JUNIT) + testImplementation(Dependencies.TRUTH) detektPlugins(Dependencies.DETEKT_FORMATTING) } @@ -44,4 +45,15 @@ tasks.test { systemProperty("working-directory", System.getProperty("working-directory")) systemProperty("output-pattern", System.getProperty("output-pattern")) systemProperty("expected-output-code", System.getProperty("expected-output-code")) + filter { + excludeTestsMatching("*IT") + } +} + +tasks.register("integrationTests") { + group = "Verification" + description = "Runs flank integration tests" + filter { + includeTestsMatching("*IT") + } } diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt new file mode 100644 index 0000000000..59b12ac795 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -0,0 +1,153 @@ +package integration + +import com.google.common.truth.Truth.assertThat +import org.junit.Assert +import org.junit.Assert.assertEquals +import utils.ProcessResult +import java.lang.StringBuilder +import kotlin.text.Regex.Companion.escape + +val androidRunCommands = listOf("firebase", "test", "android", "run") + +private val commonGcloud = """ + gcloud: + results-bucket: test-lab-[.a-zA-Z0-9_-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null +""".trimIndent() + +val gcloudDefault = """ + $commonGcloud + # Android gcloud + app: [0-9a-zA-Z\/_]*-debug.apk + test: null + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 +""".trimIndent() + +private val flankCommon = """ + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: false + project: flank-open-source + local-result-dir: results + full-junit-result: false +""".trimIndent() + +val flankDefault = """ + $flankCommon + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: single + disable-results-upload: false + default-class-test-time: 240.0 +""".trimIndent() + +private fun StringBuilder.flankOutputLine(line: String) = append(line).append("\\s*") + +fun FlankOutput.matricesWebLink(linksNumber: Int = 1) = lines.add(buildString { + flankOutputLine("Matrices webLink") + repeat(linksNumber) { + flankOutputLine("matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]*") + } +}) + +fun FlankOutput.runTests(tests: Int = 1, shards: Int = 1, matrices: Int = 1) = lines.add(buildString { + flankOutputLine("RunTests") + flankOutputLine("Uploading app-debug.apk \\.*") + flankOutputLine("$tests test / $shards shard") + flankOutputLine("$matrices matrix ids created in \\dm \\ds") + flankOutputLine("https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]*") +}) + +fun FlankOutput.fetchArtifacts() = lines.add(buildString { + flankOutputLine("FetchArtifacts") + flankOutputLine("Updating matrix file") +}) + +fun FlankOutput.costReport() = lines.add(buildString { + flankOutputLine("CostReport") + flankOutputLine("Virtual devices") + flankOutputLine("\\$(\\d*.\\d*) for \\d*m") +}) + +fun FlankOutput.resultsReport(left: Int = 1, right: Int = 1) = lines.add(buildString { + flankOutputLine("MatrixResultsReport") + flankOutputLine("$left / $right ${escape("(100.00%)")}") +}) + +private const val outputSummary_2 = "┌[\\s\\S]*┘" + +data class FlankOutput(val lines: MutableList = mutableListOf()) + +infix fun String.containsFlankOutput(block: FlankOutput.() -> Unit) = assertThat(this).run { + FlankOutput().apply(block).lines.forEach { containsMatch(it) } +} + +fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals(""" + Exit code: + expected $expectedExitCode + actual ${result.exitCode} + Output: + ${result.output} +""".trimIndent(), + 0, + result.exitCode +) + +object Assert { + infix fun that(input: String) = input +} + +data class OutcomeSummary(val lines: MutableList = mutableListOf()) + +fun outcomeSummary() = buildString { + flankOutputLine("┌[\\s\\S") +} + + +infix fun String.containsOutcomeSummary(block: OutcomeSummary.() -> Unit) = assertThat(this).run { + OutcomeSummary().apply(block).lines.forEach { containsMatch((it)) } +} + +enum class TestOutcome { + SUCCESS, FLAKY, FAILURE +} diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt new file mode 100644 index 0000000000..689bb448ba --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -0,0 +1,281 @@ +package integration + +import com.google.common.truth.Truth.assertThat +import FlankCommand +import TestParameters +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import run +import toAndroidTestParameters +import utils.toStringMap +import kotlin.text.Regex.Companion.escape + + +const val outputSummary = "┌[\\s\\S]*┘" +const val emptyLines = "\\s*" + +class SanityRoboIT { + private lateinit var testParams: TestParameters + + @Before + fun setUp() { + testParams = System.getProperties().toStringMap().toAndroidTestParameters() + } + + @Test + fun `should run sanity robo`() { + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/sanity_robo.yml", + params = androidRunCommands + ).run("./") + + + assertExitCode(result, 0) + + Assert that result.output containsFlankOutput { + matricesWebLink(1) + runTests(0, 0, 1) + fetchArtifacts() + costReport() + resultsReport() + } + } + + @Test + fun testa() { + listOf() + println() + } +} + +val bbreg = """ + AndroidArgs + gcloud: + results-bucket: test-lab-[.a-zA-Z0-9_-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: [0-9a-zA-Z\/_]*-debug.apk + test: null + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 50 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + - class actually.any.test.Clazz + disable-sharding: false + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: single + disable-results-upload: true + default-class-test-time: 240.0 + $emptyLines + RunTests + Uploading app-debug.apk \.* + 0 test / 0 shard + $emptyLines + 1 matrix ids created in \dm \ds + https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + $emptyLines + Matrices webLink + matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* + [\s]*0[\s\S]*FINISHED[\s]* + $emptyLines + FetchArtifacts + $emptyLines + Updating matrix file + $emptyLines + CostReport + Virtual devices + ${escape("$")}(\d*.\d*) for \d*m + $emptyLines + MatrixResultsReport + 1 / 1 ${escape("(100.00%)")} + $emptyLines + $outputSummary + $emptyLines + Matrices webLink + matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* +""".trimIndent() + +val test = """ + version: local_snapshot + revision: 55476259c1d391bf779affaa0ed9dc30692279de + + AndroidArgs + gcloud: + results-bucket: test-lab-v9cn46bb990nx-kz69ymd4nm9aq + results-dir: 2020-11-05_14-57-23.204686_dGGA + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: null + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 50 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + - class actually.any.test.Clazz + disable-sharding: false + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: multi + disable-results-upload: true + default-class-test-time: 240.0 + + RunTests + Uploading app-debug.apk . + 0 test / 0 shard + + 1 matrix ids created in 0m 4s + https://console.developers.google.com/storage/browser/test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-11-05_14-57-23.204686_dGGA + + Matrices webLink + matrix-2leuyoubnp892 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/8504866218843143096/executions/bs.b2bb28a35d2f2766 + + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. + 2m 16s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Installing apps. + 2m 16s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Installing apps. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. + 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. + 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. + 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. + 3m 52s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Stopped logcat recording. + 3m 58s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting results processing. Attempt: 1 + 4m 4s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 FINISHED + 4m 4s matrix-2leuyoubnp892 FINISHED + + FetchArtifacts + + Updating matrix file + + CostReport + Virtual devices + ${'$'}0.03 for 2m + + MatrixResultsReport + 1 / 1 (100.00%) + ┌─────────┬──────────────────────┬────────────────────────────┬──────────────┐ + │ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ + ├─────────┼──────────────────────┼────────────────────────────┼──────────────┤ + │ success │ matrix-2leuyoubnp892 │ NexusLowRes-28-en-portrait │ --- │ + └─────────┴──────────────────────┴────────────────────────────┴──────────────┘ + + Matrices webLink + matrix-2leuyoubnp892 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/8504866218843143096/executions/bs.b2bb28a35d2f2766 +""".trimIndent() diff --git a/integration_tests/src/test/resources/all_test_filtered_android.yml b/integration_tests/src/test/resources/all_test_filtered_android.yml new file mode 100644 index 0000000000..8205ae3475 --- /dev/null +++ b/integration_tests/src/test/resources/all_test_filtered_android.yml @@ -0,0 +1,8 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + test-targets: + - class non.existing.Class + +flank: + disable-sharding: true diff --git a/integration_tests/src/test/resources/all_test_filtered_ios.yml b/integration_tests/src/test/resources/all_test_filtered_ios.yml new file mode 100644 index 0000000000..b9a9b8e895 --- /dev/null +++ b/integration_tests/src/test/resources/all_test_filtered_ios.yml @@ -0,0 +1,5 @@ +gcloud: + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + test-targets: + - class nonExisting/Class diff --git a/integration_tests/src/test/resources/flank_android.yml b/integration_tests/src/test/resources/flank_android.yml index 592a046a13..0960b66b57 100644 --- a/integration_tests/src/test/resources/flank_android.yml +++ b/integration_tests/src/test/resources/flank_android.yml @@ -3,4 +3,5 @@ gcloud: test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk flank: + disable-results-upload: true disable-sharding: true diff --git a/integration_tests/src/test/resources/flank_android_full.yml b/integration_tests/src/test/resources/flank_android_full.yml new file mode 100644 index 0000000000..a890005a79 --- /dev/null +++ b/integration_tests/src/test/resources/flank_android_full.yml @@ -0,0 +1,12 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + robo-script: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json + +flank: + disable-sharding: false + max-test-shards: 50 + disable-results-upload: true + additional-app-test-apks: + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk + - test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk diff --git a/integration_tests/src/test/resources/flank_ios.yml b/integration_tests/src/test/resources/flank_ios.yml index 1f7350242b..4701605de6 100644 --- a/integration_tests/src/test/resources/flank_ios.yml +++ b/integration_tests/src/test/resources/flank_ios.yml @@ -1,3 +1,5 @@ gcloud: test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun +flank: + disable-results-upload: true diff --git a/integration_tests/src/test/resources/sanity_robo.yml b/integration_tests/src/test/resources/sanity_robo.yml new file mode 100644 index 0000000000..5c04ba3a78 --- /dev/null +++ b/integration_tests/src/test/resources/sanity_robo.yml @@ -0,0 +1,7 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk +flank: + disable-results-upload: true + max-test-shards: 50 + test-targets-always-run: + - class actually.any.test.Clazz From 69f9f3f97569054ba52f7321f905eaee7a951936 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Tue, 10 Nov 2020 18:37:49 +0100 Subject: [PATCH 02/32] Add full IT --- integration_tests/build.gradle.kts | 13 ++ .../src/test/kotlin/FlankCommand.kt | 2 +- .../kotlin/integration/AllTestFilteredIT.kt | 37 +++ .../test/kotlin/integration/AndroidFullIT.kt | 203 ++++++++++++++++ .../test/kotlin/integration/DefaultConfigs.kt | 83 +++++++ .../integration/IntergrationTestsUtils.kt | 216 ++++++++++-------- .../test/kotlin/integration/SanityRoboIT.kt | 171 +++++--------- .../src/test/kotlin/utils/CommandHelper.kt | 31 +-- .../test/resources/all_test_filtered_ios.yml | 3 +- .../src/test/resources/flank_android_full.yml | 3 +- .../src/test/resources/sanity_robo.yml | 5 - 11 files changed, 521 insertions(+), 246 deletions(-) create mode 100644 integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt create mode 100644 integration_tests/src/test/kotlin/integration/AndroidFullIT.kt create mode 100644 integration_tests/src/test/kotlin/integration/DefaultConfigs.kt diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index 394fc12a09..b262dcf561 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -1,3 +1,6 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + plugins { java kotlin(Plugins.Kotlin.PLUGIN_JVM) @@ -56,4 +59,14 @@ tasks.register("integrationTests") { filter { includeTestsMatching("*IT") } + testLogging { + events("skipped", "failed") + exceptionFormat = TestExceptionFormat.FULL + } + maxParallelForks = 4 +} + +val compileTestKotlin: KotlinCompile by tasks +compileTestKotlin.kotlinOptions { + freeCompilerArgs = listOf("-Xinline-classes") } diff --git a/integration_tests/src/test/kotlin/FlankCommand.kt b/integration_tests/src/test/kotlin/FlankCommand.kt index 343970d8e3..9b5ba12853 100644 --- a/integration_tests/src/test/kotlin/FlankCommand.kt +++ b/integration_tests/src/test/kotlin/FlankCommand.kt @@ -5,4 +5,4 @@ data class FlankCommand(val flankPath: String, val ymlPath: String, val params: private fun FlankCommand.create() = "java -jar $flankPath ${params.joinToString(separator = " ")} -c=$ymlPath" -fun FlankCommand.run(workingDirectory: String) = create().runCommand(File(workingDirectory)) +fun FlankCommand.run(workingDirectory: String, testSuite: String = "") = create().runCommand(File(workingDirectory), testSuite) diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt new file mode 100644 index 0000000000..a946c10d1d --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -0,0 +1,37 @@ +package integration + +import FlankCommand +import run +import org.junit.Test + +class AllTestFilteredIT { + + @Test + fun `filter all tests - android`() { + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/all_test_filtered_android.yml", + params = androidRunCommands + ).run( + workingDirectory = "./", + testSuite = this::class.java.simpleName + ) + + assertExitCode(result, 1) + + Assert that result.output.removeUnicode() contains { + customConfig( + "test" with "[0-9a-zA-Z\\/_]*.apk", + "disable-sharding" with true, + "test-targets" with listOf( + "- class non.existing.Class" + ) + ) + output { + noneTestRan() + androidShards(tests = 0) + } + noOutcomeSummary() + } + } +} diff --git a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt new file mode 100644 index 0000000000..fea3b6cadc --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt @@ -0,0 +1,203 @@ +package integration + +import FlankCommand +import TestParameters +import org.junit.Before +import org.junit.Test +import run +import toAndroidTestParameters +import utils.toStringMap + + +class AndroidFullIT { + + @Test + fun `flank full option run`() { +// val result = FlankCommand( +// flankPath = "../test_runner/build/libs/flank.jar", +// ymlPath = "./src/test/resources/flank_android_full.yml", +// params = androidRunCommands +// ).run("./", this::class.java.simpleName) +// +// +// assertExitCode(result, 10) +// +// Assert that result.output.removeUnicode() contains { + Assert that testa contains { + customConfig( + "disable-sharding" with false, + "max-test-shards" with 50, + "additional-app-test-apks" with listOf( + "- app: null", + " test: [0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk", + "- app: null", + " test: [0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk", + "- app: null", + " test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk", + ), + "robo-script" with "[0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json", + "use-orchestrator" with false, + "output-style" with "single" + ) + output { + matricesWebLink(linksNumber = 4) + androidShards( + tests = 11, + classes = 8, + shards = 19, + matrices = 4 + ) + fetchArtifacts() + costReport( + virtual = true, + physical = true + ) + resultsReport() + } + outcomeSummary { + success(number = 3) + failure(number = 1) + flaky(number = 0) + } + } + } +} + + +val testa = """ + version: local_snapshot +revision: c56cdec8b3586a64f3ff644a74372524613ee301 + +AndroidArgs + gcloud: + results-bucket: test-lab-v9cn46bb990nx-kz69ymd4nm9aq + results-dir: 2020-11-10_11-26-27.200686_GnSv + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: null + additional-apks: + auto-google-login: false + use-orchestrator: false + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + test-targets-for-shard: + + flank: + max-test-shards: 50 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: false + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + - app: null + test: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + - app: null + test: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk + - app: null + test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: single + disable-results-upload: false + default-class-test-time: 240.0 + +RunTests + + Smart Flank cache hit: 0% (0 / 9) + Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s + + + Smart Flank cache hit: 0% (0 / 9) + Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s + + + Smart Flank cache hit: 0% (0 / 1) + Shard times: 120s + +Saved 3 shards to android_shards.json + Uploading android_shards.json . + Uploading app-debug.apk . + Uploading app-multiple-error-debug-androidTest.apk . Uploading app-multiple-success-debug-androidTest.apk . Uploading MainActivity_robo_script.json . + + + 11 tests + 8 parameterized classes / 19 shards + + 4 matrix ids created in 0m 12s + https://console.developers.google.com/storage/browser/test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-11-10_11-26-27.200686_GnSv + +Matrices webLink + matrix-3rggyncbmbsu2 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6674251027591133234/executions/bs.8a98b8a47771a4c2 + matrix-2if6zvcy09fym https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5087540697861399228 + matrix-uv04kgwx4efqa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579/executions/bs.752e21ad35ba9970 + matrix-r9ejzytvt6jaa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 + +FetchArtifacts + ................... + Updating matrix file + +CostReport + Virtual devices + ${'$'}0.35 for 21m + + Uploading CostReport.txt . +MatrixResultsReport + 2 / 4 (50.00%) + 2 matrices failed + +┌─────────┬──────────────────────┬────────────────────────────┬────────────────────────────────┐ +│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ +├─────────┼──────────────────────┼────────────────────────────┼────────────────────────────────┤ +│ success │ matrix-3rggyncbmbsu2 │ NexusLowRes-28-en-portrait │ --- │ +│ success │ matrix-2if6zvcy09fym │ NexusLowRes-28-en-portrait │ 20 test cases passed │ +│ failure │ matrix-r9ejzytvt6jaa │ NexusLowRes-28-en-portrait │ 5 test cases failed, 15 passed │ +│ failure │ matrix-uv04kgwx4efqa │ NexusLowRes-28-en-portrait │ 0 test cases failed │ +└─────────┴──────────────────────┴────────────────────────────┴────────────────────────────────┘ +More details are available at: +https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 +https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579 + + Uploading MatrixResultsReport.txt . + Uploading HtmlErrorReport.html . + Uploading JUnitReport.xml . + +Matrices webLink + matrix-3rggyncbmbsu2 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6674251027591133234/executions/bs.8a98b8a47771a4c2 + matrix-r9ejzytvt6jaa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 + matrix-2if6zvcy09fym https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5087540697861399228 + matrix-uv04kgwx4efqa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579/executions/bs.752e21ad35ba9970 +""".trimIndent() diff --git a/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt b/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt new file mode 100644 index 0000000000..d452468502 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt @@ -0,0 +1,83 @@ +package integration + +val androidRunCommands = listOf("firebase", "test", "android", "run") +val iosRunCommands = listOf("firebase", "test", "ios", "run") +private const val EMPTY_STRING = "" +private const val NULL_STRING = "null" + +val defaultCommonConfig = mapOf( + "results-bucket" to "test-lab-[a-zA-Z0-9-]*", + "results-dir" to "[.a-zA-Z0-9_-]*", + "record-video" to false, + "timeout" to "15m", + "async" to false, + "client-details" to EMPTY_STRING, + "network-profile" to NULL_STRING, + "results-history-name" to NULL_STRING, + "num-flaky-test-attempts" to 0, + "directories-to-pull" to EMPTY_STRING, + "other-files" to EMPTY_STRING, + "max-test-shards" to 1, + "shard-time" to -1, + "num-test-runs" to 1, + "smart-flank-gcs-path" to EMPTY_STRING, + "smart-flank-disable-upload" to false, + "default-test-time" to 120.0, + "use-average-test-time-for-new-tests" to false, + "files-to-download" to EMPTY_STRING, + "test-targets-always-run" to EMPTY_STRING, + "disable-sharding" to false, + "project" to "flank-open-source", + "local-result-dir" to "results", + "full-junit-result" to false, + "keep-file-path" to false, + "ignore-failed-tests" to false, + "output-style" to "verbose", + "disable-results-upload" to false, + "default-class-test-time" to 240.0, + "run-timeout" to -1, + "scenario-numbers" to EMPTY_STRING, +) + +val defaultAndroidConfig = defaultCommonConfig + mapOf( + "app" to "[0-9a-zA-Z\\/_]*.apk", + "test" to NULL_STRING, + "additional-apks" to EMPTY_STRING, + "auto-google-login" to false, + "use-orchestrator" to true, + "grant-permissions" to "all", + "type" to NULL_STRING, + "scenario-labels" to EMPTY_STRING, + "obb-files" to EMPTY_STRING, + "obb-names" to EMPTY_STRING, + "performance-metrics" to false, + "num-uniform-shards" to NULL_STRING, + "test-runner-class" to NULL_STRING, + "test-targets" to EMPTY_STRING, + "robo-directives" to EMPTY_STRING, + "robo-script" to NULL_STRING, + "device" to """ + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait +""".replace(System.lineSeparator(), "[\\s]*"), + "additional-app-test-apks" to EMPTY_STRING, + "legacy-junit-result" to false, +) + +val defaultIosConfig = mapOf( + "test" to "[0-9a-zA-Z\\/_]*.zip", + "xctestrun-file" to "[0-9a-zA-Z\\/_].xctestrun", + "xcode-version" to NULL_STRING, + "device" to """ + - model: iphone8 + version: 12.0 + locale: en + orientation: portrait +""".replace(System.lineSeparator(), "[\\s]*"), + "additional-ipas" to EMPTY_STRING, + "type" to "xctest", + "app" to EMPTY_STRING, + "test-special-entitlements" to false +) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 59b12ac795..08f26fbbb1 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -1,153 +1,171 @@ package integration import com.google.common.truth.Truth.assertThat -import org.junit.Assert import org.junit.Assert.assertEquals import utils.ProcessResult import java.lang.StringBuilder import kotlin.text.Regex.Companion.escape -val androidRunCommands = listOf("firebase", "test", "android", "run") - -private val commonGcloud = """ - gcloud: - results-bucket: test-lab-[.a-zA-Z0-9_-]* - results-dir: [.a-zA-Z0-9_-]* - record-video: false - timeout: 15m - async: false - client-details: - network-profile: null - results-history-name: null -""".trimIndent() - -val gcloudDefault = """ - $commonGcloud - # Android gcloud - app: [0-9a-zA-Z\/_]*-debug.apk - test: null - additional-apks: - auto-google-login: false - use-orchestrator: true - directories-to-pull: - grant-permissions: all - type: null - other-files: - scenario-numbers: - scenario-labels: - obb-files: - obb-names: - performance-metrics: false - num-uniform-shards: null - test-runner-class: null - test-targets: - robo-directives: - robo-script: null - device: - - model: NexusLowRes - version: 28 - locale: en - orientation: portrait - num-flaky-test-attempts: 0 -""".trimIndent() - -private val flankCommon = """ - flank: - max-test-shards: 1 - shard-time: -1 - num-test-runs: 1 - smart-flank-gcs-path: - smart-flank-disable-upload: false - default-test-time: 120.0 - use-average-test-time-for-new-tests: false - files-to-download: - test-targets-always-run: - disable-sharding: false - project: flank-open-source - local-result-dir: results - full-junit-result: false -""".trimIndent() - -val flankDefault = """ - $flankCommon - # Android Flank Yml - keep-file-path: false - additional-app-test-apks: - run-timeout: -1 - legacy-junit-result: false - ignore-failed-tests: false - output-style: single - disable-results-upload: false - default-class-test-time: 240.0 -""".trimIndent() +fun FlankAssertion.defaultConfig() = customConfig() + +fun FlankAssertion.customConfig(vararg customChecks: Pair = emptyArray()) { + val checks = defaultAndroidConfig.toMutableMap() + customChecks.forEach { checks[it.first] = it.second } + checks.entries + .forEach { + if (it.makeKVRegex().find(input) == null) { + val message = """ + |Incorrect value for [${it.key}]. + |Expected: + | ${it.key}: ${ + it.value.toString() + .replace("-", "${System.lineSeparator()} -") + .replace("""\s*""", "") + } + |Was: + | ${ + "${it.key}:[\\s\\S]{10,60}" + .toRegex() + .find(input) + ?.value + ?.split(System.lineSeparator()) + ?.joinToString { value -> "${System.lineSeparator()} " + value.trim() } + }(...) + |Output: + |$input""".trimMargin() + throw AssertionError(message) + } + } +} + +private fun MutableMap.MutableEntry.makeKVRegex() = "${key}:\\s*${value}".toRegex() + +infix fun String.with(value: Any) = this to value.toString() + +infix fun String.with(values: List) = this to buildString { values.forEach(::flankOutputLine) } private fun StringBuilder.flankOutputLine(line: String) = append(line).append("\\s*") fun FlankOutput.matricesWebLink(linksNumber: Int = 1) = lines.add(buildString { flankOutputLine("Matrices webLink") repeat(linksNumber) { - flankOutputLine("matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]*") + flankOutputLine("matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)?") } }) -fun FlankOutput.runTests(tests: Int = 1, shards: Int = 1, matrices: Int = 1) = lines.add(buildString { +private fun shards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1, os: String) = buildString { + val testString = if (tests > 0) "tests" else "test" + val classString = if (classes > 0) "classes" else "class" + val shardString = if (shards > 0) "shards" else "shard" flankOutputLine("RunTests") - flankOutputLine("Uploading app-debug.apk \\.*") - flankOutputLine("$tests test / $shards shard") + flankOutputLine("[\\S\\s]*") + flankOutputLine("Saved $shards shards to ${os}_shards.json") + flankOutputLine("Uploading ${os}_shards.json .") + if (tests >= 0) { + flankOutputLine("Uploading app-debug.apk [\\s\\S]*") + if (classes == 0) + flankOutputLine("$tests $testString / $shards $shardString") + else + flankOutputLine("$tests $testString + $classes parameterized $classString / $shards $shardString") + } flankOutputLine("$matrices matrix ids created in \\dm \\ds") flankOutputLine("https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]*") +} + +fun FlankOutput.androidShards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1) = lines.add(shards(tests, classes, shards, matrices, "android")) + +fun FlankOutput.iosShards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1) = lines.add(shards(tests, classes, shards, matrices, "ios")) + +fun FlankOutput.resultsReport(left: Int = 1, right: Int = 1) = lines.add(buildString { + flankOutputLine("MatrixResultsReport") + flankOutputLine("$left / $right ${escape("(100.00%)")}") }) +fun FlankOutput.noneTestRan() = lines.apply { + add(buildString { + flankOutputLine("RunTests") + flankOutputLine("No tests for [a-zA-Z0-9._-]*.apk") + }) + add("There are no tests to run") +} + fun FlankOutput.fetchArtifacts() = lines.add(buildString { flankOutputLine("FetchArtifacts") + flankOutputLine("\\.*") flankOutputLine("Updating matrix file") }) -fun FlankOutput.costReport() = lines.add(buildString { +fun FlankOutput.costReport(virtual: Boolean, physical: Boolean) = lines.add(buildString { flankOutputLine("CostReport") - flankOutputLine("Virtual devices") - flankOutputLine("\\$(\\d*.\\d*) for \\d*m") -}) - -fun FlankOutput.resultsReport(left: Int = 1, right: Int = 1) = lines.add(buildString { - flankOutputLine("MatrixResultsReport") - flankOutputLine("$left / $right ${escape("(100.00%)")}") + if (virtual) { + flankOutputLine("Virtual devices") + flankOutputLine("\\$(\\d*.\\d*) for \\d*m") + } + if (physical) { + flankOutputLine("Physical devices") + flankOutputLine("\\$(\\d*.\\d*) for \\d*m") + } }) -private const val outputSummary_2 = "┌[\\s\\S]*┘" - data class FlankOutput(val lines: MutableList = mutableListOf()) -infix fun String.containsFlankOutput(block: FlankOutput.() -> Unit) = assertThat(this).run { +infix fun FlankAssertion.output(block: FlankOutput.() -> Unit) = assertThat(input).run { FlankOutput().apply(block).lines.forEach { containsMatch(it) } } -fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals(""" +fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals( + """ Exit code: expected $expectedExitCode actual ${result.exitCode} Output: ${result.output} """.trimIndent(), - 0, + expectedExitCode, result.exitCode ) object Assert { - infix fun that(input: String) = input + infix fun that(input: String) = FlankAssertion(input) } -data class OutcomeSummary(val lines: MutableList = mutableListOf()) +@Suppress("EXPERIMENTAL_FEATURE_WARNING") +inline class FlankAssertion(val input: String) + +inline infix fun FlankAssertion.contains(block: FlankAssertion.() -> Unit) = this.run(block) + +data class OutcomeSummary(val matcher: MutableMap = mutableMapOf().withDefault { 0 }) { + fun success(number: Int) = matcher.put(TestOutcome.SUCCESS, number) + + fun failure(number: Int) = matcher.put(TestOutcome.FAILURE, number) -fun outcomeSummary() = buildString { - flankOutputLine("┌[\\s\\S") + fun flaky(number: Int) = matcher.put(TestOutcome.FLAKY, number) } +fun FlankAssertion.outcomeSummary(block: OutcomeSummary.() -> Unit) = + OutcomeSummary().apply(block).matcher.entries.forEach { (outcome, times) -> + val actual = outcome.regex.findAll(input).toList().size + if (actual != times) throw AssertionError( + """Incorrect number of ${outcome.name} + | expected: $times + | but was: $actual + |Output: + |${"┌[\\s\\S]*┘".toRegex().find(input)?.value?.trimIndent()} + """.trimMargin() + ) + } -infix fun String.containsOutcomeSummary(block: OutcomeSummary.() -> Unit) = assertThat(this).run { - OutcomeSummary().apply(block).lines.forEach { containsMatch((it)) } +fun FlankAssertion.noOutcomeSummary() { + if ("┌[\\s\\S]*┘".toRegex().matches(input)) throw AssertionError("There should be no outcome table.") } -enum class TestOutcome { - SUCCESS, FLAKY, FAILURE +enum class TestOutcome(val regex: Regex) { + SUCCESS(fromCommon("success", "---")), + FLAKY(fromCommon("flaky", "[0-9a-z\\s,]*")), + FAILURE(fromCommon("failure", "[0-9a-z\\s,]*")), } + +private val fromCommon = + { result: String, details: String -> "│\\s$result\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s[a-zA-Z0-9-]*\\s│\\s$details\\s*│".toRegex() } + +fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "") diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt index 689bb448ba..10d0c752b1 100644 --- a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -1,147 +1,75 @@ package integration -import com.google.common.truth.Truth.assertThat import FlankCommand -import TestParameters -import org.junit.Assert.assertEquals -import org.junit.Before import org.junit.Test import run -import toAndroidTestParameters -import utils.toStringMap -import kotlin.text.Regex.Companion.escape - - -const val outputSummary = "┌[\\s\\S]*┘" -const val emptyLines = "\\s*" class SanityRoboIT { - private lateinit var testParams: TestParameters - - @Before - fun setUp() { - testParams = System.getProperties().toStringMap().toAndroidTestParameters() - } @Test - fun `should run sanity robo`() { + fun `sanity robo`() { val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/sanity_robo.yml", params = androidRunCommands - ).run("./") + ).run( + workingDirectory = "./", + testSuite = this::class.java.simpleName + ) assertExitCode(result, 0) - Assert that result.output containsFlankOutput { - matricesWebLink(1) - runTests(0, 0, 1) - fetchArtifacts() - costReport() - resultsReport() + Assert that result.output.removeUnicode() contains { + defaultConfig() + output { + matricesWebLink(linksNumber = 1) + androidShards( + tests = 0, + shards = 0, + matrices = 1 + ) + fetchArtifacts() + costReport( + virtual = true, + physical = false + ) + resultsReport() + } + outcomeSummary { + success(number = 1) + failure(number = 0) + flaky(number = 0) + } } } @Test fun testa() { - listOf() println() + Assert that test contains { +// config( +// "disable-results-upload" with true, +// "max-test-shards" with 50, +// "test-targets-always-run" with listOf( +// "- class actually.any.test.Clazz", +// "- class actually.any.test.Clazz1", +// "- class actually.any.test.Clazz2", +// "- class actually.any.test.Clazz3" +// ) +// +// ) + outcomeSummary { + success(1) +// failure(1) + flaky(0) + } + } + } } -val bbreg = """ - AndroidArgs - gcloud: - results-bucket: test-lab-[.a-zA-Z0-9_-]* - results-dir: [.a-zA-Z0-9_-]* - record-video: false - timeout: 15m - async: false - client-details: - network-profile: null - results-history-name: null - # Android gcloud - app: [0-9a-zA-Z\/_]*-debug.apk - test: null - additional-apks: - auto-google-login: false - use-orchestrator: true - directories-to-pull: - grant-permissions: all - type: null - other-files: - scenario-numbers: - scenario-labels: - obb-files: - obb-names: - performance-metrics: false - num-uniform-shards: null - test-runner-class: null - test-targets: - robo-directives: - robo-script: null - device: - - model: NexusLowRes - version: 28 - locale: en - orientation: portrait - num-flaky-test-attempts: 0 - - flank: - max-test-shards: 50 - shard-time: -1 - num-test-runs: 1 - smart-flank-gcs-path: - smart-flank-disable-upload: false - default-test-time: 120.0 - use-average-test-time-for-new-tests: false - files-to-download: - test-targets-always-run: - - class actually.any.test.Clazz - disable-sharding: false - project: flank-open-source - local-result-dir: results - full-junit-result: false - # Android Flank Yml - keep-file-path: false - additional-app-test-apks: - run-timeout: -1 - legacy-junit-result: false - ignore-failed-tests: false - output-style: single - disable-results-upload: true - default-class-test-time: 240.0 - $emptyLines - RunTests - Uploading app-debug.apk \.* - 0 test / 0 shard - $emptyLines - 1 matrix ids created in \dm \ds - https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* - $emptyLines - Matrices webLink - matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* - [\s]*0[\s\S]*FINISHED[\s]* - $emptyLines - FetchArtifacts - $emptyLines - Updating matrix file - $emptyLines - CostReport - Virtual devices - ${escape("$")}(\d*.\d*) for \d*m - $emptyLines - MatrixResultsReport - 1 / 1 ${escape("(100.00%)")} - $emptyLines - $outputSummary - $emptyLines - Matrices webLink - matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* -""".trimIndent() - -val test = """ +private val test = """ version: local_snapshot revision: 55476259c1d391bf779affaa0ed9dc30692279de @@ -193,6 +121,9 @@ val test = """ files-to-download: test-targets-always-run: - class actually.any.test.Clazz + - class actually.any.test.Clazz1 + - class actually.any.test.Clazz2 + - class actually.any.test.Clazz3 disable-sharding: false project: flank-open-source local-result-dir: results @@ -214,6 +145,9 @@ val test = """ 1 matrix ids created in 0m 4s https://console.developers.google.com/storage/browser/test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-11-05_14-57-23.204686_dGGA + Saved 0 shards to android_shards.json + Uploading android_shards.json . + Matrices webLink matrix-2leuyoubnp892 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/8504866218843143096/executions/bs.b2bb28a35d2f2766 @@ -274,6 +208,7 @@ val test = """ │ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ ├─────────┼──────────────────────┼────────────────────────────┼──────────────┤ │ success │ matrix-2leuyoubnp892 │ NexusLowRes-28-en-portrait │ --- │ + │ failure │ matrix-2leuyoubnp892 │ NexusLowRes-28-en-portrait │ 1 test cases passed, 1 skipped │ └─────────┴──────────────────────┴────────────────────────────┴──────────────┘ Matrices webLink diff --git a/integration_tests/src/test/kotlin/utils/CommandHelper.kt b/integration_tests/src/test/kotlin/utils/CommandHelper.kt index 78a29fcddd..78dff99b0c 100644 --- a/integration_tests/src/test/kotlin/utils/CommandHelper.kt +++ b/integration_tests/src/test/kotlin/utils/CommandHelper.kt @@ -4,29 +4,18 @@ import java.io.File import java.util.concurrent.TimeUnit -fun String.runCommand(workingDir: File): ProcessResult = ProcessBuilder(*split("\\s".toRegex()).toTypedArray()) - .directory(workingDir).redirectOutputForCompatibleOS().startProcess() - - -private fun ProcessBuilder.startProcess() = start().run { - waitFor(processTimeout, TimeUnit.MINUTES) - ProcessResult(exitValue(), getOsSpecificOutput()) -} - -private fun Process.getOsSpecificOutput() = if (isWindows) { - File(outputFileName).readText() + File(errorFileName).readText() -} else { - inputStream.bufferedReader().readText() + this.errorStream.bufferedReader().readText() +fun String.runCommand(workingDir: File, testSuite: String): ProcessResult { + val outFile = File("$testSuite-$outputFileName").also { it.deleteOnExit() } + val errFile = File("$testSuite-$errorFileName").also { it.deleteOnExit() } + val result = ProcessBuilder(*split("\\s".toRegex()).toTypedArray()) + .directory(workingDir) + .redirectOutput(outFile) + .redirectError(errFile) + .start().also { it.waitFor(processTimeout, TimeUnit.MINUTES) } + + return ProcessResult(result.exitValue(), File(outFile.name).readText() + File(errFile.name).readText()) } -private fun ProcessBuilder.redirectOutputForCompatibleOS() = if (isWindows) { - redirectOutput(File(outputFileName)).redirectError(File(errorFileName)) -} else { - redirectOutput(ProcessBuilder.Redirect.PIPE).redirectError(ProcessBuilder.Redirect.PIPE) -} - -private val isWindows = System.getProperty("os.name").startsWith("Windows") - private const val outputFileName = "out.log" private const val errorFileName = "error.log" private const val processTimeout = 60L diff --git a/integration_tests/src/test/resources/all_test_filtered_ios.yml b/integration_tests/src/test/resources/all_test_filtered_ios.yml index b9a9b8e895..f5046f913a 100644 --- a/integration_tests/src/test/resources/all_test_filtered_ios.yml +++ b/integration_tests/src/test/resources/all_test_filtered_ios.yml @@ -1,5 +1,6 @@ gcloud: test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun +flank: test-targets: - - class nonExisting/Class + - nonExisting/Class diff --git a/integration_tests/src/test/resources/flank_android_full.yml b/integration_tests/src/test/resources/flank_android_full.yml index a890005a79..49cea3dfad 100644 --- a/integration_tests/src/test/resources/flank_android_full.yml +++ b/integration_tests/src/test/resources/flank_android_full.yml @@ -1,11 +1,12 @@ gcloud: app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk robo-script: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json + use-orchestrator: false flank: disable-sharding: false max-test-shards: 50 - disable-results-upload: true + output-style: single additional-app-test-apks: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk diff --git a/integration_tests/src/test/resources/sanity_robo.yml b/integration_tests/src/test/resources/sanity_robo.yml index 5c04ba3a78..3ad5a34c42 100644 --- a/integration_tests/src/test/resources/sanity_robo.yml +++ b/integration_tests/src/test/resources/sanity_robo.yml @@ -1,7 +1,2 @@ gcloud: app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk -flank: - disable-results-upload: true - max-test-shards: 50 - test-targets-always-run: - - class actually.any.test.Clazz From abf43cfacd51ac8bf07c35b27d780b93ed6081da Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 11 Nov 2020 13:26:19 +0100 Subject: [PATCH 03/32] Major refactor --- .../kotlin/integration/AllTestFilteredIT.kt | 21 +- .../test/kotlin/integration/AndroidFullIT.kt | 205 ++---------------- .../integration/IntergrationTestsUtils.kt | 56 +++-- .../test/kotlin/integration/SanityRoboIT.kt | 198 +---------------- .../src/test/kotlin/utils/CommandHelper.kt | 2 +- .../compare/AllTestFilteredIT-android-compare | 69 ++++++ .../resources/compare/AndroidFullIT-compare | 1 + .../resources/compare/AndroidFullIT-compare2 | 1 + .../resources/compare/SanityRoboIT-compare | 94 ++++++++ .../src/test/resources/flank_android_full.yml | 2 +- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 16 +- .../src/main/kotlin/ftl/args/IosArgs.kt | 10 +- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 32 +-- .../src/test/kotlin/ftl/args/IosArgsTest.kt | 20 +- 14 files changed, 270 insertions(+), 457 deletions(-) create mode 100644 integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare create mode 100644 integration_tests/src/test/resources/compare/AndroidFullIT-compare create mode 100644 integration_tests/src/test/resources/compare/AndroidFullIT-compare2 create mode 100644 integration_tests/src/test/resources/compare/SanityRoboIT-compare diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt index a946c10d1d..00c732f234 100644 --- a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -1,6 +1,7 @@ package integration import FlankCommand +import com.google.common.truth.Truth.assertThat import run import org.junit.Test @@ -8,30 +9,20 @@ class AllTestFilteredIT { @Test fun `filter all tests - android`() { + val name = this::class.java.simpleName + "-android" val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/all_test_filtered_android.yml", params = androidRunCommands ).run( workingDirectory = "./", - testSuite = this::class.java.simpleName + testSuite = name ) assertExitCode(result, 1) - Assert that result.output.removeUnicode() contains { - customConfig( - "test" with "[0-9a-zA-Z\\/_]*.apk", - "disable-sharding" with true, - "test-targets" with listOf( - "- class non.existing.Class" - ) - ) - output { - noneTestRan() - androidShards(tests = 0) - } - noOutcomeSummary() - } + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertNoOutcomeSummary(resOutput) } } diff --git a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt index fea3b6cadc..c105b67655 100644 --- a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt +++ b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt @@ -1,203 +1,30 @@ package integration import FlankCommand -import TestParameters -import org.junit.Before +import com.google.common.truth.Truth.assertThat import org.junit.Test import run -import toAndroidTestParameters -import utils.toStringMap class AndroidFullIT { + private val name = this::class.java.simpleName @Test fun `flank full option run`() { -// val result = FlankCommand( -// flankPath = "../test_runner/build/libs/flank.jar", -// ymlPath = "./src/test/resources/flank_android_full.yml", -// params = androidRunCommands -// ).run("./", this::class.java.simpleName) -// -// -// assertExitCode(result, 10) -// -// Assert that result.output.removeUnicode() contains { - Assert that testa contains { - customConfig( - "disable-sharding" with false, - "max-test-shards" with 50, - "additional-app-test-apks" with listOf( - "- app: null", - " test: [0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk", - "- app: null", - " test: [0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk", - "- app: null", - " test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk", - ), - "robo-script" with "[0-9a-zA-Z\\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json", - "use-orchestrator" with false, - "output-style" with "single" - ) - output { - matricesWebLink(linksNumber = 4) - androidShards( - tests = 11, - classes = 8, - shards = 19, - matrices = 4 - ) - fetchArtifacts() - costReport( - virtual = true, - physical = true - ) - resultsReport() - } - outcomeSummary { - success(number = 3) - failure(number = 1) - flaky(number = 0) - } - } - } -} - - -val testa = """ - version: local_snapshot -revision: c56cdec8b3586a64f3ff644a74372524613ee301 - -AndroidArgs - gcloud: - results-bucket: test-lab-v9cn46bb990nx-kz69ymd4nm9aq - results-dir: 2020-11-10_11-26-27.200686_GnSv - record-video: false - timeout: 15m - async: false - client-details: - network-profile: null - results-history-name: null - # Android gcloud - app: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: null - additional-apks: - auto-google-login: false - use-orchestrator: false - directories-to-pull: - grant-permissions: all - type: null - other-files: - scenario-numbers: - scenario-labels: - obb-files: - obb-names: - performance-metrics: false - num-uniform-shards: null - test-runner-class: null - test-targets: - robo-directives: - robo-script: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json - device: - - model: NexusLowRes - version: 28 - locale: en - orientation: portrait - num-flaky-test-attempts: 0 - test-targets-for-shard: - - flank: - max-test-shards: 50 - shard-time: -1 - num-test-runs: 1 - smart-flank-gcs-path: - smart-flank-disable-upload: false - default-test-time: 120.0 - use-average-test-time-for-new-tests: false - files-to-download: - test-targets-always-run: - disable-sharding: false - project: flank-open-source - local-result-dir: results - full-junit-result: false - # Android Flank Yml - keep-file-path: false - additional-app-test-apks: - - app: null - test: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - - app: null - test: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - - app: null - test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk - run-timeout: -1 - legacy-junit-result: false - ignore-failed-tests: false - output-style: single - disable-results-upload: false - default-class-test-time: 240.0 - -RunTests - - Smart Flank cache hit: 0% (0 / 9) - Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s - - - Smart Flank cache hit: 0% (0 / 9) - Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s - + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/flank_android_full.yml", + params = androidRunCommands + ).run("./", name) - Smart Flank cache hit: 0% (0 / 1) - Shard times: 120s -Saved 3 shards to android_shards.json - Uploading android_shards.json . - Uploading app-debug.apk . - Uploading app-multiple-error-debug-androidTest.apk . Uploading app-multiple-success-debug-androidTest.apk . Uploading MainActivity_robo_script.json . + assertExitCode(result, 10) - - 11 tests + 8 parameterized classes / 19 shards - - 4 matrix ids created in 0m 12s - https://console.developers.google.com/storage/browser/test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-11-10_11-26-27.200686_GnSv - -Matrices webLink - matrix-3rggyncbmbsu2 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6674251027591133234/executions/bs.8a98b8a47771a4c2 - matrix-2if6zvcy09fym https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5087540697861399228 - matrix-uv04kgwx4efqa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579/executions/bs.752e21ad35ba9970 - matrix-r9ejzytvt6jaa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 - -FetchArtifacts - ................... - Updating matrix file - -CostReport - Virtual devices - ${'$'}0.35 for 21m - - Uploading CostReport.txt . -MatrixResultsReport - 2 / 4 (50.00%) - 2 matrices failed - -┌─────────┬──────────────────────┬────────────────────────────┬────────────────────────────────┐ -│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ -├─────────┼──────────────────────┼────────────────────────────┼────────────────────────────────┤ -│ success │ matrix-3rggyncbmbsu2 │ NexusLowRes-28-en-portrait │ --- │ -│ success │ matrix-2if6zvcy09fym │ NexusLowRes-28-en-portrait │ 20 test cases passed │ -│ failure │ matrix-r9ejzytvt6jaa │ NexusLowRes-28-en-portrait │ 5 test cases failed, 15 passed │ -│ failure │ matrix-uv04kgwx4efqa │ NexusLowRes-28-en-portrait │ 0 test cases failed │ -└─────────┴──────────────────────┴────────────────────────────┴────────────────────────────────┘ -More details are available at: -https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 -https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579 - - Uploading MatrixResultsReport.txt . - Uploading HtmlErrorReport.html . - Uploading JUnitReport.xml . - -Matrices webLink - matrix-3rggyncbmbsu2 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6674251027591133234/executions/bs.8a98b8a47771a4c2 - matrix-r9ejzytvt6jaa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/9178244087966892822 - matrix-2if6zvcy09fym https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/5087540697861399228 - matrix-uv04kgwx4efqa https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/6407905251640120579/executions/bs.752e21ad35ba9970 -""".trimIndent() + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertContainsOutcomeSummary(resOutput) { + success = 3 + failure = 1 + } + } +} diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 08f26fbbb1..15cb7633d7 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -3,6 +3,7 @@ package integration import com.google.common.truth.Truth.assertThat import org.junit.Assert.assertEquals import utils.ProcessResult +import java.io.File import java.lang.StringBuilder import kotlin.text.Regex.Companion.escape @@ -38,7 +39,7 @@ fun FlankAssertion.customConfig(vararg customChecks: Pair = empt } } -private fun MutableMap.MutableEntry.makeKVRegex() = "${key}:\\s*${value}".toRegex() +private fun MutableMap.MutableEntry.makeKVRegex() = "$key:\\s*$value".toRegex() infix fun String.with(value: Any) = this to value.toString() @@ -53,7 +54,7 @@ fun FlankOutput.matricesWebLink(linksNumber: Int = 1) = lines.add(buildString { } }) -private fun shards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1, os: String) = buildString { +private fun shards(tests: Int, classes: Int, shards: Int, matrices: Int, os: String) = buildString { val testString = if (tests > 0) "tests" else "test" val classString = if (classes > 0) "classes" else "class" val shardString = if (shards > 0) "shards" else "shard" @@ -61,20 +62,23 @@ private fun shards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: flankOutputLine("[\\S\\s]*") flankOutputLine("Saved $shards shards to ${os}_shards.json") flankOutputLine("Uploading ${os}_shards.json .") - if (tests >= 0) { - flankOutputLine("Uploading app-debug.apk [\\s\\S]*") + if (tests >= 0 && matrices > 0) { + flankOutputLine("Uploading app-debug.apk[\\s\\S]*") if (classes == 0) flankOutputLine("$tests $testString / $shards $shardString") else flankOutputLine("$tests $testString + $classes parameterized $classString / $shards $shardString") + + flankOutputLine("$matrices matrix ids created in [0-9]{1,2}m [0-9]{1,2}s") + flankOutputLine("https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]*") } - flankOutputLine("$matrices matrix ids created in \\dm \\ds") - flankOutputLine("https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]*") } -fun FlankOutput.androidShards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1) = lines.add(shards(tests, classes, shards, matrices, "android")) +fun FlankOutput.androidShards(tests: Int, classes: Int, shards: Int, matrices: Int) = + lines.add(shards(tests, classes, shards, matrices, "android")) -fun FlankOutput.iosShards(tests: Int = 1, classes: Int = 0, shards: Int = 1, matrices: Int = 1) = lines.add(shards(tests, classes, shards, matrices, "ios")) +fun FlankOutput.iosShards(tests: Int, classes: Int, shards: Int, matrices: Int) = + lines.add(shards(tests, classes, shards, matrices, "ios")) fun FlankOutput.resultsReport(left: Int = 1, right: Int = 1) = lines.add(buildString { flankOutputLine("MatrixResultsReport") @@ -134,19 +138,29 @@ inline class FlankAssertion(val input: String) inline infix fun FlankAssertion.contains(block: FlankAssertion.() -> Unit) = this.run(block) +@Suppress("SetterBackingFieldAssignment") data class OutcomeSummary(val matcher: MutableMap = mutableMapOf().withDefault { 0 }) { - fun success(number: Int) = matcher.put(TestOutcome.SUCCESS, number) + var success: Int = 0 + set(value) { + matcher[TestOutcome.SUCCESS] = value + } - fun failure(number: Int) = matcher.put(TestOutcome.FAILURE, number) + var failure: Int = 0 + set(value) { + matcher[TestOutcome.FAILURE] = value + } - fun flaky(number: Int) = matcher.put(TestOutcome.FLAKY, number) + var flaky: Int = 0 + set(value) { + matcher[TestOutcome.FLAKY] = value + } } -fun FlankAssertion.outcomeSummary(block: OutcomeSummary.() -> Unit) = +fun assertContainsOutcomeSummary(input: String, block: OutcomeSummary.() -> Unit) = OutcomeSummary().apply(block).matcher.entries.forEach { (outcome, times) -> val actual = outcome.regex.findAll(input).toList().size - if (actual != times) throw AssertionError( - """Incorrect number of ${outcome.name} + if (actual != times) throw AssertionError(""" + |Incorrect number of ${outcome.name} | expected: $times | but was: $actual |Output: @@ -155,17 +169,19 @@ fun FlankAssertion.outcomeSummary(block: OutcomeSummary.() -> Unit) = ) } -fun FlankAssertion.noOutcomeSummary() { +fun assertNoOutcomeSummary(input: String) { if ("┌[\\s\\S]*┘".toRegex().matches(input)) throw AssertionError("There should be no outcome table.") } enum class TestOutcome(val regex: Regex) { - SUCCESS(fromCommon("success", "---")), - FLAKY(fromCommon("flaky", "[0-9a-z\\s,]*")), - FAILURE(fromCommon("failure", "[0-9a-z\\s,]*")), + SUCCESS(fromCommon("success")), + FLAKY(fromCommon("flaky")), + FAILURE(fromCommon("failure")), } private val fromCommon = - { result: String, details: String -> "│\\s$result\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s[a-zA-Z0-9-]*\\s│\\s$details\\s*│".toRegex() } + { outcome: String -> "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s[a-zA-Z0-9-]*\\s│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } + +fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent() -fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "") +fun findInCompare(name: String) = File("./src/test/resources/compare/$name-compare").readText().trimIndent() diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt index 10d0c752b1..4b640729a9 100644 --- a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -1,10 +1,12 @@ package integration import FlankCommand +import com.google.common.truth.Truth.assertThat import org.junit.Test import run class SanityRoboIT { + private val name = this::class.java.simpleName @Test fun `sanity robo`() { @@ -17,200 +19,12 @@ class SanityRoboIT { testSuite = this::class.java.simpleName ) - assertExitCode(result, 0) - Assert that result.output.removeUnicode() contains { - defaultConfig() - output { - matricesWebLink(linksNumber = 1) - androidShards( - tests = 0, - shards = 0, - matrices = 1 - ) - fetchArtifacts() - costReport( - virtual = true, - physical = false - ) - resultsReport() - } - outcomeSummary { - success(number = 1) - failure(number = 0) - flaky(number = 0) - } + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertContainsOutcomeSummary(resOutput) { + success = 1 } } - - @Test - fun testa() { - println() - Assert that test contains { -// config( -// "disable-results-upload" with true, -// "max-test-shards" with 50, -// "test-targets-always-run" with listOf( -// "- class actually.any.test.Clazz", -// "- class actually.any.test.Clazz1", -// "- class actually.any.test.Clazz2", -// "- class actually.any.test.Clazz3" -// ) -// -// ) - outcomeSummary { - success(1) -// failure(1) - flaky(0) - } - } - - } } - -private val test = """ - version: local_snapshot - revision: 55476259c1d391bf779affaa0ed9dc30692279de - - AndroidArgs - gcloud: - results-bucket: test-lab-v9cn46bb990nx-kz69ymd4nm9aq - results-dir: 2020-11-05_14-57-23.204686_dGGA - record-video: false - timeout: 15m - async: false - client-details: - network-profile: null - results-history-name: null - # Android gcloud - app: /Users/gogo/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: null - additional-apks: - auto-google-login: false - use-orchestrator: true - directories-to-pull: - grant-permissions: all - type: null - other-files: - scenario-numbers: - scenario-labels: - obb-files: - obb-names: - performance-metrics: false - num-uniform-shards: null - test-runner-class: null - test-targets: - robo-directives: - robo-script: null - device: - - model: NexusLowRes - version: 28 - locale: en - orientation: portrait - num-flaky-test-attempts: 0 - - flank: - max-test-shards: 50 - shard-time: -1 - num-test-runs: 1 - smart-flank-gcs-path: - smart-flank-disable-upload: false - default-test-time: 120.0 - use-average-test-time-for-new-tests: false - files-to-download: - test-targets-always-run: - - class actually.any.test.Clazz - - class actually.any.test.Clazz1 - - class actually.any.test.Clazz2 - - class actually.any.test.Clazz3 - disable-sharding: false - project: flank-open-source - local-result-dir: results - full-junit-result: false - # Android Flank Yml - keep-file-path: false - additional-app-test-apks: - run-timeout: -1 - legacy-junit-result: false - ignore-failed-tests: false - output-style: multi - disable-results-upload: true - default-class-test-time: 240.0 - - RunTests - Uploading app-debug.apk . - 0 test / 0 shard - - 1 matrix ids created in 0m 4s - https://console.developers.google.com/storage/browser/test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-11-05_14-57-23.204686_dGGA - - Saved 0 shards to android_shards.json - Uploading android_shards.json . - - Matrices webLink - matrix-2leuyoubnp892 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/8504866218843143096/executions/bs.b2bb28a35d2f2766 - - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 0s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 PENDING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 0m 38s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 RUNNING - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 1m 40s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Preparing device. - 2m 16s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Installing apps. - 2m 16s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Installing apps. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 2m 28s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting remote robo test. - 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. - 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. - 3m 34s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Retrieving Post-test Package Stats information from the device. - 3m 52s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Stopped logcat recording. - 3m 58s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 Starting results processing. Attempt: 1 - 4m 4s matrix-2leuyoubnp892 NexusLowRes-28 shard-0 FINISHED - 4m 4s matrix-2leuyoubnp892 FINISHED - - FetchArtifacts - - Updating matrix file - - CostReport - Virtual devices - ${'$'}0.03 for 2m - - MatrixResultsReport - 1 / 1 (100.00%) - ┌─────────┬──────────────────────┬────────────────────────────┬──────────────┐ - │ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ - ├─────────┼──────────────────────┼────────────────────────────┼──────────────┤ - │ success │ matrix-2leuyoubnp892 │ NexusLowRes-28-en-portrait │ --- │ - │ failure │ matrix-2leuyoubnp892 │ NexusLowRes-28-en-portrait │ 1 test cases passed, 1 skipped │ - └─────────┴──────────────────────┴────────────────────────────┴──────────────┘ - - Matrices webLink - matrix-2leuyoubnp892 https://console.firebase.google.com/project/flank-open-source/testlab/histories/bh.da0c237aaa33732/matrices/8504866218843143096/executions/bs.b2bb28a35d2f2766 -""".trimIndent() diff --git a/integration_tests/src/test/kotlin/utils/CommandHelper.kt b/integration_tests/src/test/kotlin/utils/CommandHelper.kt index 78dff99b0c..7fce235143 100644 --- a/integration_tests/src/test/kotlin/utils/CommandHelper.kt +++ b/integration_tests/src/test/kotlin/utils/CommandHelper.kt @@ -12,7 +12,7 @@ fun String.runCommand(workingDir: File, testSuite: String): ProcessResult { .redirectOutput(outFile) .redirectError(errFile) .start().also { it.waitFor(processTimeout, TimeUnit.MINUTES) } - + File("$testSuite-compare").writeText(File(outFile.name).readText() + File(errFile.name).readText()) return ProcessResult(result.exitValue(), File(outFile.name).readText() + File(errFile.name).readText()) } diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare new file mode 100644 index 0000000000..f7341a466d --- /dev/null +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -0,0 +1,69 @@ +AndroidArgs + gcloud: + results-bucket: test-lab-[a-zA-Z0-9-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + - class non.existing.Class + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path:\s + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: true + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: verbose + disable-results-upload: false + default-class-test-time: 240.0 + +RunTests + No tests for app-single-success-debug-androidTest.apk + +Saved 0 shards to android_shards.json + Uploading android_shards.json \.* + +There are no tests to run. diff --git a/integration_tests/src/test/resources/compare/AndroidFullIT-compare b/integration_tests/src/test/resources/compare/AndroidFullIT-compare new file mode 100644 index 0000000000..d57ca52361 --- /dev/null +++ b/integration_tests/src/test/resources/compare/AndroidFullIT-compare @@ -0,0 +1 @@ +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* 11 tests \+ 8 parameterized classes / 19 shards 4 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 3 / 4 \(\d{1,2}\.\d{1,2}\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 b/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 new file mode 100644 index 0000000000..43e5b8e2aa --- /dev/null +++ b/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 @@ -0,0 +1 @@ +Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare new file mode 100644 index 0000000000..2772d0ae97 --- /dev/null +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -0,0 +1,94 @@ +AndroidArgs + gcloud: + results-bucket: test-lab-[a-zA-Z0-9-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: null + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path:\s + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: false + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: verbose + disable-results-upload: false + default-class-test-time: 240.0 + +RunTests + Uploading app-debug.apk \.* + 0 test / 0 shard + + 1 matrix ids created in \d{1,2}m \d{1,2}s + https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + +Matrices webLink + matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* + + [\s\S]* + +FetchArtifacts + + Updating matrix file + +CostReport + Virtual devices + \$\d{1,2}.\d{1,2} for \d{1,2}m + + Uploading CostReport.txt \.* +MatrixResultsReport + 1 / 1 \(100\.00\%\) +┌─────────┬──────────────────────┬────────────────────────────┬──────────────┐ +│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ +├─────────┼──────────────────────┼────────────────────────────┼──────────────┤ +│ success │ matrix-[a-zA-Z0-9\s]*│ NexusLowRes-28-en-portrait │ --- │ +└─────────┴──────────────────────┴────────────────────────────┴──────────────┘ + Uploading MatrixResultsReport.txt \.* + Uploading JUnitReport.xml \.* + +Matrices webLink + matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* diff --git a/integration_tests/src/test/resources/flank_android_full.yml b/integration_tests/src/test/resources/flank_android_full.yml index 49cea3dfad..e5f5b3fe22 100644 --- a/integration_tests/src/test/resources/flank_android_full.yml +++ b/integration_tests/src/test/resources/flank_android_full.yml @@ -10,4 +10,4 @@ flank: additional-app-test-apks: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - - test: gs://flank-open-source.appspot.com/test/app-single-success-debug-androidTest.apk + - test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 6b8b3e42fa..29c6531d78 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -37,23 +37,23 @@ AndroidArgs record-video: $recordVideo timeout: $testTimeout async: $async - client-details: ${ArgsToString.mapToString(clientDetails)} + client-details:${ArgsToString.mapToString(clientDetails)} network-profile: $networkProfile results-history-name: $resultsHistoryName # Android gcloud app: $appApk test: $testApk - additional-apks: ${ArgsToString.listToString(additionalApks)} + additional-apks:${ArgsToString.listToString(additionalApks)} auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator - directories-to-pull: ${ArgsToString.listToString(directoriesToPull)} + directories-to-pull:${ArgsToString.listToString(directoriesToPull)} grant-permissions: $grantPermissions type: ${type?.ymlName} - other-files: ${ArgsToString.mapToString(otherFiles)} - scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)} - scenario-labels: ${ArgsToString.listToString(scenarioLabels)} - obb-files: ${ArgsToString.listToString(obbFiles)} - obb-names: ${ArgsToString.listToString(obbNames)} + other-files:${ArgsToString.mapToString(otherFiles)} + scenario-numbers:${ArgsToString.listToString(scenarioNumbers)} + scenario-labels:${ArgsToString.listToString(scenarioLabels)} + obb-files:${ArgsToString.listToString(obbFiles)} + obb-names:${ArgsToString.listToString(obbNames)} performance-metrics: $performanceMetrics num-uniform-shards: $numUniformShards test-runner-class: $testRunnerClass diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 1909c2f26e..9757967db7 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -33,7 +33,7 @@ IosArgs record-video: $recordVideo timeout: $testTimeout async: $async - client-details: ${ArgsToString.mapToString(clientDetails)} + client-details:${ArgsToString.mapToString(clientDetails)} network-profile: $networkProfile results-history-name: $resultsHistoryName # iOS gcloud @@ -42,10 +42,10 @@ IosArgs xcode-version: $xcodeVersion device:${ArgsToString.objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts - directories-to-pull: ${ArgsToString.listToString(directoriesToPull)} - other-files: ${ArgsToString.mapToString(otherFiles)} - additional-ipas: ${ArgsToString.listToString(additionalIpas)} - scenario-numbers: ${ArgsToString.listToString(scenarioNumbers)} + directories-to-pull:${ArgsToString.listToString(directoriesToPull)} + other-files:${ArgsToString.mapToString(otherFiles)} + additional-ipas:${ArgsToString.listToString(additionalIpas)} + scenario-numbers:${ArgsToString.listToString(scenarioNumbers)} type: ${type?.ymlName} app: $app test-special-entitlements: $testSpecialEntitlements diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index e8d1e1ab55..676b34922a 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -302,7 +302,7 @@ AndroidArgs record-video: false timeout: 70m async: true - client-details: + client-details: key1: value1 key2: value2 network-profile: LTE @@ -310,23 +310,23 @@ AndroidArgs # Android gcloud app: $appApkAbsolutePath test: $testApkAbsolutePath - additional-apks: + additional-apks: - $testErrorApkAbsolutePath - $testFlakyApkAbsolutePath auto-google-login: false use-orchestrator: false - directories-to-pull: + directories-to-pull: - /sdcard/screenshots - /sdcard/screenshots2 grant-permissions: all type: instrumentation - other-files: + other-files: /sdcard/dir1/file1.txt: $appApkAbsolutePath /sdcard/dir2/file2.jpg: $testApkAbsolutePath - scenario-numbers: - scenario-labels: - obb-files: - obb-names: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: com.foo.TestRunner @@ -393,23 +393,23 @@ AndroidArgs record-video: false timeout: 15m async: false - client-details: + client-details: network-profile: null results-history-name: null # Android gcloud app: $appApkAbsolutePath test: $testApkAbsolutePath - additional-apks: + additional-apks: auto-google-login: false use-orchestrator: true - directories-to-pull: + directories-to-pull: grant-permissions: all type: null - other-files: - scenario-numbers: - scenario-labels: - obb-files: - obb-names: + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index ed67c4cde9..93df770f6f 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -217,7 +217,7 @@ IosArgs record-video: false timeout: 70m async: true - client-details: + client-details: key1: value1 key2: value2 network-profile: LTE @@ -236,14 +236,14 @@ IosArgs locale: c orientation: default num-flaky-test-attempts: 4 - directories-to-pull: - other-files: + directories-to-pull: + other-files: com.my.app:/Documents/file.txt: local/file.txt /private/var/mobile/Media/file.jpg: gs://bucket/file.jpg - additional-ipas: + additional-ipas: - $testIpa1 - $testIpa2 - scenario-numbers: + scenario-numbers: type: xctest app: test-special-entitlements: true @@ -292,7 +292,7 @@ IosArgs record-video: false timeout: 15m async: false - client-details: + client-details: network-profile: null results-history-name: null # iOS gcloud @@ -305,10 +305,10 @@ IosArgs locale: en orientation: portrait num-flaky-test-attempts: 0 - directories-to-pull: - other-files: - additional-ipas: - scenario-numbers: + directories-to-pull: + other-files: + additional-ipas: + scenario-numbers: type: xctest app: test-special-entitlements: false From 183760a1538c96d18bf4d47f79bc06e7e38533ae Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 11 Nov 2020 13:27:32 +0100 Subject: [PATCH 04/32] remove --- .../src/test/resources/compare/AndroidFullIT-compare2 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 integration_tests/src/test/resources/compare/AndroidFullIT-compare2 diff --git a/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 b/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 deleted file mode 100644 index 43e5b8e2aa..0000000000 --- a/integration_tests/src/test/resources/compare/AndroidFullIT-compare2 +++ /dev/null @@ -1 +0,0 @@ -Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file From d06c2ac3ec3698550e787e12325ba5a45642bf0e Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 11 Nov 2020 13:31:24 +0100 Subject: [PATCH 05/32] Update gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e3ffefe4ee..40c1eb16b4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ results xcuserdata/ test_projects/ios/*/build* android_shards.json +ios_shards.json +*.log From 7bea932d88d7c8d6d112b33d89b663058394a0d5 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 11 Nov 2020 13:41:35 +0100 Subject: [PATCH 06/32] Update template --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fcd43837ed..1fa9873836 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,3 +9,4 @@ Fixes # - [ ] Documented - [ ] Unit tested +- [ ] Integration tests updated From 3512e49801999ae654f5b794f4554cd63a914931 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 12 Nov 2020 07:26:40 +0100 Subject: [PATCH 07/32] Add tests --- .../kotlin/integration/AllTestFilteredIT.kt | 21 ++- .../test/kotlin/integration/AndroidFullIT.kt | 2 +- .../test/kotlin/integration/DefaultConfigs.kt | 83 ------------ .../test/kotlin/integration/IgnoreFailedIT.kt | 30 +++++ .../integration/IntergrationTestsUtils.kt | 125 +----------------- .../kotlin/integration/MultipleDevicesIT.kt | 31 +++++ .../test/kotlin/integration/RunTimeoutIT.kt | 28 ++++ .../test/kotlin/integration/SanityRoboIT.kt | 2 +- .../kotlin/integration/TestFilteringIT.kt | 30 +++++ .../src/test/kotlin/utils/CommandHelper.kt | 2 +- .../{ => cases}/all_test_filtered_android.yml | 0 .../{ => cases}/all_test_filtered_ios.yml | 0 .../cases/flank_android_ignore_failed.yml | 7 + .../flank_android_multiple_apk.yml} | 0 .../cases/flank_android_multiple_devices.yml | 20 +++ .../cases/flank_android_run_timeout.yml | 8 ++ .../resources/{ => cases}/sanity_robo.yml | 0 .../cases/test_filtering_android.yml | 10 ++ .../compare/AllTestFilteredIT-ios-compare | 79 +++++++++++ .../resources/compare/IgnoreFailedIT-compare | 1 + .../compare/MultipleDevicesIT-android-compare | 1 + .../resources/compare/RunTimeoutIT-compare | 82 ++++++++++++ .../resources/compare/TestFilteringIT-compare | 98 ++++++++++++++ 23 files changed, 451 insertions(+), 209 deletions(-) delete mode 100644 integration_tests/src/test/kotlin/integration/DefaultConfigs.kt create mode 100644 integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt create mode 100644 integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt create mode 100644 integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt create mode 100644 integration_tests/src/test/kotlin/integration/TestFilteringIT.kt rename integration_tests/src/test/resources/{ => cases}/all_test_filtered_android.yml (100%) rename integration_tests/src/test/resources/{ => cases}/all_test_filtered_ios.yml (100%) create mode 100644 integration_tests/src/test/resources/cases/flank_android_ignore_failed.yml rename integration_tests/src/test/resources/{flank_android_full.yml => cases/flank_android_multiple_apk.yml} (100%) create mode 100644 integration_tests/src/test/resources/cases/flank_android_multiple_devices.yml create mode 100644 integration_tests/src/test/resources/cases/flank_android_run_timeout.yml rename integration_tests/src/test/resources/{ => cases}/sanity_robo.yml (100%) create mode 100644 integration_tests/src/test/resources/cases/test_filtering_android.yml create mode 100644 integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare create mode 100644 integration_tests/src/test/resources/compare/IgnoreFailedIT-compare create mode 100644 integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare create mode 100644 integration_tests/src/test/resources/compare/RunTimeoutIT-compare create mode 100644 integration_tests/src/test/resources/compare/TestFilteringIT-compare diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt index 00c732f234..58c3e3871b 100644 --- a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -12,7 +12,7 @@ class AllTestFilteredIT { val name = this::class.java.simpleName + "-android" val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/all_test_filtered_android.yml", + ymlPath = "./src/test/resources/cases/all_test_filtered_android.yml", params = androidRunCommands ).run( workingDirectory = "./", @@ -25,4 +25,23 @@ class AllTestFilteredIT { assertThat(resOutput).containsMatch(findInCompare(name)) assertNoOutcomeSummary(resOutput) } + + @Test + fun `filter all tests - ios`() { + val name = this::class.java.simpleName + "-ios" + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/cases/all_test_filtered_ios.yml", + params = iosRunCommands + ).run( + workingDirectory = "./", + testSuite = name + ) + + assertExitCode(result, 0) + + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertNoOutcomeSummary(resOutput) + } } diff --git a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt index c105b67655..08afc64c60 100644 --- a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt +++ b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt @@ -13,7 +13,7 @@ class AndroidFullIT { fun `flank full option run`() { val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/flank_android_full.yml", + ymlPath = "./src/test/resources/cases/flank_android_multiple_apk.yml", params = androidRunCommands ).run("./", name) diff --git a/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt b/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt deleted file mode 100644 index d452468502..0000000000 --- a/integration_tests/src/test/kotlin/integration/DefaultConfigs.kt +++ /dev/null @@ -1,83 +0,0 @@ -package integration - -val androidRunCommands = listOf("firebase", "test", "android", "run") -val iosRunCommands = listOf("firebase", "test", "ios", "run") -private const val EMPTY_STRING = "" -private const val NULL_STRING = "null" - -val defaultCommonConfig = mapOf( - "results-bucket" to "test-lab-[a-zA-Z0-9-]*", - "results-dir" to "[.a-zA-Z0-9_-]*", - "record-video" to false, - "timeout" to "15m", - "async" to false, - "client-details" to EMPTY_STRING, - "network-profile" to NULL_STRING, - "results-history-name" to NULL_STRING, - "num-flaky-test-attempts" to 0, - "directories-to-pull" to EMPTY_STRING, - "other-files" to EMPTY_STRING, - "max-test-shards" to 1, - "shard-time" to -1, - "num-test-runs" to 1, - "smart-flank-gcs-path" to EMPTY_STRING, - "smart-flank-disable-upload" to false, - "default-test-time" to 120.0, - "use-average-test-time-for-new-tests" to false, - "files-to-download" to EMPTY_STRING, - "test-targets-always-run" to EMPTY_STRING, - "disable-sharding" to false, - "project" to "flank-open-source", - "local-result-dir" to "results", - "full-junit-result" to false, - "keep-file-path" to false, - "ignore-failed-tests" to false, - "output-style" to "verbose", - "disable-results-upload" to false, - "default-class-test-time" to 240.0, - "run-timeout" to -1, - "scenario-numbers" to EMPTY_STRING, -) - -val defaultAndroidConfig = defaultCommonConfig + mapOf( - "app" to "[0-9a-zA-Z\\/_]*.apk", - "test" to NULL_STRING, - "additional-apks" to EMPTY_STRING, - "auto-google-login" to false, - "use-orchestrator" to true, - "grant-permissions" to "all", - "type" to NULL_STRING, - "scenario-labels" to EMPTY_STRING, - "obb-files" to EMPTY_STRING, - "obb-names" to EMPTY_STRING, - "performance-metrics" to false, - "num-uniform-shards" to NULL_STRING, - "test-runner-class" to NULL_STRING, - "test-targets" to EMPTY_STRING, - "robo-directives" to EMPTY_STRING, - "robo-script" to NULL_STRING, - "device" to """ - - model: NexusLowRes - version: 28 - locale: en - orientation: portrait -""".replace(System.lineSeparator(), "[\\s]*"), - "additional-app-test-apks" to EMPTY_STRING, - "legacy-junit-result" to false, -) - -val defaultIosConfig = mapOf( - "test" to "[0-9a-zA-Z\\/_]*.zip", - "xctestrun-file" to "[0-9a-zA-Z\\/_].xctestrun", - "xcode-version" to NULL_STRING, - "device" to """ - - model: iphone8 - version: 12.0 - locale: en - orientation: portrait -""".replace(System.lineSeparator(), "[\\s]*"), - "additional-ipas" to EMPTY_STRING, - "type" to "xctest", - "app" to EMPTY_STRING, - "test-special-entitlements" to false -) diff --git a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt new file mode 100644 index 0000000000..5db3b5e7d5 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt @@ -0,0 +1,30 @@ +package integration + +import FlankCommand +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import run + +class IgnoreFailedIT { + + @Test + fun `return with exit code 0 for failed tests`() { + val name = this::class.java.simpleName + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/cases/flank_android_ignore_failed.yml", + params = androidRunCommands + ).run( + workingDirectory = "./", + testSuite = name + ) + + assertExitCode(result, 0) + + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertContainsOutcomeSummary(resOutput) { + failure = 1 + } + } +} diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 15cb7633d7..a1b22f584b 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -1,121 +1,11 @@ package integration -import com.google.common.truth.Truth.assertThat import org.junit.Assert.assertEquals import utils.ProcessResult import java.io.File -import java.lang.StringBuilder -import kotlin.text.Regex.Companion.escape -fun FlankAssertion.defaultConfig() = customConfig() - -fun FlankAssertion.customConfig(vararg customChecks: Pair = emptyArray()) { - val checks = defaultAndroidConfig.toMutableMap() - customChecks.forEach { checks[it.first] = it.second } - checks.entries - .forEach { - if (it.makeKVRegex().find(input) == null) { - val message = """ - |Incorrect value for [${it.key}]. - |Expected: - | ${it.key}: ${ - it.value.toString() - .replace("-", "${System.lineSeparator()} -") - .replace("""\s*""", "") - } - |Was: - | ${ - "${it.key}:[\\s\\S]{10,60}" - .toRegex() - .find(input) - ?.value - ?.split(System.lineSeparator()) - ?.joinToString { value -> "${System.lineSeparator()} " + value.trim() } - }(...) - |Output: - |$input""".trimMargin() - throw AssertionError(message) - } - } -} - -private fun MutableMap.MutableEntry.makeKVRegex() = "$key:\\s*$value".toRegex() - -infix fun String.with(value: Any) = this to value.toString() - -infix fun String.with(values: List) = this to buildString { values.forEach(::flankOutputLine) } - -private fun StringBuilder.flankOutputLine(line: String) = append(line).append("\\s*") - -fun FlankOutput.matricesWebLink(linksNumber: Int = 1) = lines.add(buildString { - flankOutputLine("Matrices webLink") - repeat(linksNumber) { - flankOutputLine("matrix-[a-zA-Z0-9]+ https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)?") - } -}) - -private fun shards(tests: Int, classes: Int, shards: Int, matrices: Int, os: String) = buildString { - val testString = if (tests > 0) "tests" else "test" - val classString = if (classes > 0) "classes" else "class" - val shardString = if (shards > 0) "shards" else "shard" - flankOutputLine("RunTests") - flankOutputLine("[\\S\\s]*") - flankOutputLine("Saved $shards shards to ${os}_shards.json") - flankOutputLine("Uploading ${os}_shards.json .") - if (tests >= 0 && matrices > 0) { - flankOutputLine("Uploading app-debug.apk[\\s\\S]*") - if (classes == 0) - flankOutputLine("$tests $testString / $shards $shardString") - else - flankOutputLine("$tests $testString + $classes parameterized $classString / $shards $shardString") - - flankOutputLine("$matrices matrix ids created in [0-9]{1,2}m [0-9]{1,2}s") - flankOutputLine("https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]*") - } -} - -fun FlankOutput.androidShards(tests: Int, classes: Int, shards: Int, matrices: Int) = - lines.add(shards(tests, classes, shards, matrices, "android")) - -fun FlankOutput.iosShards(tests: Int, classes: Int, shards: Int, matrices: Int) = - lines.add(shards(tests, classes, shards, matrices, "ios")) - -fun FlankOutput.resultsReport(left: Int = 1, right: Int = 1) = lines.add(buildString { - flankOutputLine("MatrixResultsReport") - flankOutputLine("$left / $right ${escape("(100.00%)")}") -}) - -fun FlankOutput.noneTestRan() = lines.apply { - add(buildString { - flankOutputLine("RunTests") - flankOutputLine("No tests for [a-zA-Z0-9._-]*.apk") - }) - add("There are no tests to run") -} - -fun FlankOutput.fetchArtifacts() = lines.add(buildString { - flankOutputLine("FetchArtifacts") - flankOutputLine("\\.*") - flankOutputLine("Updating matrix file") -}) - -fun FlankOutput.costReport(virtual: Boolean, physical: Boolean) = lines.add(buildString { - flankOutputLine("CostReport") - if (virtual) { - flankOutputLine("Virtual devices") - flankOutputLine("\\$(\\d*.\\d*) for \\d*m") - } - if (physical) { - flankOutputLine("Physical devices") - flankOutputLine("\\$(\\d*.\\d*) for \\d*m") - } -}) - -data class FlankOutput(val lines: MutableList = mutableListOf()) - -infix fun FlankAssertion.output(block: FlankOutput.() -> Unit) = assertThat(input).run { - FlankOutput().apply(block).lines.forEach { containsMatch(it) } -} +val androidRunCommands = listOf("firebase", "test", "android", "run") +val iosRunCommands = listOf("firebase", "test", "ios", "run") fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals( """ @@ -129,15 +19,6 @@ fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals( result.exitCode ) -object Assert { - infix fun that(input: String) = FlankAssertion(input) -} - -@Suppress("EXPERIMENTAL_FEATURE_WARNING") -inline class FlankAssertion(val input: String) - -inline infix fun FlankAssertion.contains(block: FlankAssertion.() -> Unit) = this.run(block) - @Suppress("SetterBackingFieldAssignment") data class OutcomeSummary(val matcher: MutableMap = mutableMapOf().withDefault { 0 }) { var success: Int = 0 @@ -180,7 +61,7 @@ enum class TestOutcome(val regex: Regex) { } private val fromCommon = - { outcome: String -> "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s[a-zA-Z0-9-]*\\s│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } + { outcome: String -> "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent() diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt new file mode 100644 index 0000000000..80153cf5fc --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -0,0 +1,31 @@ +package integration + +import FlankCommand +import com.google.common.truth.Truth.assertThat +import run +import org.junit.Test + +class MultipleDevicesIT { + + @Test + fun `run tests on multiple devices - android`() { + val name = this::class.java.simpleName + "-android" + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/cases/flank_android_multiple_devices.yml", + params = androidRunCommands + ).run( + workingDirectory = "./", + testSuite = name + ) + + assertExitCode(result, 10) + + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertContainsOutcomeSummary(resOutput) { + success = 6 + failure = 3 + } + } +} diff --git a/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt new file mode 100644 index 0000000000..57d1e21d46 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt @@ -0,0 +1,28 @@ +package integration + +import FlankCommand +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import run + +class RunTimeoutIT { + + @Test + fun `cancel test run on timeout`() { + val name = this::class.java.simpleName + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/cases/flank_android_run_timeout.yml", + params = androidRunCommands + ).run( + workingDirectory = "./", + testSuite = name + ) + + assertExitCode(result, 1) + + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertNoOutcomeSummary(resOutput) + } +} diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt index 4b640729a9..4714634c81 100644 --- a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -12,7 +12,7 @@ class SanityRoboIT { fun `sanity robo`() { val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/sanity_robo.yml", + ymlPath = "./src/test/resources/cases/sanity_robo.yml", params = androidRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt new file mode 100644 index 0000000000..d7dd2951f5 --- /dev/null +++ b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt @@ -0,0 +1,30 @@ +package integration + +import FlankCommand +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import run + +class TestFilteringIT { + + @Test + fun `run test from only one apk`() { + val name = this::class.java.simpleName + val result = FlankCommand( + flankPath = "../test_runner/build/libs/flank.jar", + ymlPath = "./src/test/resources/cases/test_filtering_android.yml", + params = androidRunCommands + ).run( + workingDirectory = "./", + testSuite = name + ) + + assertExitCode(result, 0) + + val resOutput = result.output.removeUnicode() + assertThat(resOutput).containsMatch(findInCompare(name)) + assertContainsOutcomeSummary(resOutput) { + success = 1 + } + } +} diff --git a/integration_tests/src/test/kotlin/utils/CommandHelper.kt b/integration_tests/src/test/kotlin/utils/CommandHelper.kt index 7fce235143..4ce7251f8e 100644 --- a/integration_tests/src/test/kotlin/utils/CommandHelper.kt +++ b/integration_tests/src/test/kotlin/utils/CommandHelper.kt @@ -12,7 +12,7 @@ fun String.runCommand(workingDir: File, testSuite: String): ProcessResult { .redirectOutput(outFile) .redirectError(errFile) .start().also { it.waitFor(processTimeout, TimeUnit.MINUTES) } - File("$testSuite-compare").writeText(File(outFile.name).readText() + File(errFile.name).readText()) +// File("$testSuite-compare").writeText(File(outFile.name).readText() + File(errFile.name).readText()) // to remove return ProcessResult(result.exitValue(), File(outFile.name).readText() + File(errFile.name).readText()) } diff --git a/integration_tests/src/test/resources/all_test_filtered_android.yml b/integration_tests/src/test/resources/cases/all_test_filtered_android.yml similarity index 100% rename from integration_tests/src/test/resources/all_test_filtered_android.yml rename to integration_tests/src/test/resources/cases/all_test_filtered_android.yml diff --git a/integration_tests/src/test/resources/all_test_filtered_ios.yml b/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml similarity index 100% rename from integration_tests/src/test/resources/all_test_filtered_ios.yml rename to integration_tests/src/test/resources/cases/all_test_filtered_ios.yml diff --git a/integration_tests/src/test/resources/cases/flank_android_ignore_failed.yml b/integration_tests/src/test/resources/cases/flank_android_ignore_failed.yml new file mode 100644 index 0000000000..a987478b65 --- /dev/null +++ b/integration_tests/src/test/resources/cases/flank_android_ignore_failed.yml @@ -0,0 +1,7 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-error-debug-androidTest.apk + +flank: + disable-results-upload: true + ignore-failed-tests: true diff --git a/integration_tests/src/test/resources/flank_android_full.yml b/integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml similarity index 100% rename from integration_tests/src/test/resources/flank_android_full.yml rename to integration_tests/src/test/resources/cases/flank_android_multiple_apk.yml diff --git a/integration_tests/src/test/resources/cases/flank_android_multiple_devices.yml b/integration_tests/src/test/resources/cases/flank_android_multiple_devices.yml new file mode 100644 index 0000000000..3b6fd89559 --- /dev/null +++ b/integration_tests/src/test/resources/cases/flank_android_multiple_devices.yml @@ -0,0 +1,20 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + use-orchestrator: false + num-flaky-test-attempts: 2 + device: + - model: NexusLowRes + version: 28 + - model: Pixel2 + version: 28 + - model: HUR + version: 28 + +flank: + disable-sharding: false + max-test-shards: 5 + output-style: single + additional-app-test-apks: + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk + - test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk diff --git a/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml b/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml new file mode 100644 index 0000000000..f629b86964 --- /dev/null +++ b/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml @@ -0,0 +1,8 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + +flank: + disable-results-upload: true + disable-sharding: true + run-timeout: 1m diff --git a/integration_tests/src/test/resources/sanity_robo.yml b/integration_tests/src/test/resources/cases/sanity_robo.yml similarity index 100% rename from integration_tests/src/test/resources/sanity_robo.yml rename to integration_tests/src/test/resources/cases/sanity_robo.yml diff --git a/integration_tests/src/test/resources/cases/test_filtering_android.yml b/integration_tests/src/test/resources/cases/test_filtering_android.yml new file mode 100644 index 0000000000..c8f2b68a26 --- /dev/null +++ b/integration_tests/src/test/resources/cases/test_filtering_android.yml @@ -0,0 +1,10 @@ +gcloud: + app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + test-targets: + - class com.example.test_app.InstrumentedTest#test2 + +flank: + disable-sharding: true + additional-app-test-apks: + - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare new file mode 100644 index 0000000000..bea3fcb8c2 --- /dev/null +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare @@ -0,0 +1,79 @@ +IosArgs + gcloud: + results-bucket: test-lab-[a-zA-Z0-9-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # iOS gcloud + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + xctestrun-file: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + xcode-version: null + device: + - model: iphone8 + version: 12.0 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + directories-to-pull: + other-files: + additional-ipas: + scenario-numbers: + type: xctest + app: + test-special-entitlements: false + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path:\s + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + test-targets-always-run: + files-to-download: + keep-file-path: false + full-junit-result: false + # iOS flank + test-targets: + - nonExisting/Class + disable-sharding: false + project: flank-open-source + local-result-dir: results + run-timeout: -1 + ignore-failed-tests: false + output-style: verbose + disable-results-upload: false + default-class-test-time: 240.0 + +RunTests +Found xctest: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest +isMacOS = true \(mac os x\) +nm -U "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" +nm -gU "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" | xargs -s 262144 xcrun swift-demangle +Saved 0 shards to ios_shards.json + Uploading ios_shards.json \.* + Uploading earlgrey_example.zip \*. + 0 test / 0 shard + + 0 matrix ids created in \d{1,2}m \d{1,2}s + https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + +Matrices webLink + + +FetchArtifacts + +CostReport + No cost. 0m + + Uploading CostReport.txt \.* +MatrixResultsReport + 0 / 0 \(NaN\%\) + Uploading MatrixResultsReport.txt \.* + +Matrices webLink diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare new file mode 100644 index 0000000000..e6a147d69a --- /dev/null +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -0,0 +1 @@ +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test / 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 / 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare new file mode 100644 index 0000000000..7c0894f491 --- /dev/null +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -0,0 +1 @@ +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* 11 tests \+ 8 parameterized classes / 11 shards 3 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 2 / 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]*/matrix_\d/\s*){17} Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare new file mode 100644 index 0000000000..e936bf3a43 --- /dev/null +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -0,0 +1,82 @@ +version: local_snapshot +revision: 451aadbe4bfa0fa64f6cab20713bc228e897437e + +WARNING: disable-sharding enabled with max-test-shards = 1, Flank will ignore max-test-shard and disable sharding. +AndroidArgs + gcloud: + results-bucket: test-lab-[a-zA-Z0-9-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path:\s + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: true + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + run-timeout: 1m + legacy-junit-result: false + ignore-failed-tests: false + output-style: verbose + disable-results-upload: true + default-class-test-time: 240.0 + +RunTests +Saved 1 shards to android_shards.json + Uploading app-debug.apk \.* + Uploading app-single-success-debug-androidTest.apk \.* + 1 test / 1 shard + + 1 matrix ids created in \d{1,2}m \d{1,2}s + https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + +Matrices webLink + matrix-[0-9a-zA-Z\/_.-]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? + +[\s\S]* + +Canceling flank due to timeout +CancelMatrices + Cancelling 1x matrices diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare new file mode 100644 index 0000000000..f307e50450 --- /dev/null +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -0,0 +1,98 @@ +AndroidArgs + gcloud: + results-bucket: test-lab-[a-zA-Z0-9-]* + results-dir: [.a-zA-Z0-9_-]* + record-video: false + timeout: 15m + async: false + client-details: + network-profile: null + results-history-name: null + # Android gcloud + app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + additional-apks: + auto-google-login: false + use-orchestrator: true + directories-to-pull: + grant-permissions: all + type: null + other-files: + scenario-numbers: + scenario-labels: + obb-files: + obb-names: + performance-metrics: false + num-uniform-shards: null + test-runner-class: null + test-targets: + - class com.example.test_app.InstrumentedTest#test2 + robo-directives: + robo-script: null + device: + - model: NexusLowRes + version: 28 + locale: en + orientation: portrait + num-flaky-test-attempts: 0 + + flank: + max-test-shards: 1 + shard-time: -1 + num-test-runs: 1 + smart-flank-gcs-path: + smart-flank-disable-upload: false + default-test-time: 120.0 + use-average-test-time-for-new-tests: false + files-to-download: + test-targets-always-run: + disable-sharding: true + project: flank-open-source + local-result-dir: results + full-junit-result: false + # Android Flank Yml + keep-file-path: false + additional-app-test-apks: + - app: null + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + run-timeout: -1 + legacy-junit-result: false + ignore-failed-tests: false + output-style: verbose + disable-results-upload: false + default-class-test-time: 240.0 + +RunTests + No tests for app-single-success-debug-androidTest.apk + +Saved 1 shards to android_shards.json + Uploading android_shards.json \.* + Uploading app-debug.apk \.* + Uploading app-multiple-success-debug-androidTest.apk \.* + 1 test / 1 shard + + 1 matrix ids created in \d{1,2}m \d{1,2}s + https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + +Matrices webLink + matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? + + [\s\S]* + +FetchArtifacts + \.* + Updating matrix file + +CostReport + Virtual devices + \$\d{1,2}.\d{1,2} for \d{1,2}m + + Uploading CostReport.txt \.* +MatrixResultsReport + 1 / 1 \(100\.00\%\) +[\s\S]* + Uploading MatrixResultsReport.txt \.* + Uploading JUnitReport.xml \.* + +Matrices webLink + matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? From 3f26e3eed5b406300afb58b1229bd6de180e6d33 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 12 Nov 2020 08:05:53 +0100 Subject: [PATCH 08/32] Linting --- integration_tests/src/test/kotlin/integration/AndroidFullIT.kt | 1 - integration_tests/src/test/kotlin/utils/CommandHelper.kt | 2 -- 2 files changed, 3 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt index 08afc64c60..b0238f7865 100644 --- a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt +++ b/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt @@ -5,7 +5,6 @@ import com.google.common.truth.Truth.assertThat import org.junit.Test import run - class AndroidFullIT { private val name = this::class.java.simpleName diff --git a/integration_tests/src/test/kotlin/utils/CommandHelper.kt b/integration_tests/src/test/kotlin/utils/CommandHelper.kt index 4ce7251f8e..5acae62cc9 100644 --- a/integration_tests/src/test/kotlin/utils/CommandHelper.kt +++ b/integration_tests/src/test/kotlin/utils/CommandHelper.kt @@ -3,7 +3,6 @@ package utils import java.io.File import java.util.concurrent.TimeUnit - fun String.runCommand(workingDir: File, testSuite: String): ProcessResult { val outFile = File("$testSuite-$outputFileName").also { it.deleteOnExit() } val errFile = File("$testSuite-$errorFileName").also { it.deleteOnExit() } @@ -12,7 +11,6 @@ fun String.runCommand(workingDir: File, testSuite: String): ProcessResult { .redirectOutput(outFile) .redirectError(errFile) .start().also { it.waitFor(processTimeout, TimeUnit.MINUTES) } -// File("$testSuite-compare").writeText(File(outFile.name).readText() + File(errFile.name).readText()) // to remove return ProcessResult(result.exitValue(), File(outFile.name).readText() + File(errFile.name).readText()) } From ac1d73641aeb5161200fc8ab5137953d2f7b3784 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 12 Nov 2020 08:08:45 +0100 Subject: [PATCH 09/32] Remove redundant changes --- .../src/test/resources/compare/RunTimeoutIT-compare | 4 ---- integration_tests/src/test/resources/flank_android.yml | 1 - integration_tests/src/test/resources/flank_ios.yml | 2 -- 3 files changed, 7 deletions(-) diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index e936bf3a43..7cd4a15b88 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -1,7 +1,3 @@ -version: local_snapshot -revision: 451aadbe4bfa0fa64f6cab20713bc228e897437e - -WARNING: disable-sharding enabled with max-test-shards = 1, Flank will ignore max-test-shard and disable sharding. AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* diff --git a/integration_tests/src/test/resources/flank_android.yml b/integration_tests/src/test/resources/flank_android.yml index 0960b66b57..592a046a13 100644 --- a/integration_tests/src/test/resources/flank_android.yml +++ b/integration_tests/src/test/resources/flank_android.yml @@ -3,5 +3,4 @@ gcloud: test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk flank: - disable-results-upload: true disable-sharding: true diff --git a/integration_tests/src/test/resources/flank_ios.yml b/integration_tests/src/test/resources/flank_ios.yml index 4701605de6..1f7350242b 100644 --- a/integration_tests/src/test/resources/flank_ios.yml +++ b/integration_tests/src/test/resources/flank_ios.yml @@ -1,5 +1,3 @@ gcloud: test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun -flank: - disable-results-upload: true From bb72e7429ca53583c2c65607fea483d2261f6eac Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 12 Nov 2020 08:58:26 +0100 Subject: [PATCH 10/32] Refactor multiple apks case --- integration_tests/build.gradle.kts | 2 +- .../kotlin/integration/{AndroidFullIT.kt => MultipleApksIT.kt} | 2 +- .../compare/{AndroidFullIT-compare => MultipleApksIT-compare} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename integration_tests/src/test/kotlin/integration/{AndroidFullIT.kt => MultipleApksIT.kt} (96%) rename integration_tests/src/test/resources/compare/{AndroidFullIT-compare => MultipleApksIT-compare} (73%) diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index b262dcf561..4f1db9a193 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -57,7 +57,7 @@ tasks.register("integrationTests") { group = "Verification" description = "Runs flank integration tests" filter { - includeTestsMatching("*IT") + includeTestsMatching("*MultipleApksIT") } testLogging { events("skipped", "failed") diff --git a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt similarity index 96% rename from integration_tests/src/test/kotlin/integration/AndroidFullIT.kt rename to integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index b0238f7865..1cbb26a342 100644 --- a/integration_tests/src/test/kotlin/integration/AndroidFullIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -5,7 +5,7 @@ import com.google.common.truth.Truth.assertThat import org.junit.Test import run -class AndroidFullIT { +class MultipleApksIT { private val name = this::class.java.simpleName @Test diff --git a/integration_tests/src/test/resources/compare/AndroidFullIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare similarity index 73% rename from integration_tests/src/test/resources/compare/AndroidFullIT-compare rename to integration_tests/src/test/resources/compare/MultipleApksIT-compare index d57ca52361..4f434d8a0f 100644 --- a/integration_tests/src/test/resources/compare/AndroidFullIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* 11 tests \+ 8 parameterized classes / 19 shards 4 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 3 / 4 \(\d{1,2}\.\d{1,2}\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* 11 tests \+ 8 parameterized classes / 19 shards 4 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 3 / 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file From a8eccebc9813eb9133d2c6b8ef77ce8d9f8ec6ba Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 12 Nov 2020 09:03:57 +0100 Subject: [PATCH 11/32] Fix filter --- integration_tests/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index 4f1db9a193..b262dcf561 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -57,7 +57,7 @@ tasks.register("integrationTests") { group = "Verification" description = "Runs flank integration tests" filter { - includeTestsMatching("*MultipleApksIT") + includeTestsMatching("*IT") } testLogging { events("skipped", "failed") From e3370a192f38b6c2983318b70b3c3f7c79e7a345 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 12:14:31 +0100 Subject: [PATCH 12/32] Cr fixes and turn off ios test on windows --- .../src/test/kotlin/integration/AllTestFilteredIT.kt | 7 +++++-- .../src/test/kotlin/integration/IgnoreFailedIT.kt | 3 +-- .../src/test/kotlin/integration/IntergrationTestsUtils.kt | 7 +++++++ .../src/test/kotlin/integration/MultipleDevicesIT.kt | 3 ++- .../src/test/kotlin/integration/RunTimeoutIT.kt | 3 +-- .../src/test/kotlin/integration/TestFilteringIT.kt | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt index 58c3e3871b..e71caa89b9 100644 --- a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -2,14 +2,16 @@ package integration import FlankCommand import com.google.common.truth.Truth.assertThat +import org.junit.Assume.assumeFalse import run import org.junit.Test class AllTestFilteredIT { + private val name = this::class.java.simpleName @Test fun `filter all tests - android`() { - val name = this::class.java.simpleName + "-android" + val name = "$name-android" val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/all_test_filtered_android.yml", @@ -28,7 +30,8 @@ class AllTestFilteredIT { @Test fun `filter all tests - ios`() { - val name = this::class.java.simpleName + "-ios" + assumeFalse(isWindows) + val name = "$name-ios" val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/all_test_filtered_ios.yml", diff --git a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt index 5db3b5e7d5..5591e3b68f 100644 --- a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt +++ b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt @@ -6,10 +6,9 @@ import org.junit.Test import run class IgnoreFailedIT { - + private val name = this::class.java.simpleName @Test fun `return with exit code 0 for failed tests`() { - val name = this::class.java.simpleName val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/flank_android_ignore_failed.yml", diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index a1b22f584b..5cff7dad39 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -66,3 +66,10 @@ private val fromCommon = fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent() fun findInCompare(name: String) = File("./src/test/resources/compare/$name-compare").readText().trimIndent() + + +private val osName = System.getProperty("os.name")?.toLowerCase() ?: "" + +val isWindows: Boolean by lazy { + osName.indexOf("win") >= 0 +} diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 80153cf5fc..6fe8854972 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -6,10 +6,11 @@ import run import org.junit.Test class MultipleDevicesIT { + private val name = this::class.java.simpleName @Test fun `run tests on multiple devices - android`() { - val name = this::class.java.simpleName + "-android" + val name = "$name-android" val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/flank_android_multiple_devices.yml", diff --git a/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt index 57d1e21d46..057283194e 100644 --- a/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt +++ b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt @@ -6,10 +6,9 @@ import org.junit.Test import run class RunTimeoutIT { - + private val name = this::class.java.simpleName @Test fun `cancel test run on timeout`() { - val name = this::class.java.simpleName val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/flank_android_run_timeout.yml", diff --git a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt index d7dd2951f5..a49021684e 100644 --- a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt +++ b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt @@ -6,10 +6,10 @@ import org.junit.Test import run class TestFilteringIT { + private val name = this::class.java.simpleName @Test fun `run test from only one apk`() { - val name = this::class.java.simpleName val result = FlankCommand( flankPath = "../test_runner/build/libs/flank.jar", ymlPath = "./src/test/resources/cases/test_filtering_android.yml", From b842121bec5cca6e572fad2d60aca3a158d6c83a Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 12:24:55 +0100 Subject: [PATCH 13/32] Fix android filtered integration test on windows --- .../test/resources/compare/AllTestFilteredIT-android-compare | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index f7341a466d..cda8efa7b5 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true From 31ef14099a7ebf5def90c1bdf7654b9917dd7407 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 13:10:10 +0100 Subject: [PATCH 14/32] Fix RunTimeout integration test on windows --- .../src/test/resources/compare/RunTimeoutIT-compare | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index 7cd4a15b88..8b3e2eabae 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true @@ -66,10 +66,10 @@ Saved 1 shards to android_shards.json 1 test / 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s - https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink - matrix-[0-9a-zA-Z\/_.-]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? + matrix-[0-9a-zA-Z\/_.-]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* From 526d5860904f706bef44892188d24e21ae7c16d5 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 15:47:37 +0100 Subject: [PATCH 15/32] Fix test filtering on windows --- .../integration/IntergrationTestsUtils.kt | 5 ++- .../resources/compare/TestFilteringIT-compare | 38 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 5cff7dad39..5aec3e9dbc 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -61,7 +61,10 @@ enum class TestOutcome(val regex: Regex) { } private val fromCommon = - { outcome: String -> "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } + { outcome: String -> + if (isWindows) "\\?\\ssuccess\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex() + else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() + } fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent() diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index f307e50450..85e3ff7fbc 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true @@ -35,12 +35,12 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 - +[\s\S]* flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 - smart-flank-gcs-path: + smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false @@ -54,45 +54,43 @@ AndroidArgs keep-file-path: false additional-app-test-apks: - app: null - test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk + test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: verbose disable-results-upload: false default-class-test-time: 240.0 - +[\s\S]* RunTests No tests for app-single-success-debug-androidTest.apk - +[\s\S]* Saved 1 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* Uploading app-multiple-success-debug-androidTest.apk \.* - 1 test / 1 shard - + 1 test \/ 1 shard +[\s\S]* 1 matrix ids created in \d{1,2}m \d{1,2}s - https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* - + https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* +[\s\S]* Matrices webLink - matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? - - [\s\S]* - + matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? +[\s\S]* FetchArtifacts \.* Updating matrix file - +[\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m - +[\s\S]* Uploading CostReport.txt \.* MatrixResultsReport - 1 / 1 \(100\.00\%\) + 1 \/ 1 \(100\.00\%\) [\s\S]* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* - +[\s\S]* Matrices webLink - matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? + matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? From 0555f8780e6ac442c0e3e7950c7810a5cb907873 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 16:18:30 +0100 Subject: [PATCH 16/32] Fix Sanity test on windows --- .../resources/compare/SanityRoboIT-compare | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 2772d0ae97..3b0ac21b3e 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -9,7 +9,7 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false @@ -61,34 +61,32 @@ AndroidArgs RunTests Uploading app-debug.apk \.* - 0 test / 0 shard + 0 test \/ 0 shard 1 matrix ids created in \d{1,2}m \d{1,2}s - https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* + https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink - matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* - - [\s\S]* - + matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* +[\s\S]* FetchArtifacts - +[\s\S]* Updating matrix file - +[\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m - +[\s\S]* Uploading CostReport.txt \.* MatrixResultsReport - 1 / 1 \(100\.00\%\) -┌─────────┬──────────────────────┬────────────────────────────┬──────────────┐ -│ OUTCOME │ MATRIX ID │ TEST AXIS VALUE │ TEST DETAILS │ -├─────────┼──────────────────────┼────────────────────────────┼──────────────┤ -│ success │ matrix-[a-zA-Z0-9\s]*│ NexusLowRes-28-en-portrait │ --- │ -└─────────┴──────────────────────┴────────────────────────────┴──────────────┘ + 1 \/ 1 \(100\.00\%\) +.* +[│\?] OUTCOME [│\?] MATRIX ID [│\?] TEST AXIS VALUE [│\?] TEST DETAILS [│\?] +.* +[│\?] success [│\?] matrix-[a-zA-Z0-9\s]*[│\?] NexusLowRes-28-en-portrait [│\?] --- [│\?] +.* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* - +[\s\S]* Matrices webLink - matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*/executions/[.a-zA-Z0-9_-]* + matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* From 4e93e1c608430d76d52ad0bbbe48c7531d767b63 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 17:00:26 +0100 Subject: [PATCH 17/32] Fix IgnoreFailed tests on windows --- .../src/test/resources/compare/IgnoreFailedIT-compare | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index e6a147d69a..25e355e8ce 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test / 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 / 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file From 5447245eb71aa82a436df6ff74ccd39670bbc84b Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 17:06:04 +0100 Subject: [PATCH 18/32] Update IntergrationTestsUtils.kt --- .../src/test/kotlin/integration/IntergrationTestsUtils.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 5aec3e9dbc..6cde53b0f5 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -62,7 +62,7 @@ enum class TestOutcome(val regex: Regex) { private val fromCommon = { outcome: String -> - if (isWindows) "\\?\\ssuccess\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex() + if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex() else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } From 6e408f5da5acd1da12d766feb46c5ecd2846068c Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 17:39:54 +0100 Subject: [PATCH 19/32] Fix multiple apks it on windows --- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 4f434d8a0f..24e5214b86 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* 11 tests \+ 8 parameterized classes / 19 shards 4 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9_-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 3 / 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 [\s\S]* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 [\s\S]* RunTests [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s [\s\S]* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards [\s\S]* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* [\s\S]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* [\s\S]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file From d6ebc8ab3bbc552088bb13d3cee1bca4147c50f3 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 13 Nov 2020 18:29:09 +0100 Subject: [PATCH 20/32] Fix multiple apks tests on windows --- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 24e5214b86..fab0b0d3c2 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 [\s\S]* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 [\s\S]* RunTests [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s [\s\S]* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s [\s\S]* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards [\s\S]* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* [\s\S]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* [\s\S]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file \s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file From 687b80b14472bee085e12165b2fc52562bfaf3d3 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 16 Nov 2020 08:45:45 +0100 Subject: [PATCH 21/32] Update TestFilteringIT-compare --- .../resources/compare/TestFilteringIT-compare | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index 85e3ff7fbc..6134fe9914 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -35,7 +35,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 -[\s\S]* +\s* flank: max-test-shards: 1 shard-time: -1 @@ -61,36 +61,36 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 -[\s\S]* +\s* RunTests No tests for app-single-success-debug-androidTest.apk -[\s\S]* +\s* Saved 1 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* Uploading app-multiple-success-debug-androidTest.apk \.* 1 test \/ 1 shard -[\s\S]* +\s* 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* -[\s\S]* +\s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file -[\s\S]* +\s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m -[\s\S]* +\s* Uploading CostReport.txt \.* MatrixResultsReport 1 \/ 1 \(100\.00\%\) [\s\S]* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* -[\s\S]* +\s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? From 869873b0d4f86c04ab5df567b2ebe0564bf09b93 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 16 Nov 2020 09:07:29 +0100 Subject: [PATCH 22/32] Update SanityRoboIT-compare --- .../src/test/resources/compare/SanityRoboIT-compare | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 3b0ac21b3e..9b9afe3594 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -70,13 +70,13 @@ Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* [\s\S]* FetchArtifacts -[\s\S]* +\s* Updating matrix file -[\s\S]* +\s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m -[\s\S]* +\s* Uploading CostReport.txt \.* MatrixResultsReport 1 \/ 1 \(100\.00\%\) @@ -87,6 +87,6 @@ MatrixResultsReport .* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* -[\s\S]* +\s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* From f7f2d48d535029d1aea2305bce530c5295f27d0f Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Tue, 17 Nov 2020 06:18:58 +0100 Subject: [PATCH 23/32] Review changes --- .../src/test/kotlin/integration/AllTestFilteredIT.kt | 8 ++++---- .../src/test/kotlin/integration/IgnoreFailedIT.kt | 4 ++-- .../src/test/kotlin/integration/IntergrationTestsUtils.kt | 4 +++- .../src/test/kotlin/integration/MultipleApksIT.kt | 5 ++--- .../src/test/kotlin/integration/MultipleDevicesIT.kt | 4 ++-- .../src/test/kotlin/integration/RunTimeoutIT.kt | 4 ++-- .../src/test/kotlin/integration/SanityRoboIT.kt | 4 ++-- .../src/test/kotlin/integration/TestFilteringIT.kt | 4 ++-- 8 files changed, 19 insertions(+), 18 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt index e71caa89b9..ee95a5ad96 100644 --- a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -13,8 +13,8 @@ class AllTestFilteredIT { fun `filter all tests - android`() { val name = "$name-android" val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/all_test_filtered_android.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/all_test_filtered_android.yml", params = androidRunCommands ).run( workingDirectory = "./", @@ -33,8 +33,8 @@ class AllTestFilteredIT { assumeFalse(isWindows) val name = "$name-ios" val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/all_test_filtered_ios.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/all_test_filtered_ios.yml", params = iosRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt index 5591e3b68f..8d6ab1b1f5 100644 --- a/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt +++ b/integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt @@ -10,8 +10,8 @@ class IgnoreFailedIT { @Test fun `return with exit code 0 for failed tests`() { val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/flank_android_ignore_failed.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/flank_android_ignore_failed.yml", params = androidRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 6cde53b0f5..8635339668 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -4,6 +4,9 @@ import org.junit.Assert.assertEquals import utils.ProcessResult import java.io.File +const val FLANK_JAR_PATH = "../test_runner/build/libs/flank.jar" +const val CONFIGS_PATH = "./src/test/resources/cases" + val androidRunCommands = listOf("firebase", "test", "android", "run") val iosRunCommands = listOf("firebase", "test", "ios", "run") @@ -70,7 +73,6 @@ fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimInd fun findInCompare(name: String) = File("./src/test/resources/compare/$name-compare").readText().trimIndent() - private val osName = System.getProperty("os.name")?.toLowerCase() ?: "" val isWindows: Boolean by lazy { diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index 1cbb26a342..02ec40cae8 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -11,12 +11,11 @@ class MultipleApksIT { @Test fun `flank full option run`() { val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/flank_android_multiple_apk.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/flank_android_multiple_apk.yml", params = androidRunCommands ).run("./", name) - assertExitCode(result, 10) val resOutput = result.output.removeUnicode() diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 6fe8854972..64ead4b72f 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -12,8 +12,8 @@ class MultipleDevicesIT { fun `run tests on multiple devices - android`() { val name = "$name-android" val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/flank_android_multiple_devices.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/flank_android_multiple_devices.yml", params = androidRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt index 057283194e..010d275fd3 100644 --- a/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt +++ b/integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt @@ -10,8 +10,8 @@ class RunTimeoutIT { @Test fun `cancel test run on timeout`() { val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/flank_android_run_timeout.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/flank_android_run_timeout.yml", params = androidRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt index 4714634c81..0e7e811889 100644 --- a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -11,8 +11,8 @@ class SanityRoboIT { @Test fun `sanity robo`() { val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/sanity_robo.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/sanity_robo.yml", params = androidRunCommands ).run( workingDirectory = "./", diff --git a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt index a49021684e..e81e259cc5 100644 --- a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt +++ b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt @@ -11,8 +11,8 @@ class TestFilteringIT { @Test fun `run test from only one apk`() { val result = FlankCommand( - flankPath = "../test_runner/build/libs/flank.jar", - ymlPath = "./src/test/resources/cases/test_filtering_android.yml", + flankPath = FLANK_JAR_PATH, + ymlPath = "$CONFIGS_PATH/test_filtering_android.yml", params = androidRunCommands ).run( workingDirectory = "./", From 768d27ef5f7f9047a6a120ac1676a0f480a6c6a5 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 19 Nov 2020 08:25:03 +0100 Subject: [PATCH 24/32] Update compare files --- .../resources/compare/AllTestFilteredIT-android-compare | 1 + .../src/test/resources/compare/IgnoreFailedIT-compare | 2 +- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- .../resources/compare/MultipleDevicesIT-android-compare | 2 +- .../src/test/resources/compare/RunTimeoutIT-compare | 1 + .../src/test/resources/compare/SanityRoboIT-compare | 8 ++++---- .../src/test/resources/compare/TestFilteringIT-compare | 8 ++++---- test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 4 ++-- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 9 ++++----- 9 files changed, 19 insertions(+), 18 deletions(-) diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index cda8efa7b5..febcc48a4e 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -35,6 +35,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 + test-targets-for-shard: flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index 25e355e8ce..624e070820 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index fab0b0d3c2..0f2c867301 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file \s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index 7c0894f491..a351d0e843 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* 11 tests \+ 8 parameterized classes / 11 shards 3 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* FetchArtifacts \.* Updating matrix file CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 2 / 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]*/matrix_\d/\s*){17} Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* 11 tests \+ 8 parameterized classes / 11 shards 3 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 2 / 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]*/matrix_\d/\s*){17} FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index 8b3e2eabae..05f087eed2 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -34,6 +34,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 + test-targets-for-shard: flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 9b9afe3594..9c46be481e 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -34,6 +34,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 + test-targets-for-shard: flank: max-test-shards: 1 @@ -69,10 +70,6 @@ RunTests Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* [\s\S]* -FetchArtifacts -\s* - Updating matrix file -\s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m @@ -87,6 +84,9 @@ MatrixResultsReport .* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* +FetchArtifacts + \.* + Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*\/executions\/[.a-zA-Z0-9_-]* diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index 6134fe9914..011195ffa6 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -35,6 +35,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 0 + test-targets-for-shard: \s* flank: max-test-shards: 1 @@ -77,10 +78,6 @@ Saved 1 shards to android_shards.json Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* -FetchArtifacts - \.* - Updating matrix file -\s* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m @@ -91,6 +88,9 @@ MatrixResultsReport [\s\S]* Uploading MatrixResultsReport.txt \.* Uploading JUnitReport.xml \.* +FetchArtifacts + \.* + Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 29c6531d78..165c4c257c 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -60,9 +60,9 @@ AndroidArgs test-targets:${ArgsToString.listToString(testTargets)} robo-directives:${ArgsToString.objectsToString(roboDirectives)} robo-script: $roboScript - device: ${ArgsToString.objectsToString(devices)} + device:${ArgsToString.objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts - test-targets-for-shard: ${ArgsToString.listOfListToString(testTargetsForShard)} + test-targets-for-shard:${ArgsToString.listOfListToString(testTargetsForShard)} fail-fast: $failFast flank: diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 676b34922a..71da120585 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -335,7 +335,7 @@ AndroidArgs - class com.example.app.ExampleUiTest#testFails robo-directives: robo-script: null - device: + device: - model: NexusLowRes version: 23 locale: en @@ -345,7 +345,7 @@ AndroidArgs locale: en orientation: portrait num-flaky-test-attempts: 3 - test-targets-for-shard: + test-targets-for-shard: fail-fast: false flank: @@ -416,13 +416,13 @@ AndroidArgs test-targets: robo-directives: robo-script: null - device: + device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 - test-targets-for-shard: + test-targets-for-shard: fail-fast: false flank: @@ -1229,7 +1229,6 @@ AndroidArgs AndroidArgs.load(yaml).validate() } - @Test fun `cli fail-fast`() { val cli = AndroidRunCommand() From fb2c0902298cc43116396f5162fcdbe8fe8b8c24 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 20 Nov 2020 14:15:32 +0100 Subject: [PATCH 25/32] Fix integration tests In case when we have - in user name --- .../resources/compare/AllTestFilteredIT-android-compare | 4 ++-- .../src/test/resources/compare/IgnoreFailedIT-compare | 2 +- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- .../src/test/resources/compare/RunTimeoutIT-compare | 4 ++-- .../src/test/resources/compare/SanityRoboIT-compare | 2 +- .../src/test/resources/compare/TestFilteringIT-compare | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index febcc48a4e..14b2ca33b7 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk - test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index 624e070820..63da702cb5 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 0f2c867301..203bc55263 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index 05f087eed2..dd3dcdf2ed 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk - test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 9c46be481e..dab42f37cf 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -9,7 +9,7 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index 011195ffa6..cdf13afd7b 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk - test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true @@ -55,7 +55,7 @@ AndroidArgs keep-file-path: false additional-app-test-apks: - app: null - test: [0-9a-zA-Z\\\/_.-:]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk + test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false From ee8190f04215e6e98f4f6ca99247fb2eb38cfb01 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 23 Nov 2020 17:22:53 +0100 Subject: [PATCH 26/32] Fix multiple devices and multiple apk tests --- .../src/test/kotlin/integration/MultipleApksIT.kt | 8 ++++++++ .../src/test/resources/compare/MultipleApksIT-compare | 2 +- .../resources/compare/MultipleDevicesIT-android-compare | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index 02ec40cae8..2792723c8d 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -20,6 +20,14 @@ class MultipleApksIT { val resOutput = result.output.removeUnicode() assertThat(resOutput).containsMatch(findInCompare(name)) + listOf( + "Uploading app-multiple-success-debug-androidTest.apk", + "Uploading app-multiple-error-debug-androidTest.apk", + "Uploading MainActivity_robo_script.json" + ).forEach { + assertThat(resOutput).contains(it) + } + assertContainsOutcomeSummary(resOutput) { success = 3 failure = 1 diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 203bc55263..720c5913c1 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*)(?=.*Uploading MainActivity_robo_script\.json \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index a351d0e843..317d80fbf8 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-error-debug-androidTest.apk - app: null test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 9\) Shard times: 240s, 240s, 360s, 360s, 360s Smart Flank cache hit: 0\% \(0 / 1\) Shard times: 120s Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* 11 tests \+ 8 parameterized classes / 11 shards 3 matrix ids created in \d{1,2}m \d{1,2}s https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m Uploading CostReport.txt \.* MatrixResultsReport 2 / 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https://console.developers.google.com/storage/browser/test-lab-[a-zA-Z0-9-]*/[.a-zA-Z0-9_-]*/matrix_\d/\s*){17} FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https://console.firebase.google.com/project/flank-open-source/testlab/histories/[.a-zA-Z0-9_-]*/matrices/[.a-zA-Z0-9_-]*(/executions/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading CostReport.txt \.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\/matrix_\d\/\s*){17} FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file From 7d22941a68f8265279729014ef73b57e272d40ab Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 24 Nov 2020 14:23:27 +0100 Subject: [PATCH 27/32] Fix integration tests on windows --- .../src/test/kotlin/integration/MultipleDevicesIT.kt | 9 +++++++++ .../resources/compare/MultipleDevicesIT-android-compare | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 64ead4b72f..dcb76bcba9 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -4,6 +4,7 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import run import org.junit.Test +import java.io.File class MultipleDevicesIT { private val name = this::class.java.simpleName @@ -23,7 +24,15 @@ class MultipleDevicesIT { assertExitCode(result, 10) val resOutput = result.output.removeUnicode() + File("test.log").writeText(resOutput) assertThat(resOutput).containsMatch(findInCompare(name)) + listOf( + "Uploading app-multiple-success-debug-androidTest.apk", + "Uploading app-multiple-error-debug-androidTest.apk", + "Uploading performanceMetrics.json" + ).forEach { + assertThat(resOutput).contains(it) + } assertContainsOutcomeSummary(resOutput) { success = 6 failure = 3 diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index 317d80fbf8..92cc423515 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/apk\/app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* (?=.*Uploading app-multiple-success-debug-androidTest\.apk \.*)(?=.*Uploading app-multiple-error-debug-androidTest\.apk \.*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading CostReport.txt \.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading performanceMetrics.json \.* (Performance metrics uploaded to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\/matrix_\d\/\s*){17} FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading CostReport.txt \.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* [\s\S]* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file From 1b5df511a946dd5910ee2d04703202bb6c872233 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 24 Nov 2020 15:04:46 +0100 Subject: [PATCH 28/32] Update MultipleDevicesIT.kt --- .../src/test/kotlin/integration/MultipleDevicesIT.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index dcb76bcba9..bc0e3ba566 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -4,7 +4,6 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import run import org.junit.Test -import java.io.File class MultipleDevicesIT { private val name = this::class.java.simpleName @@ -24,7 +23,6 @@ class MultipleDevicesIT { assertExitCode(result, 10) val resOutput = result.output.removeUnicode() - File("test.log").writeText(resOutput) assertThat(resOutput).containsMatch(findInCompare(name)) listOf( "Uploading app-multiple-success-debug-androidTest.apk", From 93fbabcc3a9ad2a7aa91e5d42054a918395532ac Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 25 Nov 2020 12:27:58 +0100 Subject: [PATCH 29/32] Cr fixes --- .../src/test/kotlin/integration/IntergrationTestsUtils.kt | 5 +++++ .../src/test/kotlin/integration/MultipleApksIT.kt | 6 ++---- .../src/test/kotlin/integration/MultipleDevicesIT.kt | 6 ++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index 8635339668..b14f9d1127 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -1,5 +1,6 @@ package integration +import com.google.common.truth.Truth import org.junit.Assert.assertEquals import utils.ProcessResult import java.io.File @@ -40,6 +41,10 @@ data class OutcomeSummary(val matcher: MutableMap = mutableMap } } +fun assertContainsUploads(input: String, vararg uploads: String) = uploads.forEach { + Truth.assertThat(input).contains(it) +} + fun assertContainsOutcomeSummary(input: String, block: OutcomeSummary.() -> Unit) = OutcomeSummary().apply(block).matcher.entries.forEach { (outcome, times) -> val actual = outcome.regex.findAll(input).toList().size diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index 2792723c8d..9dd7176bff 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -20,13 +20,11 @@ class MultipleApksIT { val resOutput = result.output.removeUnicode() assertThat(resOutput).containsMatch(findInCompare(name)) - listOf( + assertContainsUploads(resOutput, "Uploading app-multiple-success-debug-androidTest.apk", "Uploading app-multiple-error-debug-androidTest.apk", "Uploading MainActivity_robo_script.json" - ).forEach { - assertThat(resOutput).contains(it) - } + ) assertContainsOutcomeSummary(resOutput) { success = 3 diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index bc0e3ba566..62b1b32248 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -24,13 +24,11 @@ class MultipleDevicesIT { val resOutput = result.output.removeUnicode() assertThat(resOutput).containsMatch(findInCompare(name)) - listOf( + assertContainsUploads(resOutput, "Uploading app-multiple-success-debug-androidTest.apk", "Uploading app-multiple-error-debug-androidTest.apk", "Uploading performanceMetrics.json" - ).forEach { - assertThat(resOutput).contains(it) - } + ) assertContainsOutcomeSummary(resOutput) { success = 6 failure = 3 From 075ff25c767f3d1e402cef17407eb2aceab72b33 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 25 Nov 2020 14:29:31 +0100 Subject: [PATCH 30/32] Cr changes --- .../src/test/kotlin/integration/IntergrationTestsUtils.kt | 4 ++-- .../src/test/kotlin/integration/MultipleApksIT.kt | 6 +++--- .../src/test/kotlin/integration/MultipleDevicesIT.kt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index b14f9d1127..ac97af7f35 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -1,6 +1,6 @@ package integration -import com.google.common.truth.Truth +import com.google.common.truth.Truth.assertThat import org.junit.Assert.assertEquals import utils.ProcessResult import java.io.File @@ -42,7 +42,7 @@ data class OutcomeSummary(val matcher: MutableMap = mutableMap } fun assertContainsUploads(input: String, vararg uploads: String) = uploads.forEach { - Truth.assertThat(input).contains(it) + assertThat(input).contains("Uploading $it") } fun assertContainsOutcomeSummary(input: String, block: OutcomeSummary.() -> Unit) = diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index 9dd7176bff..fec202a434 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -21,9 +21,9 @@ class MultipleApksIT { val resOutput = result.output.removeUnicode() assertThat(resOutput).containsMatch(findInCompare(name)) assertContainsUploads(resOutput, - "Uploading app-multiple-success-debug-androidTest.apk", - "Uploading app-multiple-error-debug-androidTest.apk", - "Uploading MainActivity_robo_script.json" + "app-multiple-success-debug-androidTest.apk", + "app-multiple-error-debug-androidTest.apk", + "MainActivity_robo_script.json" ) assertContainsOutcomeSummary(resOutput) { diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 62b1b32248..6be6bf2921 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -25,9 +25,9 @@ class MultipleDevicesIT { val resOutput = result.output.removeUnicode() assertThat(resOutput).containsMatch(findInCompare(name)) assertContainsUploads(resOutput, - "Uploading app-multiple-success-debug-androidTest.apk", - "Uploading app-multiple-error-debug-androidTest.apk", - "Uploading performanceMetrics.json" + "app-multiple-success-debug-androidTest.apk", + "app-multiple-error-debug-androidTest.apk", + "performanceMetrics.json" ) assertContainsOutcomeSummary(resOutput) { success = 6 From 99df937f3c3d1a0afb2ac9e70f76e0150c271548 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Tue, 1 Dec 2020 16:16:15 +0100 Subject: [PATCH 31/32] Update compare and yml files --- .../resources/cases/all_test_filtered_ios.yml | 4 ++-- .../compare/AllTestFilteredIT-ios-compare | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml b/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml index f5046f913a..fa94f64036 100644 --- a/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml +++ b/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml @@ -1,6 +1,6 @@ gcloud: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip - xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip + xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun flank: test-targets: - nonExisting/Class diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare index bea3fcb8c2..2d7d80ee8d 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare @@ -9,12 +9,12 @@ IosArgs network-profile: null results-history-name: null # iOS gcloud - test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip - xctestrun-file: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + test: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip + xctestrun-file: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun xcode-version: null device: - model: iphone8 - version: 12.0 + version: 13.6 locale: en orientation: portrait num-flaky-test-attempts: 0 @@ -51,10 +51,10 @@ IosArgs default-class-test-time: 240.0 RunTests -Found xctest: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest +Found xctest: [0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest isMacOS = true \(mac os x\) -nm -U "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" -nm -gU "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" | xargs -s 262144 xcrun swift-demangle +nm -U "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" +nm -gU "[0-9a-zA-Z\/_.-]*/test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/Debug-iphoneos/EarlGreyExampleSwift.app/PlugIns/EarlGreyExampleSwiftTests.xctest/EarlGreyExampleSwiftTests" | xargs -s 262144 xcrun swift-demangle Saved 0 shards to ios_shards.json Uploading ios_shards.json \.* Uploading earlgrey_example.zip \*. @@ -66,8 +66,6 @@ Saved 0 shards to ios_shards.json Matrices webLink -FetchArtifacts - CostReport No cost. 0m @@ -75,5 +73,7 @@ CostReport MatrixResultsReport 0 / 0 \(NaN\%\) Uploading MatrixResultsReport.txt \.* +FetchArtifacts + Matrices webLink From 8a302c8262160f8bf472512d5957b413801dad69 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 3 Dec 2020 16:33:42 +0100 Subject: [PATCH 32/32] Update compare files --- integration_tests/build.gradle.kts | 2 +- .../test/resources/compare/AllTestFilteredIT-android-compare | 1 + .../src/test/resources/compare/AllTestFilteredIT-ios-compare | 1 + .../src/test/resources/compare/IgnoreFailedIT-compare | 2 +- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- .../test/resources/compare/MultipleDevicesIT-android-compare | 2 +- .../src/test/resources/compare/RunTimeoutIT-compare | 1 + .../src/test/resources/compare/SanityRoboIT-compare | 1 + .../src/test/resources/compare/TestFilteringIT-compare | 1 + 9 files changed, 9 insertions(+), 4 deletions(-) diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index b262dcf561..96e0b548ac 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -63,7 +63,7 @@ tasks.register("integrationTests") { events("skipped", "failed") exceptionFormat = TestExceptionFormat.FULL } - maxParallelForks = 4 + maxParallelForks = Runtime.getRuntime().availableProcessors() / 2 } val compileTestKotlin: KotlinCompile by tasks diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index 14b2ca33b7..93070c031d 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -36,6 +36,7 @@ AndroidArgs orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: + fail-fast: false flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare index 2d7d80ee8d..fef70fb828 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare @@ -25,6 +25,7 @@ IosArgs type: xctest app: test-special-entitlements: false + fail-fast: false flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index 63da702cb5..20ed80a9ec 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading app-debug.apk \.* Uploading app-single-error-debug-androidTest.apk \.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 720c5913c1..b8b380ee96 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading CostReport.txt \.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index 92cc423515..9462cac430 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading CostReport.txt \.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* [\s\S]* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading android_shards.json \.* Uploading app-debug.apk \.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]* \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading CostReport.txt \.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* [\s\S]* FetchArtifacts \.* Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index dd3dcdf2ed..68308a85c8 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -35,6 +35,7 @@ AndroidArgs orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: + fail-fast: false flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index dab42f37cf..5205e69aae 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -35,6 +35,7 @@ AndroidArgs orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: + fail-fast: false flank: max-test-shards: 1 diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index cdf13afd7b..c1d756b138 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -36,6 +36,7 @@ AndroidArgs orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: + fail-fast: false \s* flank: max-test-shards: 1