From a5dbd8f5072b7b560a8a15b02560bf6392726504 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 5 Oct 2020 17:09:49 +0200 Subject: [PATCH 01/13] Add dump shards on run ios/android tests --- .../test/android/AndroidRunCommand.kt | 10 ++++++++-- .../src/main/kotlin/ftl/run/DumpShards.kt | 2 +- .../ftl/run/platform/RunAndroidTests.kt | 20 +++++++++++++++++-- .../kotlin/ftl/run/platform/RunIosTests.kt | 3 ++- .../android/GetAndroidMatrixShards.kt | 6 +++--- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index a17e15e2d8..c7434386dc 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -1,6 +1,7 @@ package ftl.cli.firebase.test.android import ftl.args.AndroidArgs +import ftl.args.isInstrumentationTest import ftl.args.validate import ftl.cli.firebase.test.CommonRunCommand import ftl.config.FtlConstants @@ -46,8 +47,13 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() runBlocking { - if (dumpShards) dumpShards(args = config, obfuscatedOutput = obfuscate) - else newTestRun(config) + when { + dumpShards -> dumpShards(args = config, obfuscatedOutput = obfuscate) + config.isInstrumentationTest -> { + newTestRun(config) + } + else -> newTestRun(config) + } } } diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 5965993d7f..998117b667 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -42,7 +42,7 @@ fun dumpShards( ) } -private fun saveShardChunks( +fun saveShardChunks( shardFilePath: String, shards: Any, size: Int, diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 58021b637a..a628b443da 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -7,6 +7,7 @@ import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix import ftl.gc.GcToolResults import ftl.http.executeWithRetry +import ftl.run.ANDROID_SHARD_FILE import ftl.run.model.InstrumentationTestContext import ftl.run.model.TestResult import ftl.run.platform.android.createAndroidTestConfig @@ -18,6 +19,10 @@ import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage import ftl.run.platform.common.beforeRunTests import ftl.run.exception.FlankGeneralError +import ftl.run.model.AndroidMatrixTestShards +import ftl.run.model.AndroidTestContext +import ftl.run.platform.android.asMatrixTestShards +import ftl.run.saveShardChunks import ftl.shard.Chunk import ftl.shard.testCases import kotlinx.coroutines.Deferred @@ -40,8 +45,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val otherGcsFiles = args.uploadOtherFiles(runGcsPath) val additionalApks = args.uploadAdditionalApks(runGcsPath) - args.createAndroidTestContexts() - .upload(args.resultsBucket, runGcsPath) + args.createAndroidTestContexts().dumpShards().upload(args.resultsBucket, runGcsPath) .forEachIndexed { index, context -> if (context is InstrumentationTestContext) { ignoredTestsShardChunks += context.ignoredTestCases @@ -64,6 +68,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS if (testMatrices.isEmpty()) throw FlankGeneralError("There are no tests to run.") println(beforeRunMessage(args, allTestShardChunks)) + TestResult( matrixMap = afterRunTests(testMatrices.awaitAll(), runGcsPath, stopwatch, args), shardChunks = allTestShardChunks.testCases, @@ -71,6 +76,17 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS ) } +private fun List.dumpShards() = apply { + filterIsInstance().asMatrixTestShards().saveShards() +} + +private fun AndroidMatrixTestShards.saveShards() = saveShardChunks( + shardFilePath = ANDROID_SHARD_FILE, + shards = this, + size = size, + obfuscatedOutput = false +) + private suspend fun executeAndroidTestMatrix( runCount: Int, createTestMatrix: () -> Testing.Projects.TestMatrices.Create diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index 9514d880c8..d16ee44d91 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -9,6 +9,7 @@ import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry import ftl.ios.Xctestrun +import ftl.run.dumpShards import ftl.run.model.TestResult import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage @@ -40,7 +41,7 @@ internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope val xcTestGcsPath = getXcTestGcPath(iosArgs, runGcsPath) println(beforeRunMessage(iosArgs, iosArgs.testShardChunks)) - + dumpShards(iosArgs) repeat(runCount) { jobs += iosArgs.testShardChunks.map { testTargets -> async(Dispatchers.IO) { diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt index ea4c3b03b0..c3345d32cc 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt @@ -6,12 +6,12 @@ import ftl.run.model.AndroidTestShards import ftl.run.model.InstrumentationTestContext import ftl.shard.testCases -suspend fun AndroidArgs.getAndroidMatrixShards(): AndroidMatrixTestShards = this - .createAndroidTestContexts() +suspend fun AndroidArgs.getAndroidMatrixShards(): AndroidMatrixTestShards = + createAndroidTestContexts() .filterIsInstance() .asMatrixTestShards() -private fun List.asMatrixTestShards(): AndroidMatrixTestShards = +fun List.asMatrixTestShards(): AndroidMatrixTestShards = map { testApks -> AndroidTestShards( app = testApks.app.local, From aacae4e3db69920ea808611056171b5b35a664a1 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 6 Oct 2020 13:11:24 +0200 Subject: [PATCH 02/13] Added tests and upload on gcs --- android_shards.json | 31 +++++++++++++++++++ ios_shards.json | 21 +++++++++++++ .../ftl/run/platform/RunAndroidTests.kt | 6 ++-- .../kotlin/ftl/run/platform/RunIosTests.kt | 6 +++- test_runner/src/test/kotlin/Debug.kt | 8 ++--- .../test/android/AndroidRunCommandTest.kt | 29 +++++++++++++++++ .../kotlin/ftl/fixtures/simple-ios-flank.yml | 4 +-- .../test_app_cases/flank-multiple-flaky.yml | 4 +-- 8 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 android_shards.json create mode 100644 ios_shards.json diff --git a/android_shards.json b/android_shards.json new file mode 100644 index 0000000000..7699594a26 --- /dev/null +++ b/android_shards.json @@ -0,0 +1,31 @@ +{ + "matrix-0": { + "app": "/Users/adamfilipowicz/Repos/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test": "/Users/adamfilipowicz/Repos/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", + "shards": { + "shard-0": [ + "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", + "class com.example.test_app.InstrumentedTest#test2" + ], + "shard-1": [ + "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized", + "class com.example.test_app.InstrumentedTest#test1" + ], + "shard-2": [ + "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner", + "class com.example.test_app.InstrumentedTest#test0" + ], + "shard-3": [ + "class com.example.test_app.ParameterizedTest", + "class com.example.test_app.bar.BarInstrumentedTest#testBar", + "class com.example.test_app.foo.FooInstrumentedTest#testFoo" + ] + }, + "junit-ignored": [ + "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", + "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", + "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", + "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" + ] + } +} \ No newline at end of file diff --git a/ios_shards.json b/ios_shards.json new file mode 100644 index 0000000000..ea664f072f --- /dev/null +++ b/ios_shards.json @@ -0,0 +1,21 @@ +[ + [ + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testLayout", + "EarlGreyExampleSwiftTests/testThatThrows", + "EarlGreyExampleSwiftTests/testWithInRoot", + "EarlGreyExampleSwiftTests/testCustomAction", + "EarlGreyExampleSwiftTests/testWithCondition", + "EarlGreyExampleSwiftTests/testBasicSelection", + "EarlGreyExampleSwiftTests/testWithCustomMatcher", + "EarlGreyExampleSwiftTests/testCollectionMatchers", + "EarlGreyExampleSwiftTests/testCatchErrorOnFailure", + "EarlGreyExampleSwiftTests/testWithCustomAssertion", + "EarlGreyExampleSwiftTests/testTableCellOutOfScreen", + "EarlGreyExampleSwiftTests/testBasicSelectionAndAction", + "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert", + "EarlGreyExampleSwiftTests/testWithCustomFailureHandler", + "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", + "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements" + ] +] \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index a628b443da..113181b206 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -5,6 +5,7 @@ import com.google.api.services.testing.model.TestMatrix import ftl.args.AndroidArgs import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix +import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry import ftl.run.ANDROID_SHARD_FILE @@ -45,7 +46,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val otherGcsFiles = args.uploadOtherFiles(runGcsPath) val additionalApks = args.uploadAdditionalApks(runGcsPath) - args.createAndroidTestContexts().dumpShards().upload(args.resultsBucket, runGcsPath) + args.createAndroidTestContexts().dumpShards(args).upload(args.resultsBucket, runGcsPath) .forEachIndexed { index, context -> if (context is InstrumentationTestContext) { ignoredTestsShardChunks += context.ignoredTestCases @@ -76,8 +77,9 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS ) } -private fun List.dumpShards() = apply { +private fun List.dumpShards(config: AndroidArgs) = apply { filterIsInstance().asMatrixTestShards().saveShards() + if (config.disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, config.resultsBucket, config.resultsDir) } private fun AndroidMatrixTestShards.saveShards() = saveShardChunks( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index d16ee44d91..b4bb77eefa 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -9,6 +9,7 @@ import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry import ftl.ios.Xctestrun +import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.model.TestResult import ftl.run.platform.common.afterRunTests @@ -37,11 +38,14 @@ internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope val shardCounter = ShardCounter() val history = GcToolResults.createToolResultsHistory(iosArgs) + dumpShards(iosArgs) + if (iosArgs.disableResultsUpload.not()) GcStorage.upload(IOS_SHARD_FILE, iosArgs.resultsBucket, iosArgs.resultsDir) + // Upload only after parsing shards to detect missing methods early. val xcTestGcsPath = getXcTestGcPath(iosArgs, runGcsPath) println(beforeRunMessage(iosArgs, iosArgs.testShardChunks)) - dumpShards(iosArgs) + repeat(runCount) { jobs += iosArgs.testShardChunks.map { testTargets -> async(Dispatchers.IO) { diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index 608364e6bf..e9c59f287d 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -13,8 +13,8 @@ fun main() { val projectId = System.getenv("GOOGLE_CLOUD_PROJECT") ?: "YOUR PROJECT ID" - val quantity = "multiple" - val type = "flaky" +// val quantity = "multiple" +// val type = "flaky" // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 withGlobalExceptionHandling { @@ -22,14 +22,14 @@ fun main() { // "--debug", "firebase", "test", - "android", + "ios", "run", // "--dry", // "--dump-shards", "--output-style=single", // "--full-junit-result", // "--legacy-junit-result", - "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", + "-c=test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml", "--project=$projectId" // "--client-details=key1=value1,key2=value2" ) diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index a716577726..e5fd55dbb7 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -4,8 +4,15 @@ import com.google.common.truth.Truth.assertThat import ftl.args.yml.AppTestPair import ftl.config.Device import ftl.config.FtlConstants +import ftl.gc.GcStorage +import ftl.run.ANDROID_SHARD_FILE +import ftl.run.dumpShards import ftl.run.exception.FlankConfigurationError +import ftl.run.saveShardChunks import ftl.test.util.FlankTestRunner +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.verify import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test @@ -523,4 +530,26 @@ class AndroidRunCommandTest { ) cmd.run() } + + @Test + fun `should dump shards on android test run`() { + mockkStatic("ftl.run.DumpShardsKt") + val runCmd = AndroidRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml" + runCmd.run() + verify { saveShardChunks(any(), any(), any(), any()) } + } + + @Test + fun `should dump shards on android test run and not upload when disable-upload-results set`() { + mockkStatic("ftl.run.DumpShardsKt") + mockkObject(GcStorage) { + val runCmd = AndroidRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml" + CommandLine(runCmd).parseArgs("--disable-results-upload") + runCmd.run() + verify { saveShardChunks(any(), any(), any(), any()) } + verify(inverse = true) { GcStorage.upload(ANDROID_SHARD_FILE, any(), any()) } + } + } } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml index 50b3a617cd..f06c76896b 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml @@ -1,4 +1,4 @@ gcloud: - test: ./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip - xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + test: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + xctestrun-file: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun results-dir: test_dir diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml index e243189f31..d9166c0b7b 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml @@ -1,6 +1,6 @@ gcloud: - app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + app: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk environment-variables: coverage: true coverageFilePath: /sdcard/ From 80b2d451a3751350e0187438d42b1798a7639b96 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 6 Oct 2020 15:57:47 +0200 Subject: [PATCH 03/13] Add ios tests --- test_runner/src/test/kotlin/Debug.kt | 8 ++--- .../firebase/test/ios/IosRunCommandTest.kt | 29 +++++++++++++++++++ .../kotlin/ftl/fixtures/simple-ios-flank.yml | 4 +-- .../test_app_cases/flank-multiple-flaky.yml | 4 +-- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index e9c59f287d..608364e6bf 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -13,8 +13,8 @@ fun main() { val projectId = System.getenv("GOOGLE_CLOUD_PROJECT") ?: "YOUR PROJECT ID" -// val quantity = "multiple" -// val type = "flaky" + val quantity = "multiple" + val type = "flaky" // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 withGlobalExceptionHandling { @@ -22,14 +22,14 @@ fun main() { // "--debug", "firebase", "test", - "ios", + "android", "run", // "--dry", // "--dump-shards", "--output-style=single", // "--full-junit-result", // "--legacy-junit-result", - "-c=test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml", + "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", "--project=$projectId" // "--client-details=key1=value1,key2=value2" ) diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index d71ababfca..18eaffd960 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -1,10 +1,17 @@ package ftl.cli.firebase.test.ios import com.google.common.truth.Truth.assertThat +import ftl.args.IosArgs import ftl.config.Device import ftl.config.FtlConstants import ftl.config.FtlConstants.isWindows +import ftl.gc.GcStorage +import ftl.run.IOS_SHARD_FILE +import ftl.run.dumpShards import ftl.test.util.FlankTestRunner +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.verify import org.junit.Assume.assumeFalse import org.junit.Rule import org.junit.Test @@ -337,4 +344,26 @@ class IosRunCommandTest { assertThat(cmd.config.common.flank.useAverageTestTimeForNewTests).isTrue() } + + @Test + fun `should dump shards on ios test run`() { + mockkStatic("ftl.run.DumpShardsKt") + val runCmd = IosRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" + runCmd.run() + verify { dumpShards(any(), any(), any()) } + } + + @Test + fun `should dump shards on ios test run and not upload when disable-upload-results set`() { + mockkStatic("ftl.run.DumpShardsKt") + mockkObject(GcStorage) { + val runCmd = IosRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" + CommandLine(runCmd).parseArgs("--disable-results-upload") + runCmd.run() + verify { dumpShards(any(), any(), any()) } + verify(inverse = true) { GcStorage.upload(IOS_SHARD_FILE, any(), any()) } + } + } } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml index f06c76896b..50b3a617cd 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml @@ -1,4 +1,4 @@ gcloud: - test: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip - xctestrun-file: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + test: ./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun results-dir: test_dir diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml index d9166c0b7b..e243189f31 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-flaky.yml @@ -1,6 +1,6 @@ gcloud: - app: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: ./test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk environment-variables: coverage: true coverageFilePath: /sdcard/ From e37baef9b9de71a5e248ce25d8bf3c9fcf794549 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 6 Oct 2020 16:30:41 +0200 Subject: [PATCH 04/13] Add android test to verify calculation shards is called one --- .../firebase/test/android/AndroidRunCommand.kt | 10 ++-------- .../test/android/AndroidRunCommandTest.kt | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index c7434386dc..a17e15e2d8 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -1,7 +1,6 @@ package ftl.cli.firebase.test.android import ftl.args.AndroidArgs -import ftl.args.isInstrumentationTest import ftl.args.validate import ftl.cli.firebase.test.CommonRunCommand import ftl.config.FtlConstants @@ -47,13 +46,8 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() runBlocking { - when { - dumpShards -> dumpShards(args = config, obfuscatedOutput = obfuscate) - config.isInstrumentationTest -> { - newTestRun(config) - } - else -> newTestRun(config) - } + if (dumpShards) dumpShards(args = config, obfuscatedOutput = obfuscate) + else newTestRun(config) } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index e5fd55dbb7..a76fd976fc 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -1,15 +1,17 @@ package ftl.cli.firebase.test.android import com.google.common.truth.Truth.assertThat +import ftl.args.AndroidArgs import ftl.args.yml.AppTestPair import ftl.config.Device import ftl.config.FtlConstants import ftl.gc.GcStorage import ftl.run.ANDROID_SHARD_FILE -import ftl.run.dumpShards import ftl.run.exception.FlankConfigurationError +import ftl.run.platform.android.createAndroidTestContexts import ftl.run.saveShardChunks import ftl.test.util.FlankTestRunner +import io.mockk.coVerify import io.mockk.mockkObject import io.mockk.mockkStatic import io.mockk.verify @@ -552,4 +554,15 @@ class AndroidRunCommandTest { verify(inverse = true) { GcStorage.upload(ANDROID_SHARD_FILE, any(), any()) } } } + + @Test + fun `should calculate shards only one time on newRun`() { + mockkStatic("ftl.run.DumpShardsKt", "ftl.run.platform.android.CreateAndroidTestContextKt") + mockkObject(GcStorage) { + val runCmd = AndroidRunCommand() + runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml" + runCmd.run() + coVerify(exactly = 1) { any().createAndroidTestContexts() } + } + } } From 1d20ee342c9de31aafff53c1163540eaa20cf918 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 7 Oct 2020 11:53:42 +0200 Subject: [PATCH 05/13] Add obfuscate option to args and use in testRun --- test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/CommonArgs.kt | 2 +- test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/IosArgs.kt | 3 ++- .../ftl/cli/firebase/test/android/AndroidRunCommand.kt | 4 ++-- .../main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt | 4 ++-- test_runner/src/main/kotlin/ftl/run/DumpShards.kt | 6 ++---- .../src/main/kotlin/ftl/run/platform/RunAndroidTests.kt | 6 +++--- test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt | 3 ++- .../kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt | 4 ++-- test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt | 4 ++-- 12 files changed, 24 insertions(+), 21 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 41377b345b..a27ce59eea 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -19,7 +19,8 @@ data class AndroidArgs( val testRunnerClass: String?, val testTargets: List, val additionalAppTestApks: List, - override val useLegacyJUnitResult: Boolean + override val useLegacyJUnitResult: Boolean, + val obfuscateDumpShards: Boolean ) : IArgs by commonArgs { companion object : AndroidArgsCompanion() diff --git a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt index 3947c30ef7..c4e1505299 100644 --- a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt @@ -37,5 +37,5 @@ data class CommonArgs( override val disableResultsUpload: Boolean, override val defaultTestTime: Double, override val defaultClassTestTime: Double, - override val useAverageTestTimeForNewTests: Boolean, + override val useAverageTestTimeForNewTests: Boolean ) : IArgs diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 28a36b4d5e..b8b2c62b91 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -36,5 +36,6 @@ fun createAndroidArgs( environmentVariables = env ) } ?: emptyList(), - useLegacyJUnitResult = flank.useLegacyJUnitResult!! + useLegacyJUnitResult = flank.useLegacyJUnitResult!!, + obfuscateDumpShards = false ) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt index bf7bf170de..6d8d179287 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt @@ -21,7 +21,8 @@ private fun createIosArgs( xctestrunZip = gcloud.test?.normalizeFilePath().orEmpty(), xctestrunFile = gcloud.xctestrunFile?.normalizeFilePath().orEmpty(), xcodeVersion = gcloud.xcodeVersion, - testTargets = flank.testTargets?.filterNotNull().orEmpty() + testTargets = flank.testTargets?.filterNotNull().orEmpty(), + obfuscateDumpShards = true ) private fun convertToShardCount(inputValue: Int) = diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 66da6631f0..1829b1a9a9 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -11,7 +11,8 @@ data class IosArgs( val xctestrunZip: String, val xctestrunFile: String, val xcodeVersion: String?, - val testTargets: List + val testTargets: List, + val obfuscateDumpShards: Boolean ) : IArgs by commonArgs { override val useLegacyJUnitResult = true diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index a17e15e2d8..32fd12e41f 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -44,9 +44,9 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() + val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate().copy(obfuscateDumpShards = obfuscate) runBlocking { - if (dumpShards) dumpShards(args = config, obfuscatedOutput = obfuscate) + if (dumpShards) dumpShards(args = config) else newTestRun(config) } } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 6603b34365..ce42189abe 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -44,10 +44,10 @@ class IosRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = IosArgs.load(Paths.get(configPath), cli = this).validate() + val config = IosArgs.load(Paths.get(configPath), cli = this).validate().copy(obfuscateDumpShards = obfuscate) if (dumpShards) { - dumpShards(args = config, obfuscatedOutput = obfuscate) + dumpShards(args = config) } else runBlocking { newTestRun(config) } diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 998117b667..791af1d992 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -15,7 +15,6 @@ import java.nio.file.Paths suspend fun dumpShards( args: AndroidArgs, shardFilePath: String = ANDROID_SHARD_FILE, - obfuscatedOutput: Boolean = false ) { if (!args.isInstrumentationTest) throw FlankConfigurationError( "Cannot dump shards for non instrumentation test, ensure test apk has been set." @@ -25,20 +24,19 @@ suspend fun dumpShards( shardFilePath = shardFilePath, shards = shards, size = shards.size, - obfuscatedOutput = obfuscatedOutput + obfuscatedOutput = args.obfuscateDumpShards ) } fun dumpShards( args: IosArgs, shardFilePath: String = IOS_SHARD_FILE, - obfuscatedOutput: Boolean = false ) { saveShardChunks( shardFilePath = shardFilePath, shards = args.testShardChunks.testCases, size = args.testShardChunks.size, - obfuscatedOutput = obfuscatedOutput + obfuscatedOutput = args.obfuscateDumpShards ) } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 113181b206..f507793b68 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -78,15 +78,15 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS } private fun List.dumpShards(config: AndroidArgs) = apply { - filterIsInstance().asMatrixTestShards().saveShards() + filterIsInstance().asMatrixTestShards().saveShards(config.obfuscateDumpShards) if (config.disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, config.resultsBucket, config.resultsDir) } -private fun AndroidMatrixTestShards.saveShards() = saveShardChunks( +private fun AndroidMatrixTestShards.saveShards(obfuscateOutput: Boolean) = saveShardChunks( shardFilePath = ANDROID_SHARD_FILE, shards = this, size = size, - obfuscatedOutput = false + obfuscatedOutput = obfuscateOutput ) private suspend fun executeAndroidTestMatrix( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index b4bb77eefa..fb7ace0525 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -39,7 +39,8 @@ internal suspend fun runIosTests(iosArgs: IosArgs): TestResult = coroutineScope val history = GcToolResults.createToolResultsHistory(iosArgs) dumpShards(iosArgs) - if (iosArgs.disableResultsUpload.not()) GcStorage.upload(IOS_SHARD_FILE, iosArgs.resultsBucket, iosArgs.resultsDir) + if (iosArgs.disableResultsUpload.not()) + GcStorage.upload(IOS_SHARD_FILE, iosArgs.resultsBucket, iosArgs.resultsDir) // Upload only after parsing shards to detect missing methods early. val xcTestGcsPath = getXcTestGcPath(iosArgs, runGcsPath) diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 18eaffd960..82f6528042 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -351,7 +351,7 @@ class IosRunCommandTest { val runCmd = IosRunCommand() runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" runCmd.run() - verify { dumpShards(any(), any(), any()) } + verify { dumpShards(any(), any()) } } @Test @@ -362,7 +362,7 @@ class IosRunCommandTest { runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-ios-flank.yml" CommandLine(runCmd).parseArgs("--disable-results-upload") runCmd.run() - verify { dumpShards(any(), any(), any()) } + verify { dumpShards(any(), any()) } verify(inverse = true) { GcStorage.upload(IOS_SHARD_FILE, any(), any()) } } } diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index 8d59dd2d0d..a21b8ea92a 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -123,7 +123,7 @@ class DumpShardsKtTest { // when val actual = runBlocking { - dumpShards(AndroidArgs.load(mixedConfigYaml), TEST_SHARD_FILE, true) + dumpShards(AndroidArgs.load(mixedConfigYaml).copy(obfuscateDumpShards = true), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } @@ -183,7 +183,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE, true) + dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } From 4321c28aa62e27c1040e177ed566a78cf08eac12 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 7 Oct 2020 11:55:16 +0200 Subject: [PATCH 06/13] Update DumpShardsKtTest.kt --- test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index a21b8ea92a..cbe0b58542 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -153,7 +153,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) + dumpShards(IosArgs.load(ios2ConfigYaml).copy(obfuscateDumpShards = false), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } From 5d997dfccd894acb89e20d993d778e80a631ba42 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 7 Oct 2020 13:08:57 +0200 Subject: [PATCH 07/13] Update CreateIosArgs.kt --- test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt index 6d8d179287..62bfe59e39 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt @@ -22,7 +22,7 @@ private fun createIosArgs( xctestrunFile = gcloud.xctestrunFile?.normalizeFilePath().orEmpty(), xcodeVersion = gcloud.xcodeVersion, testTargets = flank.testTargets?.filterNotNull().orEmpty(), - obfuscateDumpShards = true + obfuscateDumpShards = false ) private fun convertToShardCount(inputValue: Int) = From dd48f600b6621e065055004baaa2debde55dcdc6 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 7 Oct 2020 13:15:17 +0200 Subject: [PATCH 08/13] remove unnecessary json --- android_shards.json | 31 ------------------------------- ios_shards.json | 21 --------------------- 2 files changed, 52 deletions(-) delete mode 100644 android_shards.json delete mode 100644 ios_shards.json diff --git a/android_shards.json b/android_shards.json deleted file mode 100644 index 7699594a26..0000000000 --- a/android_shards.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "matrix-0": { - "app": "/Users/adamfilipowicz/Repos/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", - "test": "/Users/adamfilipowicz/Repos/flank/test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-flaky-debug-androidTest.apk", - "shards": { - "shard-0": [ - "class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed", - "class com.example.test_app.InstrumentedTest#test2" - ], - "shard-1": [ - "class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized", - "class com.example.test_app.InstrumentedTest#test1" - ], - "shard-2": [ - "class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner", - "class com.example.test_app.InstrumentedTest#test0" - ], - "shard-3": [ - "class com.example.test_app.ParameterizedTest", - "class com.example.test_app.bar.BarInstrumentedTest#testBar", - "class com.example.test_app.foo.FooInstrumentedTest#testFoo" - ] - }, - "junit-ignored": [ - "class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress", - "class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore", - "class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar", - "class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo" - ] - } -} \ No newline at end of file diff --git a/ios_shards.json b/ios_shards.json deleted file mode 100644 index ea664f072f..0000000000 --- a/ios_shards.json +++ /dev/null @@ -1,21 +0,0 @@ -[ - [ - "EarlGreyExampleSwiftTests/testWithGreyAssertions", - "EarlGreyExampleSwiftTests/testLayout", - "EarlGreyExampleSwiftTests/testThatThrows", - "EarlGreyExampleSwiftTests/testWithInRoot", - "EarlGreyExampleSwiftTests/testCustomAction", - "EarlGreyExampleSwiftTests/testWithCondition", - "EarlGreyExampleSwiftTests/testBasicSelection", - "EarlGreyExampleSwiftTests/testWithCustomMatcher", - "EarlGreyExampleSwiftTests/testCollectionMatchers", - "EarlGreyExampleSwiftTests/testCatchErrorOnFailure", - "EarlGreyExampleSwiftTests/testWithCustomAssertion", - "EarlGreyExampleSwiftTests/testTableCellOutOfScreen", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAction", - "EarlGreyExampleSwiftTests/testBasicSelectionAndAssert", - "EarlGreyExampleSwiftTests/testWithCustomFailureHandler", - "EarlGreyExampleSwiftTests/testBasicSelectionActionAssert", - "EarlGreyExampleSwiftTests/testSelectionOnMultipleElements" - ] -] \ No newline at end of file From e571d4703e85b79e8197b48c7797aa26839d40bb Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 7 Oct 2020 13:39:26 +0200 Subject: [PATCH 09/13] Update DumpShardsKtTest.kt --- test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index cbe0b58542..c2f8604f20 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -153,7 +153,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml).copy(obfuscateDumpShards = false), TEST_SHARD_FILE) + dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } @@ -183,7 +183,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml), TEST_SHARD_FILE) + dumpShards(IosArgs.load(ios2ConfigYaml).copy(obfuscateDumpShards = true), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } From 833966b80db6130e9e1812482bb9d912fd69a6d0 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 8 Oct 2020 11:37:00 +0200 Subject: [PATCH 10/13] Update GetAndroidMatrixShards.kt --- .../kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt index c3345d32cc..d68f8e0bf7 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/GetAndroidMatrixShards.kt @@ -6,8 +6,8 @@ import ftl.run.model.AndroidTestShards import ftl.run.model.InstrumentationTestContext import ftl.shard.testCases -suspend fun AndroidArgs.getAndroidMatrixShards(): AndroidMatrixTestShards = - createAndroidTestContexts() +suspend fun AndroidArgs.getAndroidMatrixShards(): AndroidMatrixTestShards = this + .createAndroidTestContexts() .filterIsInstance() .asMatrixTestShards() From 5a9e7b995ed8e65f0dc29e1252dab6cb9d41cbb6 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 8 Oct 2020 12:17:06 +0200 Subject: [PATCH 11/13] Cr fixes --- .../src/main/kotlin/ftl/args/AndroidArgsCompanion.kt | 5 +++-- .../src/main/kotlin/ftl/args/CreateAndroidArgs.kt | 5 +++-- test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt | 11 +++++++---- .../src/main/kotlin/ftl/args/IosArgsCompanion.kt | 5 +++-- .../cli/firebase/test/android/AndroidRunCommand.kt | 2 +- .../kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt | 2 +- .../src/test/kotlin/ftl/run/DumpShardsKtTest.kt | 6 ++++-- 7 files changed, 22 insertions(+), 14 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt index 3051396a42..a568bc5eda 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt @@ -38,7 +38,8 @@ open class AndroidArgsCompanion : IArgs.ICompanion { internal fun load(yamlReader: Reader, cli: AndroidRunCommand? = null) = createAndroidArgs( config = defaultAndroidConfig() + - loadAndroidConfig(reader = yamlReader) + - cli?.config + loadAndroidConfig(reader = yamlReader) + + cli?.config, + obfuscate = cli?.obfuscate ?: false ) } diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index b8b2c62b91..e5b5add82b 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -9,7 +9,8 @@ fun createAndroidArgs( config: AndroidConfig? = null, gcloud: AndroidGcloudConfig = config!!.platform.gcloud, flank: AndroidFlankConfig = config!!.platform.flank, - commonArgs: CommonArgs = config!!.prepareAndroidCommonConfig() + commonArgs: CommonArgs = config!!.prepareAndroidCommonConfig(), + obfuscate: Boolean = false ) = AndroidArgs( commonArgs = commonArgs, // gcloud @@ -37,5 +38,5 @@ fun createAndroidArgs( ) } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!!, - obfuscateDumpShards = false + obfuscateDumpShards = obfuscate ) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt index 62bfe59e39..3bd95e70b1 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt @@ -5,24 +5,27 @@ import ftl.config.ios.IosFlankConfig import ftl.config.ios.IosGcloudConfig fun createIosArgs( - config: IosConfig + config: IosConfig, + obfuscate: Boolean = false ) = createIosArgs( gcloud = config.platform.gcloud, flank = config.platform.flank, - commonArgs = config.common.createCommonArgs(config.data) + commonArgs = config.common.createCommonArgs(config.data), + obfuscate = obfuscate ) private fun createIosArgs( gcloud: IosGcloudConfig, flank: IosFlankConfig, - commonArgs: CommonArgs + commonArgs: CommonArgs, + obfuscate: Boolean = false ) = IosArgs( commonArgs = commonArgs.copy(maxTestShards = convertToShardCount(commonArgs.maxTestShards)), xctestrunZip = gcloud.test?.normalizeFilePath().orEmpty(), xctestrunFile = gcloud.xctestrunFile?.normalizeFilePath().orEmpty(), xcodeVersion = gcloud.xcodeVersion, testTargets = flank.testTargets?.filterNotNull().orEmpty(), - obfuscateDumpShards = false + obfuscateDumpShards = obfuscate ) private fun convertToShardCount(inputValue: Int) = diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt index d5cdca044c..4c3ca0493b 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt @@ -38,7 +38,8 @@ open class IosArgsCompanion : IArgs.ICompanion { internal fun load(yamlReader: Reader, cli: IosRunCommand? = null) = createIosArgs( config = defaultIosConfig() + - loadIosConfig(reader = yamlReader) + - cli?.config + loadIosConfig(reader = yamlReader) + + cli?.config, + obfuscate = cli?.obfuscate ?: false ) } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 32fd12e41f..b66ded6f7f 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -44,7 +44,7 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate().copy(obfuscateDumpShards = obfuscate) + val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate().copy() runBlocking { if (dumpShards) dumpShards(args = config) else newTestRun(config) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index ce42189abe..966f7d3bdf 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -44,7 +44,7 @@ class IosRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = IosArgs.load(Paths.get(configPath), cli = this).validate().copy(obfuscateDumpShards = obfuscate) + val config = IosArgs.load(Paths.get(configPath), cli = this).validate() if (dumpShards) { dumpShards(args = config) diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index c2f8604f20..88ddb733aa 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -3,6 +3,8 @@ package ftl.run import com.google.common.truth.Truth.assertThat import ftl.args.AndroidArgs import ftl.args.IosArgs +import ftl.cli.firebase.test.android.AndroidRunCommand +import ftl.cli.firebase.test.ios.IosRunCommand import ftl.config.FtlConstants import ftl.doctor.assertEqualsIgnoreNewlineStyle import ftl.test.util.FlankTestRunner @@ -123,7 +125,7 @@ class DumpShardsKtTest { // when val actual = runBlocking { - dumpShards(AndroidArgs.load(mixedConfigYaml).copy(obfuscateDumpShards = true), TEST_SHARD_FILE) + dumpShards(AndroidArgs.load(mixedConfigYaml, AndroidRunCommand().apply { obfuscate = true }), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } @@ -183,7 +185,7 @@ class DumpShardsKtTest { if (FtlConstants.isWindows) return // TODO Windows Linux subsytem does not contain all expected commands // when val actual = runBlocking { - dumpShards(IosArgs.load(ios2ConfigYaml).copy(obfuscateDumpShards = true), TEST_SHARD_FILE) + dumpShards(IosArgs.load(ios2ConfigYaml, IosRunCommand().apply { obfuscate = true }), TEST_SHARD_FILE) File(TEST_SHARD_FILE).apply { deleteOnExit() }.readText() } From 8c51b26392f60d5541d3d83aacf0f1132ad00b19 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 8 Oct 2020 12:37:19 +0200 Subject: [PATCH 12/13] Fix creating empty dump on robo tests --- .../src/main/kotlin/ftl/run/platform/RunAndroidTests.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index f507793b68..9699ed7478 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -3,6 +3,7 @@ package ftl.run.platform import com.google.api.services.testing.Testing import com.google.api.services.testing.model.TestMatrix import ftl.args.AndroidArgs +import ftl.args.isInstrumentationTest import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix import ftl.gc.GcStorage @@ -77,10 +78,10 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS ) } -private fun List.dumpShards(config: AndroidArgs) = apply { +private fun List.dumpShards(config: AndroidArgs) = takeIf { config.isInstrumentationTest }?.apply { filterIsInstance().asMatrixTestShards().saveShards(config.obfuscateDumpShards) if (config.disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, config.resultsBucket, config.resultsDir) -} +} ?: this private fun AndroidMatrixTestShards.saveShards(obfuscateOutput: Boolean) = saveShardChunks( shardFilePath = ANDROID_SHARD_FILE, From e797c326b5c25ef30814e1ecf09e817fc075eb48 Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Thu, 8 Oct 2020 12:53:24 +0200 Subject: [PATCH 13/13] Update test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan Góral <60390247+jan-gogo@users.noreply.github.com> --- .../kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index b66ded6f7f..b99f6985a8 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -44,7 +44,7 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate().copy() + val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() runBlocking { if (dumpShards) dumpShards(args = config) else newTestRun(config)