From a144008205845cf1687741c818fcd4936a134d92 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Tue, 19 May 2020 11:38:41 +0200 Subject: [PATCH] Fix overlapping results --- .../docs/ascii/flank.jar_-android-run.adoc | 2 +- .../flank.jar_-firebase-test-android-run.adoc | 2 +- .../ascii/flank.jar_-firebase-test-ios-run.adoc | 2 +- test_runner/docs/ascii/flank.jar_-ios-run.adoc | 2 +- .../main/kotlin/ftl/run/common/FetchArtifacts.kt | 1 + .../kotlin/ftl/run/platform/RunAndroidTests.kt | 6 +++--- test_runner/src/main/kotlin/ftl/util/ObjPath.kt | 16 ++++++++++------ test_runner/src/main/kotlin/ftl/util/Utils.kt | 2 -- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 4 +++- .../src/test/kotlin/ftl/util/ObjPathTest.kt | 6 ++---- .../src/test/kotlin/ftl/util/UtilsTest.kt | 2 +- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/test_runner/docs/ascii/flank.jar_-android-run.adoc b/test_runner/docs/ascii/flank.jar_-android-run.adoc index 423c4e6ff3..d066e6df36 100644 --- a/test_runner/docs/ascii/flank.jar_-android-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-android-run.adoc @@ -144,7 +144,7 @@ Configuration is read from flank.yml Keeps the full path of downloaded files. Required when file names are not unique. *--output-style*=__:: - Output style of execution status. Maybe be one of [verbose, multi, single]For runs with only one test execution default value is 'verbose', in other cases'multi' is used as default. Output style 'multi' is not displayed correctly on consoleswhich not supports ansi codes, to avoid corrupted output use `single` or `verbose`. + Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. *--dump-shards*:: Dumps the shards to android_shards.json for debugging diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc index 61e053632f..0f56493d15 100644 --- a/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc @@ -156,7 +156,7 @@ Configuration is read from flank.yml Keeps the full path of downloaded files. Required when file names are not unique. *--output-style*=__:: - Output style of execution status. Maybe be one of [verbose, multi, single]For runs with only one test execution default value is 'verbose', in other cases'multi' is used as default. Output style 'multi' is not displayed correctly on consoleswhich not supports ansi codes, to avoid corrupted output use `single` or `verbose`. + Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. *--dump-shards*:: Dumps the shards to android_shards.json for debugging diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc index 0534901280..e8b59f7bf7 100644 --- a/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc @@ -136,7 +136,7 @@ Configuration is read from flank.yml Keeps the full path of downloaded files. Required when file names are not unique. *--output-style*=__:: - Output style of execution status. Maybe be one of [verbose, multi, single]For runs with only one test execution default value is 'verbose', in other cases'multi' is used as default. Output style 'multi' is not displayed correctly on consoleswhich not supports ansi codes, to avoid corrupted output use `single` or `verbose`. + Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. *--dump-shards*:: Dumps the shards to ios_shards.json for debugging diff --git a/test_runner/docs/ascii/flank.jar_-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-ios-run.adoc index 08e2755ac9..f03ddd35cc 100644 --- a/test_runner/docs/ascii/flank.jar_-ios-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-ios-run.adoc @@ -128,7 +128,7 @@ Configuration is read from flank.yml Keeps the full path of downloaded files. Required when file names are not unique. *--output-style*=__:: - Output style of execution status. Maybe be one of [verbose, multi, single]For runs with only one test execution default value is 'verbose', in other cases'multi' is used as default. Output style 'multi' is not displayed correctly on consoleswhich not supports ansi codes, to avoid corrupted output use `single` or `verbose`. + Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. *--dump-shards*:: Dumps the shards to ios_shards.json for debugging diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt index 21977b7412..267ed59676 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/FetchArtifacts.kt @@ -72,6 +72,7 @@ internal fun getDownloadPath(args: IArgs, blobPath: String): Path { return Paths.get( localDir, if (args.useLocalResultDir().not()) p.objName else "", + p.matrixName, p.shardName, p.deviceName, if (args.keepFilePath) p.filePathName else "", diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 14859c5689..4af82e4302 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -38,7 +38,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val history = GcToolResults.createToolResultsHistory(args) val otherGcsFiles = args.uploadOtherFiles(runGcsPath) - args.resolveApks().forEach { apks: ResolvedApks -> + args.resolveApks().forEachIndexed { index: Int, apks: ResolvedApks -> val testShards = apks.test?.let { test -> AndroidTestShard.getTestShardChunks(args, test) } @@ -47,7 +47,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS val shardsWithAtLeastOneTest = testShards.filterAtLeastOneTest() if (shardsWithAtLeastOneTest.isEmpty()) { // No tests to run, skipping the execution. - return@forEach + return@forEachIndexed } allTestShardChunks += shardsWithAtLeastOneTest } @@ -67,7 +67,7 @@ internal suspend fun runAndroidTests(args: AndroidArgs): TestResult = coroutineS testMatrices += executeAndroidTestMatrix(runCount = args.repeatTests) { GcAndroidTestMatrix.build( androidTestConfig = androidTestConfig, - runGcsPath = runGcsPath, + runGcsPath = "$runGcsPath/matrix_$index/", additionalApkGcsPaths = uploadedApks.additionalApks, androidDeviceList = androidDeviceList, args = args, diff --git a/test_runner/src/main/kotlin/ftl/util/ObjPath.kt b/test_runner/src/main/kotlin/ftl/util/ObjPath.kt index 3352ee36a8..211465723c 100644 --- a/test_runner/src/main/kotlin/ftl/util/ObjPath.kt +++ b/test_runner/src/main/kotlin/ftl/util/ObjPath.kt @@ -7,7 +7,8 @@ data class ObjPath( val objName: String, val shardName: String, val deviceName: String, - val filePathName: String + val filePathName: String, + val matrixName: String ) { companion object { fun parse(path: String): ObjPath { @@ -15,12 +16,13 @@ data class ObjPath( val fileName = parsed.fileName.toString() val objName = parsed.getName(0).toString() - val (deviceName, shardNumber) = parsed.getName(1).toString().split("-shard_").run { + val matrixName = parsed.getName(1).toString() + val (deviceName, shardNumber) = parsed.getName(2).toString().split("-shard_").run { first() to getOrElse(1) { "0" } } val shardName = "shard_$shardNumber" - val filePathName = if (parsed.nameCount > 3) { - parsed.parent.drop(2).joinToString("/") + val filePathName = if (parsed.nameCount > 4) { + parsed.parent.drop(3).joinToString("/") } else { "" } return ObjPath( @@ -28,7 +30,8 @@ data class ObjPath( objName = objName, shardName = shardName, deviceName = deviceName, - filePathName = filePathName + filePathName = filePathName, + matrixName = matrixName ) } @@ -48,7 +51,8 @@ data class ObjPath( objName = objName, shardName = shardName, deviceName = deviceName, - filePathName = filePathName + filePathName = filePathName, + matrixName = "" ) } } diff --git a/test_runner/src/main/kotlin/ftl/util/Utils.kt b/test_runner/src/main/kotlin/ftl/util/Utils.kt index 50d697aff0..31f64bede0 100644 --- a/test_runner/src/main/kotlin/ftl/util/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/util/Utils.kt @@ -84,8 +84,6 @@ fun uniqueObjectName(): String { bucketName.append(letter) } - bucketName.append("/") - return bucketName.toString() } diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index d5fd910ce7..8526a49824 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -35,7 +35,7 @@ class TestRunnerTest { private val gcsIosPath = "2019-03-22_15-30-02.189000_frjt/shard_0/iphone8-12.0-en-portrait/TestLogs/Test-Transient Testing-2019.03.22_08-29-41--0700.xcresult/1_Test/Diagnostics/EarlGreyExampleSwiftTests-C6803D8C-4BDB-4C84-8945-9AC64056FBA4/EarlGreyExampleSwiftTests-EDBFF942-A88A-46A5-87CA-A1E29555C2CA/StandardOutputAndStandardError.txt" private val gcsAndroidPath = - "2019-03-22_15-30-02.189000_frjt/iphone8-12.0-en-portrait-shard_0/StandardOutputAndStandardError.txt" + "2019-03-22_15-30-02.189000_frjt/matrix_1/iphone8-12.0-en-portrait-shard_0/StandardOutputAndStandardError.txt" private val localResultDir = "results" private val iosArgs = mockk() private val androidArgs = mockk() @@ -138,6 +138,7 @@ class TestRunnerTest { assertThat(downloadFile).isEqualTo( Paths.get( localResultDir, + parsed.matrixName, parsed.shardName, parsed.deviceName, parsed.filePathName, @@ -159,6 +160,7 @@ class TestRunnerTest { Paths.get( localResultDir, parsed.objName, + parsed.matrixName, parsed.shardName, parsed.deviceName, parsed.filePathName, diff --git a/test_runner/src/test/kotlin/ftl/util/ObjPathTest.kt b/test_runner/src/test/kotlin/ftl/util/ObjPathTest.kt index 7bafa9a671..cc00575b68 100644 --- a/test_runner/src/test/kotlin/ftl/util/ObjPathTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/ObjPathTest.kt @@ -1,16 +1,13 @@ package ftl.util import com.google.common.truth.Truth.assertThat -import ftl.test.util.FlankTestRunner import org.junit.Test -import org.junit.runner.RunWith -@RunWith(FlankTestRunner::class) class ObjPathTest { @Test fun parse() { - val path = "2019-03-22_15-39-20.400000_ESdl/NexusLowRes-28-en-portrait-shard_0/com/package/name/b.txt" + val path = "2019-03-22_15-39-20.400000_ESdl/matrix_1/NexusLowRes-28-en-portrait-shard_0/com/package/name/b.txt" val parsed = ObjPath.parse(path) assertThat(parsed.objName).isEqualTo("2019-03-22_15-39-20.400000_ESdl") @@ -18,6 +15,7 @@ class ObjPathTest { assertThat(parsed.shardName).isEqualTo("shard_0") assertThat(parsed.deviceName).isEqualTo("NexusLowRes-28-en-portrait") assertThat(parsed.filePathName).isEqualTo("com/package/name") + assertThat(parsed.matrixName).isEqualTo("matrix_1") } @Test diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index b4b8fc5a65..267e78e88f 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -69,7 +69,7 @@ class UtilsTest { @Test fun `uniqueObjectName verifyPattern`() { val randomName = uniqueObjectName() - assertThat(randomName.length).isEqualTo(32) + assertThat(randomName.length).isEqualTo(31) assertThat(randomName[4]).isEqualTo('-') assertThat(randomName[7]).isEqualTo('-') assertThat(randomName[10]).isEqualTo('_')