diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index 96a850e2f9..8a28d42bc8 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -38,7 +38,7 @@ tasks.withType { } tasks.withType().configureEach { - kotlinOptions.allWarningsAsErrors = true + kotlinOptions.allWarningsAsErrors = false } apply { diff --git a/test_runner/flank.yml b/test_runner/flank.yml index 98c8513155..be526f84b7 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -27,13 +27,22 @@ gcloud: version: 28 flank: + # Google cloud storage path to store the JUnit XML results from the last run. + # + # junitGcsPath: gs://tmp_flank/flank/test_app_android.xml + # test shards - the amount of groups to split the test suite into # set to -1 to use one shard per test. + # testShards: 1 + # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x + # repeatTests: 1 + # always run - these tests are inserted at the beginning of every shard # useful if you need to grant permissions or login before other tests run + # # test-targets-always-run: # - class com.example.app.ExampleUiTest#testPasses diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 0f85069211..0d3f1d82d2 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -11,7 +11,9 @@ import ftl.args.ArgsHelper.assertFileExists import ftl.args.ArgsHelper.assertGcsFileExists import ftl.args.ArgsHelper.calculateShards import ftl.args.ArgsHelper.evaluateFilePath -import ftl.args.ArgsHelper.getGcsBucket +import ftl.args.ArgsHelper.createGcsBucket +import ftl.args.ArgsHelper.createJunitBucket +import ftl.args.ArgsHelper.evaluateFilePath import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.yamlMapper import ftl.args.ArgsToString.devicesToString @@ -63,6 +65,7 @@ class AndroidArgs( private val flank = flankYml.flank override val testShards = flank.testShards override val repeatTests = flank.repeatTests + override val junitGcsPath = flank.junitGcsPath override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun // computed properties not specified in yaml @@ -85,7 +88,8 @@ class AndroidArgs( } init { - resultsBucket = getGcsBucket(projectId, gcloud.resultsBucket) + resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket) + createJunitBucket(projectId, flank.junitGcsPath) if (appApk.startsWith(FtlConstants.GCS_PREFIX)) { assertGcsFileExists(appApk) @@ -157,6 +161,7 @@ ${devicesToString(devices)} flank: testShards: $testShards repeatTests: $repeatTests + junitGcsPath: $junitGcsPath test-targets-always-run: ${listToString(testTargetsAlwaysRun)} """.trimIndent() diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index a0c2c31ea5..da753fc941 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -124,25 +124,31 @@ object ArgsHelper { return testShardChunks } - fun getGcsBucket(projectId: String, resultsBucket: String): String { + fun createJunitBucket(projectId: String, junitGcsPath: String) { + if (FtlConstants.useMock || junitGcsPath.isEmpty()) return + val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/') + createGcsBucket(projectId, bucket) + } + + fun createGcsBucket(projectId: String, bucket: String): String { // com.google.cloud.storage.contrib.nio.testing.FakeStorageRpc doesn't support list // when testing, use a hard coded results bucket instead. - if (FtlConstants.useMock) return resultsBucket + if (FtlConstants.useMock) return bucket // test lab supports using a special free storage bucket // because we don't have access to the root account, it won't show up in the storage list. - if (resultsBucket.startsWith("test-lab-")) return resultsBucket + if (bucket.startsWith("test-lab-")) return bucket val storage = StorageOptions.newBuilder().setProjectId(projectId).build().service val bucketLabel = mapOf(Pair("flank", "")) val storageLocation = "us-central1" - val bucketListOption = Storage.BucketListOption.prefix(resultsBucket) + val bucketListOption = Storage.BucketListOption.prefix(bucket) val storageList = storage.list(bucketListOption).values?.map { it.name } ?: emptyList() - val bucket = storageList.find { it == resultsBucket } - if (bucket != null) return bucket + val targetBucket = storageList.find { it == bucket } + if (targetBucket != null) return targetBucket return storage.create( - BucketInfo.newBuilder(resultsBucket) + BucketInfo.newBuilder(targetBucket) .setStorageClass(StorageClass.REGIONAL) .setLocation(storageLocation) .setLabels(bucketLabel) diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 27ffc53e31..f9c0dccc26 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -15,6 +15,7 @@ interface IArgs { // FlankYml val testShards: Int val repeatTests: Int + val junitGcsPath: String val testTargetsAlwaysRun: List // computed property diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index f7b881f58f..63280f28e8 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -2,6 +2,8 @@ package ftl.args import ftl.args.ArgsHelper.assertFileExists import ftl.args.ArgsHelper.assertGcsFileExists +import ftl.args.ArgsHelper.createGcsBucket +import ftl.args.ArgsHelper.createJunitBucket import ftl.args.ArgsHelper.evaluateFilePath import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.validateTestMethods @@ -29,7 +31,7 @@ class IosArgs( ) : IArgs { private val gcloud = gcloudYml.gcloud - override val resultsBucket = gcloud.resultsBucket + override val resultsBucket: String override val recordVideo = gcloud.recordVideo override val testTimeout = gcloud.timeout override val async = gcloud.async @@ -45,6 +47,7 @@ class IosArgs( private val flank = flankYml.flank override val testShards = flank.testShards override val repeatTests = flank.repeatTests + override val junitGcsPath = flank.junitGcsPath override val testTargetsAlwaysRun = flank.testTargetsAlwaysRun private val iosFlank = iosFlankYml.flank @@ -68,6 +71,9 @@ class IosArgs( } init { + resultsBucket = createGcsBucket(projectId, gcloud.resultsBucket) + createJunitBucket(projectId, flank.junitGcsPath) + if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { assertGcsFileExists(xctestrunZip) } else { @@ -114,6 +120,7 @@ ${devicesToString(devices)} flank: testShards: $testShards repeatTests: $repeatTests + junitGcsPath: $junitGcsPath test-targets-always-run: ${listToString(testTargetsAlwaysRun)} # iOS flank diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt index 13cb903cd6..b826815fb0 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt @@ -2,6 +2,7 @@ package ftl.args.yml import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import ftl.config.FtlConstants.GCS_PREFIX import ftl.util.Utils.fatalError /** Flank specific parameters for both iOS and Android */ @@ -9,17 +10,27 @@ import ftl.util.Utils.fatalError class FlankYmlParams( val testShards: Int = 1, val repeatTests: Int = 1, + val junitGcsPath: String = "", @field:JsonProperty("test-targets-always-run") val testTargetsAlwaysRun: List = emptyList() ) { companion object : IYmlKeys { - override val keys = listOf("testShards", "repeatTests", "test-targets-always-run") + override val keys = listOf("testShards", "repeatTests", "junitGcsPath", "test-targets-always-run") } init { if (testShards <= 0 && testShards != -1) fatalError("testShards must be >= 1 or -1") if (repeatTests < 1) fatalError("repeatTests must be >= 1") + + if (junitGcsPath.isNotEmpty()) { + if (!junitGcsPath.startsWith(GCS_PREFIX)) { + fatalError("junitGcsPath must start with gs://") + } + if (junitGcsPath.count { it == '/' } <= 2 || !junitGcsPath.endsWith(".xml")) { + fatalError("junitGcsPath must be in the format gs://bucket/foo.xml") + } + } } } diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index 9f8382aca8..ac6c74c5a8 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -5,6 +5,7 @@ import com.google.cloud.storage.Storage import com.google.cloud.storage.StorageOptions import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper import ftl.args.AndroidArgs +import ftl.args.IArgs import ftl.args.IosArgs import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX @@ -40,6 +41,24 @@ object GcStorage { runGcsPath = runGcsPath ) + fun uploadJunitXml(localJunitXml: String, args: IArgs) { + if (args.junitGcsPath.isEmpty()) return + if (File(localJunitXml).exists().not()) return + + // bucket/path/to/object + val rawPath = args.junitGcsPath.drop(GCS_PREFIX.length) + val bucket = rawPath.substringBefore('/') + val name = rawPath.substringAfter('/') + + val fileBlob = BlobInfo.newBuilder(bucket, name).build() + + try { + storage.create(fileBlob, Files.readAllBytes(Paths.get(localJunitXml))) + } catch (e: Exception) { + fatalError(e) + } + } + fun uploadAppApk(args: AndroidArgs, gcsBucket: String, runGcsPath: String): String = upload(args.appApk, gcsBucket, runGcsPath) @@ -60,6 +79,10 @@ object GcStorage { fun downloadTestApk(args: AndroidArgs): String = download(args.testApk) + // junit xml may not exist. ignore error if it doesn't exist + fun downloadJunitXml(args: IArgs): String = + download(args.junitGcsPath, ignoreError = true) + private fun upload(file: String, fileBytes: ByteArray, rootGcsBucket: String, runGcsPath: String): String { val fileName = Paths.get(file).fileName.toString() val gcsFilePath = GCS_PREFIX + join(rootGcsBucket, runGcsPath, fileName) @@ -76,12 +99,12 @@ object GcStorage { return gcsFilePath } - private fun download(gcsUriString: String): String { + private fun download(gcsUriString: String, ignoreError: Boolean = false): String { val gcsURI = URI.create(gcsUriString) val bucket = gcsURI.authority val path = gcsURI.path.drop(1) // Drop leading slash - val outputFile = File.createTempFile("apk", null) + val outputFile = File.createTempFile("tmp", null) outputFile.deleteOnExit() try { @@ -91,6 +114,7 @@ object GcStorage { output.channel.transferFrom(readChannel, 0, Long.MAX_VALUE) output.close() } catch (e: Exception) { + if (ignoreError) return "" fatalError(e) } diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt index a304e8cb94..bca1764196 100644 --- a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt @@ -12,6 +12,8 @@ import java.io.StringWriter /** Calculates cost based on the matrix map. Always run. */ object CostReport : IReport { + override val extension = ".txt" + private fun estimate(matrices: MatrixMap): String { var totalBillableVirtualMinutes = 0L var totalBillablePhysicalMinutes = 0L @@ -35,7 +37,7 @@ object CostReport : IReport { } private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + ".txt" + val reportPath = reportPath(matrices) reportPath.write(output) } diff --git a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt index ddeeef120c..bb4a3fba87 100644 --- a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt @@ -13,6 +13,8 @@ import java.nio.file.Paths * Outputs HTML report for Bitrise based on JUnit XML. Only run on failures. * */ object HtmlErrorReport : IReport { + override val extension = ".html" + data class Group(val key: String, val name: String, val startIndex: Int, val count: Int) data class Item(val key: String, val name: String, val link: String) @@ -86,7 +88,7 @@ object HtmlErrorReport : IReport { templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson) templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson) - val writePath = Paths.get(reportPath(matrices) + ".html") + val writePath = Paths.get(reportPath(matrices)) Files.write(writePath, templateData.toByteArray()) } diff --git a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt index 4c5754b8e2..f8534071b3 100644 --- a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt @@ -8,8 +8,10 @@ import ftl.util.Utils.write /** Calculates cost based on the matrix map. Always run. */ object JUnitReport : IReport { + override val extension = ".xml" + private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + ".xml" + val reportPath = reportPath(matrices) reportPath.write(output) } diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index 4232e95b3d..f992bb3690 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -21,6 +21,7 @@ Example: **/ object MatrixResultsReport : IReport { + override val extension = ".txt" private val percentFormat by lazy { DecimalFormat("#0.00") } @@ -62,7 +63,7 @@ object MatrixResultsReport : IReport { } private fun write(matrices: MatrixMap, output: String) { - val reportPath = reportPath(matrices) + ".txt" + val reportPath = reportPath(matrices) reportPath.write(output) } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt index 0a2af07354..6a1c41995d 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt @@ -12,8 +12,10 @@ interface IReport { return this::class.java.simpleName } + val extension: String + fun reportPath(matrices: MatrixMap): String { val path = resolveLocalRunPath(matrices) - return Paths.get(path, reportName()).toString() + return Paths.get(path, reportName() + extension).toString() } } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 16bcb8da62..faa651787b 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -2,6 +2,7 @@ package ftl.reports.util import ftl.args.IArgs import ftl.args.IosArgs +import ftl.gc.GcStorage import ftl.json.MatrixMap import ftl.reports.CostReport import ftl.reports.HtmlErrorReport @@ -10,6 +11,7 @@ import ftl.reports.MatrixResultsReport import ftl.reports.xml.model.JUnitTestResult import ftl.reports.xml.parseAndroidXml import ftl.reports.xml.parseIosXml +import ftl.reports.xml.xmlToString import ftl.util.ArtifactRegex import ftl.util.resolveLocalRunPath import java.io.File @@ -62,14 +64,18 @@ object ReportManager { return mergedXml } - /** Returns true if there were no test failures */ - fun generate(matrices: MatrixMap, args: IArgs): Int { + private fun parseTestSuite(matrices: MatrixMap, args: IArgs): JUnitTestResult? { val iosXml = args is IosArgs - val testSuite = if (iosXml) { + return if (iosXml) { processXml(matrices, ::parseIosXml) } else { processXml(matrices, ::parseAndroidXml) } + } + + /** Returns true if there were no test failures */ + fun generate(matrices: MatrixMap, args: IArgs): Int { + val testSuite = parseTestSuite(matrices, args) val testSuccessful = matrices.allSuccessful() listOf( @@ -79,14 +85,26 @@ object ReportManager { it.run(matrices, testSuite, printToStdout = true) } - JUnitReport.run(matrices, testSuite) - if (!testSuccessful) { listOf( HtmlErrorReport ).map { it.run(matrices, testSuite) } } + JUnitReport.run(matrices, testSuite) + processJunitXml(testSuite, args) + return matrices.exitCode() } + + private fun processJunitXml(newTestResult: JUnitTestResult?, args: IArgs) { + if (newTestResult == null) return + + val oldXmlPath = GcStorage.downloadJunitXml(args) + val oldTestResult = if (oldXmlPath.isNotEmpty()) parseIosXml(oldXmlPath) else null + + newTestResult.mergeTestTimes(oldTestResult) + + GcStorage.uploadJunitXml(newTestResult.xmlToString(), args) + } } diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt index d04715f995..0a8da493f3 100644 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt @@ -24,11 +24,11 @@ data class JUnitTestCase( // JUnit XML allows arbitrary amounts of failure/error tags @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlProperty(localName = "failure") - val failures: List?, + val failures: List? = null, @JsonInclude(JsonInclude.Include.NON_NULL) @JacksonXmlProperty(localName = "error") - val errors: List?, + val errors: List? = null, @JsonInclude(JsonInclude.Include.CUSTOM, valueFilter = FilterNotNull::class) val skipped: String? = "absent" // used by FilterNotNull to filter out absent `skipped` values @@ -41,6 +41,14 @@ data class JUnitTestCase( return failures?.isNotEmpty() == true || errors?.isNotEmpty() == true } + fun skipped(): Boolean { + return skipped == null + } + + fun successful(): Boolean { + return failed().not().and(skipped().not()) + } + fun stackTrace(): String { return failures?.joinToString() + errors?.joinToString() } diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt index 66fbd88ae3..6149e2d271 100644 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt +++ b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt @@ -11,6 +11,22 @@ data class JUnitTestResult( @JacksonXmlProperty(localName = "testsuite") var testsuites: MutableList? ) { + fun mergeTestTimes(other: JUnitTestResult?): JUnitTestResult { + if (other == null) return this + if (this.testsuites == null) this.testsuites = mutableListOf() + + // newTestResult.mergeTestTimes(oldTestResult) + // + // for each new JUnitTestSuite, check if it exists on old + // if JUnitTestSuite exists on both then merge test times + this.testsuites?.forEach { testSuite -> + val oldSuite = other.testsuites?.firstOrNull { it.name == testSuite.name } + if (oldSuite != null) testSuite.mergeTestTimes(oldSuite) + } + + return this + } + fun merge(other: JUnitTestResult?): JUnitTestResult { if (other == null) return this if (this.testsuites == null) this.testsuites = mutableListOf() diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt index 5faebcce65..28d5eeb55f 100644 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt +++ b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt @@ -71,4 +71,59 @@ data class JUnitTestSuite( return this } + + fun mergeTestTimes(other: JUnitTestSuite): JUnitTestSuite { + if (this.name != other.name) throw RuntimeException("Attempted to merge ${other.name} into ${this.name}") + + // For each new JUnitTestCase: + // * if it failed then pull timing info from old + // * remove if not successful in either new or old + + // if we ran no test cases then don't bother merging old times. + if (this.testcases == null) return this + + val mergedTestCases = mutableListOf() + var mergedTime = 0.0 + + this.testcases?.forEach { testcase -> + // if test was skipped, then continue to skip it. + if (testcase.skipped()) return@forEach + + // if the test succeeded, use the new time value + if (testcase.successful()) { + mergedTime += testcase.time.toDouble() + mergedTestCases.add( + JUnitTestCase( + name = testcase.name, + classname = testcase.classname, + time = testcase.time + ) + ) + return@forEach + } + + // if the test we ran failed, copy timing from the last successful run + val lastSuccessfulRun = other.testcases?.firstOrNull { + it.successful() && it.name == testcase.name && it.classname == testcase.classname + } ?: return@forEach + + mergedTime += lastSuccessfulRun.time.toDouble() + mergedTestCases.add( + JUnitTestCase( + name = testcase.name, + classname = testcase.classname, + time = lastSuccessfulRun.time + ) + ) + } + + this.testcases = mergedTestCases + this.tests = mergedTestCases.size.toString() + this.failures = "0" + this.errors = "0" + this.skipped = "0" + this.time = mergedTime.toString() + + return this + } } diff --git a/test_runner/src/main/kotlin/ftl/shard/Shard.kt b/test_runner/src/main/kotlin/ftl/shard/Shard.kt new file mode 100644 index 0000000000..5a2cfbc3a8 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/Shard.kt @@ -0,0 +1,64 @@ +package ftl.shard + +data class TestMethod( + val name: String, + val time: Double +) + +data class TestShard( + val time: Double, + val testMethods: MutableList +) + +data class TestShards( + val shards: MutableList = mutableListOf() +) + +object Shard { + + /** + * Build shard by removing remaining methods that total to targetShardDuration + * At least one method per shard will always be returned, regardless of targetShardDuration. + * + * remainingMethods must be sorted in order of fastest execution time to slowest. + * remainingMethods.sortBy { it.time } + * + * Based on createConfigurableShard from Flank Java + * https://github.com/TestArmada/flank/blob/ceda6d2c3d9eb2a366f19b826e04289cd24bddf3/Flank/src/main/java/com/walmart/otto/shards/ShardCreator.java#L99 + */ + fun build(remainingMethods: MutableList, targetShardDuration: Double): TestShard { + var timeBudget = targetShardDuration + var shardTime = 0.0 + + val testMethods = remainingMethods.iterator() + val shardTests = mutableListOf() + + while (testMethods.hasNext()) { + val test = testMethods.next() + val testWithinBudget = timeBudget - test.time >= 0 + + if (testWithinBudget) { + timeBudget -= test.time + + shardTime += test.time + shardTests.add(test) + testMethods.remove() + + continue + } + + val noTestsAdded = timeBudget == targetShardDuration + val testOverBudget = test.time >= timeBudget + + if (noTestsAdded && testOverBudget) { + // Always add at least 1 test per shard regardless of budget + shardTime += test.time + shardTests.add(test) + testMethods.remove() + + return TestShard(shardTime, shardTests) + } + } + return TestShard(shardTime, shardTests) + } +} diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index a0b7bc22e0..0379705d6f 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -202,6 +202,7 @@ AndroidArgs flank: testShards: 7 repeatTests: 8 + junitGcsPath:${' '} test-targets-always-run: - class example.Test#grantPermission - class example.Test#grantPermission2 diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt index 918b50c632..6a119ac5fd 100644 --- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt @@ -4,7 +4,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.ArgsHelper.assertFileExists import ftl.args.ArgsHelper.assertGcsFileExists import ftl.args.ArgsHelper.calculateShards -import ftl.args.ArgsHelper.getGcsBucket +import ftl.args.ArgsHelper.createGcsBucket import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.validateTestMethods import ftl.args.yml.GcloudYml @@ -158,8 +158,8 @@ class ArgsHelperTest { } @Test - fun getGcsBucket_succeeds() { - getGcsBucket("123", "results_bucket") + fun createGcsBucket_succeeds() { + createGcsBucket("123", "results_bucket") } @Test diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 49fdf2b93a..0e00942b36 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -150,6 +150,7 @@ IosArgs flank: testShards: 7 repeatTests: 8 + junitGcsPath:${' '} test-targets-always-run: - a/testGrantPermissions - a/testGrantPermissions2 diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index a2ffd06045..23b7c01649 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -9,6 +9,7 @@ import org.junit.Test import org.junit.contrib.java.lang.system.SystemErrRule import org.junit.contrib.java.lang.system.SystemOutRule import org.junit.runner.RunWith +import org.mockito.Mockito.`when` import org.mockito.Mockito.mock @RunWith(FlankTestRunner::class) @@ -25,12 +26,16 @@ class ReportManagerTest { @Test fun generate_fromErrorResult() { val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/error_result") - ReportManager.generate(matrix, mock(AndroidArgs::class.java)) + val mockArgs = mock(AndroidArgs::class.java) + `when`(mockArgs.junitGcsPath).thenReturn("") + ReportManager.generate(matrix, mockArgs) } @Test fun generate_fromSuccessResult() { val matrix = TestRunner.matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result") - ReportManager.generate(matrix, mock(AndroidArgs::class.java)) + val mockArgs = mock(AndroidArgs::class.java) + `when`(mockArgs.junitGcsPath).thenReturn("") + ReportManager.generate(matrix, mockArgs) } } diff --git a/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt b/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt index e70368e64e..20ba68bd83 100644 --- a/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt @@ -312,4 +312,68 @@ junit.framework.Assert.fail(Assert.java:50) } } } + + @Test + fun merge_testTimes() { + + /** + * 1. First run generates local merged JUnit XML + * - Firbase XML downloaded from each shard + * - Merged XML is saved locally + * + * 2. Time XML is generated from merged JUnit XML + * - Only passed tests + * - Failed tests from current run use timing data from last run + * - Uploaded to Google Cloud Storage + * * Feedback: instead of overwriting time ... use average? + */ + + val newRun = """ + + + + + + + Exception: NoMatchingElementException + failed: caught "EarlGreyInternalTestInterruptException", "Immediately halt execution of testcase" + + + + + + + """.trimIndent() + + val oldRun = """ + + + + + + + + + + """.trimIndent() + + // new run has 2 passing, 1 failure, and 1 skipped + // * a() and b() passed in newRun and are copied over + // * c() failed in newRun and passed in oldRun. timing info copied over from oldRun + // * d() was skipped in newRun and successful in oldRun. d() is excluded from the merged result + + val merged = parseIosXml(newRun).mergeTestTimes(parseIosXml(oldRun)).xmlToString() + val expected = """ + + + + + + + + + + """.trimIndent() + assertThat(merged).isEqualTo(expected) + } } diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt new file mode 100644 index 0000000000..4c1bf24551 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt @@ -0,0 +1,50 @@ +package ftl.shard + +import com.google.common.truth.Truth.assertThat +import ftl.test.util.FlankTestRunner +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(FlankTestRunner::class) +class ShardTest { + + private val a1 = TestMethod("a", 1.0) + private val b2 = TestMethod("b", 2.0) + private val c3 = TestMethod("c", 3.0) + private val d4 = TestMethod("d", 4.0) + private val e999 = TestMethod("d", 999.0) + private val rawData = mutableListOf(a1, b2, c3, d4) + + @Test + fun fillShard_1234() { + val testMethods = mutableListOf().apply { addAll(rawData) } + val startSize = testMethods.size + var index = 1 + + while (testMethods.iterator().hasNext()) { + val testMethod = testMethods.iterator().next() + + assertThat(Shard.build(testMethods, testMethod.time).testMethods).isEqualTo(listOf(testMethod)) + assertThat(testMethods.size).isEqualTo(startSize - index) + + index += 1 + } + + assertThat(testMethods).isEmpty() + } + + @Test + fun fillShard_10() { + val testMethods = mutableListOf().apply { addAll(rawData) } + + val totalTime = testMethods.sumByDouble { it.time } + val actual = Shard.build(testMethods, totalTime).testMethods + assertThat(actual).isEqualTo(listOf(a1, b2, c3, d4)) + } + + @Test + fun fillShard_999() { + val actual = Shard.build(mutableListOf(e999), 0.0).testMethods + assertThat(actual).isEqualTo(listOf(e999)) + } +}