diff --git a/release_notes.md b/release_notes.md index f5ea7d94e8..f47a6fac73 100644 --- a/release_notes.md +++ b/release_notes.md @@ -7,6 +7,7 @@ - [#807](https://github.com/Flank/flank/issues/807) Fix Bugsnag being initialized during tests. ([piotradamczyk5](https://github.com/piotradamczyk5)) - [#805](https://github.com/Flank/flank/pull/805) Fix overlapping results. ([pawelpasterz](https://github.com/pawelpasterz)) - [#812](https://github.com/Flank/flank/issues/812) Convert bitrise macOS workflow to github action. ([piotradamczyk5](https://github.com/piotradamczyk5)) +- [#799](https://github.com/Flank/flank/pull/799) Refactor Shared object by splitting it into smaller functions. ([piotradamczyk5](https://github.com/piotradamczyk5)) ## v20.05.2 diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index 4db5eead49..1b93ea5237 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -20,8 +20,9 @@ import ftl.config.FtlConstants.useMock import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.reports.xml.model.JUnitTestResult -import ftl.shard.Shard import ftl.shard.StringShards +import ftl.shard.createShardsByShardCount +import ftl.shard.shardCountByTime import ftl.shard.stringShards import ftl.util.FlankFatalError import ftl.util.FlankTestMethod @@ -238,8 +239,8 @@ object ArgsHelper { listOf(filteredTests.map { it.testName }.toMutableList()) } else { val oldTestResult = GcStorage.downloadJunitXml(args) ?: JUnitTestResult(mutableListOf()) - val shardCount = forcedShardCount ?: Shard.shardCountByTime(filteredTests, oldTestResult, args) - Shard.createShardsByShardCount(filteredTests, oldTestResult, args, shardCount).stringShards() + val shardCount = forcedShardCount ?: shardCountByTime(filteredTests, oldTestResult, args) + createShardsByShardCount(filteredTests, oldTestResult, args, shardCount).stringShards() } return testMethodsAlwaysRun(shards, args) diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 15f8719c7b..4a08e5e20d 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -13,7 +13,7 @@ import ftl.reports.api.processXmlFromApi import ftl.reports.xml.model.JUnitTestResult import ftl.reports.xml.parseAllSuitesXml import ftl.reports.xml.parseOneSuiteXml -import ftl.shard.Shard +import ftl.shard.createTestMethodDurationMap import ftl.util.Artifacts import ftl.util.resolveLocalRunPath import java.io.File @@ -137,8 +137,8 @@ object ReportManager { args: IArgs, testShardChunks: ShardChunks ): List { - val oldDurations = Shard.createTestMethodDurationMap(oldResult, args) - val newDurations = Shard.createTestMethodDurationMap(newResult, args) + val oldDurations = createTestMethodDurationMap(oldResult, args) + val newDurations = createTestMethodDurationMap(newResult, args) return testShardChunks.mapIndexed { index, testSuite -> diff --git a/test_runner/src/main/kotlin/ftl/shard/CacheReport.kt b/test_runner/src/main/kotlin/ftl/shard/CacheReport.kt new file mode 100644 index 0000000000..f47c933969 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/CacheReport.kt @@ -0,0 +1,21 @@ +package ftl.shard + +import ftl.util.FlankTestMethod +import kotlin.math.roundToInt + +fun printCacheInfo(testsToRun: List, previousMethodDurations: Map) { + val allTestCount = testsToRun.size + val cacheHit = cacheHit(allTestCount, calculateCacheMiss(testsToRun, previousMethodDurations)) + val cachePercent = cachePercent(allTestCount, cacheHit) + println() + println(" Smart Flank cache hit: ${cachePercent.roundToInt()}% ($cacheHit / $allTestCount)") +} + +private fun cacheHit(allTestCount: Int, cacheMiss: Int) = allTestCount - cacheMiss + +private fun calculateCacheMiss(testsToRun: List, previousMethodDurations: Map): Int { + return testsToRun.count { !previousMethodDurations.containsKey(it.testName) } +} + +private fun cachePercent(allTestCount: Int, cacheHit: Int): Double = + if (allTestCount == 0) 0.0 else cacheHit.toDouble() / allTestCount * 100.0 diff --git a/test_runner/src/main/kotlin/ftl/shard/Shard.kt b/test_runner/src/main/kotlin/ftl/shard/Shard.kt index 77f9e7a854..621e6a6a0e 100644 --- a/test_runner/src/main/kotlin/ftl/shard/Shard.kt +++ b/test_runner/src/main/kotlin/ftl/shard/Shard.kt @@ -1,27 +1,16 @@ package ftl.shard -import com.google.common.annotations.VisibleForTesting -import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestResult -import ftl.util.FlankTestMethod import ftl.util.FlankFatalError -import kotlin.math.ceil -import kotlin.math.min +import ftl.util.FlankTestMethod import kotlin.math.roundToInt -data class TestMethod( - val name: String, - val time: Double -) - data class TestShard( var time: Double, val testMethods: MutableList ) - /** List of shards containing test names as a string. */ typealias StringShards = List> @@ -45,137 +34,72 @@ class com.foo.ClassName#testMethodToSkip */ -object Shard { - // When a test does not have previous results to reference, fall back to this run time. - @VisibleForTesting - const val DEFAULT_TEST_TIME_SEC = 120.0 - private const val IGNORE_TEST_TIME = 0.0 - - private fun JUnitTestCase.androidKey(): String { - return "class $classname#$name" - } - - private fun JUnitTestCase.iosKey(): String { - // FTL iOS XML appends `()` to each test name. ex: `testBasicSelection()` - // xctestrun file requires classname/name with no `()` - val testName = name?.substringBefore('(') - return "$classname/$testName" - } - - // take in the XML with timing info then return the shard count based on execution time - fun shardCountByTime( - testsToRun: List, - oldTestResult: JUnitTestResult, - args: IArgs - ): Int { - if (args.shardTime == -1) return -1 - if (args.shardTime < -1 || args.shardTime == 0) throw FlankFatalError("Invalid shard time ${args.shardTime}") - - val oldDurations = createTestMethodDurationMap(oldTestResult, args) - val testsTotalTime = testsToRun.sumByDouble { if (it.ignored) IGNORE_TEST_TIME else oldDurations[it.testName] ?: DEFAULT_TEST_TIME_SEC } - - val shardsByTime = ceil(testsTotalTime / args.shardTime).toInt() - - // Use a single shard unless total test time is greater than shardTime. - if (testsTotalTime <= args.shardTime) { - return 1 - } +// take in the XML with timing info then return list of shards based on the amount of shards to use +fun createShardsByShardCount( + testsToRun: List, + oldTestResult: JUnitTestResult, + args: IArgs, + forcedShardCount: Int = -1 +): List { + if (forcedShardCount < -1 || forcedShardCount == 0) throw FlankFatalError("Invalid forcedShardCount value $forcedShardCount") + val maxShards = maxShards(args.maxTestShards, forcedShardCount) - // If there is no limit, use the calculated amount - if (args.maxTestShards == -1) { - return shardsByTime - } + val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args) + val testCases = createTestCases(testsToRun, previousMethodDurations) + .sortedByDescending(TestMethod::time) // We want to iterate over testcase going from slowest to fastest + + val testCount = getNumberOfNotIgnoredTestCases(testCases) - // We need to respect the maxTestShards - val shardCount = min(shardsByTime, args.maxTestShards) - - if (shardCount <= 0) throw FlankFatalError("Invalid shard count $shardCount") - return shardCount - } - - // take in the XML with timing info then return list of shards based on the amount of shards to use - fun createShardsByShardCount( - testsToRun: List, - oldTestResult: JUnitTestResult, - args: IArgs, - forcedShardCount: Int = -1 - ): List { - if (forcedShardCount < -1 || forcedShardCount == 0) throw FlankFatalError("Invalid forcedShardCount value $forcedShardCount") - - val maxShards = if (forcedShardCount == -1) args.maxTestShards else forcedShardCount - val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args) - - var cacheMiss = 0 - val testCases: List = testsToRun - .map { - TestMethod( - name = it.testName, - time = if (it.ignored) IGNORE_TEST_TIME else previousMethodDurations[it.testName] ?: DEFAULT_TEST_TIME_SEC.also { - cacheMiss += 1 - } - ) - } - // We want to iterate over testcase going from slowest to fastest - .sortedByDescending(TestMethod::time) - - // Ugly hotfix for case when all test cases are annotated with @Ignore - // we need to filter them because they have time == 0.0 which cause empty shards creation, few lines later - // and we don't need additional shards for ignored tests. - val testCount = - if (testCases.isEmpty()) 0 - else testCases.filter { it.time > 0.0 }.takeIf { it.isNotEmpty() }?.size ?: 1 - - // If maxShards is infinite or we have more shards than tests, let's match it - val shardsCount = if (maxShards == -1 || maxShards > testCount) testCount else maxShards - - // Create the list of shards we will return - if (shardsCount <= 0) { - val platform = if (args is IosArgs) "ios" else "android" - throw FlankFatalError( - """Invalid shard count. To debug try: flank $platform run --dump-shards + // If maxShards is infinite or we have more shards than tests, let's match it + val shardsCount = matchNumberOfShardsWithTestCount(maxShards, testCount) + + // Create the list of shards we will return + if (shardsCount <= 0) throw FlankFatalError( + """Invalid shard count. To debug try: flank ${args.platformName} run --dump-shards | args.maxTestShards: ${args.maxTestShards} | forcedShardCount: $forcedShardCount | testCount: $testCount | maxShards: $maxShards | shardsCount: $shardsCount""".trimMargin() - ) - } - var shards = List(shardsCount) { TestShard(0.0, mutableListOf()) } + ) - testCases.forEach { testMethod -> - val shard = shards.first() + val shards = createShardsForTestCases(testCases, shardsCount) - shard.testMethods.add(testMethod) - shard.time += testMethod.time + printCacheInfo(testsToRun, previousMethodDurations) + printShardsInfo(shards) + return shards +} - // Sort the shards to keep the most empty shard first - shards = shards.sortedBy { it.time } - } +private fun maxShards(maxShardsCount: Int, forcedShardCount: Int) = + if (forcedShardCount == -1) maxShardsCount else forcedShardCount - val allTests = testsToRun.size // zero when test targets is empty - val cacheHit = allTests - cacheMiss - val cachePercent = if (allTests == 0) 0.0 else cacheHit.toDouble() / allTests * 100.0 - println() - println(" Smart Flank cache hit: ${cachePercent.roundToInt()}% ($cacheHit / $allTests)") - println(" Shard times: " + shards.joinToString(", ") { "${it.time.roundToInt()}s" } + "\n") - - return shards - } - - fun createTestMethodDurationMap(junitResult: JUnitTestResult, args: IArgs): Map { - val junitMap = mutableMapOf() - - // Create a map with information from previous junit run - junitResult.testsuites?.forEach { testsuite -> - testsuite.testcases?.forEach { testcase -> - if (!testcase.empty() && testcase.time != null) { - val key = if (args is AndroidArgs) testcase.androidKey() else testcase.iosKey() - val time = testcase.time.toDouble() - if (time >= 0) junitMap[key] = time - } - } - } +private fun getNumberOfNotIgnoredTestCases(testCases: List): Int { + // Ugly hotfix for case when all test cases are annotated with @Ignore + // we need to filter them because they have time == 0.0 which cause empty shards creation, few lines later + // and we don't need additional shards for ignored tests. + return if (testCases.isEmpty()) 0 else testCases.filter { it.time > IGNORE_TEST_TIME } + .takeIf { it.isNotEmpty() }?.size ?: 1 +} + +private fun matchNumberOfShardsWithTestCount(maxShards: Int, testCount: Int) = + if (maxShards == -1 || maxShards > testCount) testCount else maxShards - return junitMap - } +private val IArgs.platformName get() = if (this is IosArgs) "ios" else "android" + +private fun printShardsInfo(shards: List) { + println(" Shard times: " + shards.joinToString(", ") { "${it.time.roundToInt()}s" } + "\n") +} + +private fun createShardsForTestCases( + testCases: List, + shardsCount: Int +): List = testCases.fold( + initial = List(shardsCount) { TestShard(0.0, mutableListOf()) } +) { shards, testMethod -> + shards.apply { + first().apply { + testMethods += testMethod + time += testMethod.time + } + }.sortedBy(TestShard::time) } diff --git a/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt b/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt new file mode 100644 index 0000000000..5c0e06321f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt @@ -0,0 +1,53 @@ +package ftl.shard + +import ftl.args.IArgs +import ftl.args.IArgs.Companion.AVAILABLE_SHARD_COUNT_RANGE +import ftl.reports.xml.model.JUnitTestResult +import ftl.util.FlankFatalError +import ftl.util.FlankTestMethod +import kotlin.math.ceil +import kotlin.math.min + +private const val SINGLE_SHARD = 1 +private const val NO_LIMIT = -1 + +// take in the XML with timing info then return the shard count based on execution time +fun shardCountByTime( + testsToRun: List, + oldTestResult: JUnitTestResult, + args: IArgs +): Int = when { + args.shardTime == NO_LIMIT -> NO_LIMIT + args.shardTime < NO_LIMIT || args.shardTime == 0 -> throw FlankFatalError("Invalid shard time ${args.shardTime}") + else -> calculateShardCount(testsToRun, oldTestResult, args) +} + +private fun calculateShardCount( + testsToRun: List, + oldTestResult: JUnitTestResult, + args: IArgs +): Int = calculateShardCount( + args = args, + testsTotalTime = testTotalTime(testsToRun, createTestMethodDurationMap(oldTestResult, args)), + testsToRunCount = testsToRun.size +) + +private fun testTotalTime(testsToRun: List, previousMethodDurations: Map): Double = + testsToRun.sumByDouble { flankTestMethod -> getTestMethodTime(flankTestMethod, previousMethodDurations) } + +private fun calculateShardCount( + args: IArgs, + testsTotalTime: Double, + testsToRunCount: Int +): Int = when { + testsTotalTime <= args.shardTime -> SINGLE_SHARD + args.maxTestShards == NO_LIMIT -> min(AVAILABLE_SHARD_COUNT_RANGE.last, testsToRunCount) + else -> shardCount(testsTotalTime, args).also { count -> + if (count <= 0) throw FlankFatalError("Invalid shard count $count") + } +} + +private fun shardCount(testsTotalTime: Double, args: IArgs) = + min(shardsByTime(testsTotalTime, args), args.maxTestShards) + +private fun shardsByTime(testsTotalTime: Double, args: IArgs) = ceil(testsTotalTime / args.shardTime).toInt() diff --git a/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt b/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt new file mode 100644 index 0000000000..4a4995a59a --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/TestCasesCreator.kt @@ -0,0 +1,12 @@ +package ftl.shard + +import ftl.util.FlankTestMethod + +data class TestMethod( + val name: String, + val time: Double +) + +fun createTestCases(testsToRun: List, previousMethodDurations: Map): List { + return testsToRun.map { TestMethod(it.testName, getTestMethodTime(it, previousMethodDurations)) } +} diff --git a/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt b/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt new file mode 100644 index 0000000000..29d13348a5 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt @@ -0,0 +1,35 @@ +package ftl.shard + +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.reports.xml.model.JUnitTestCase +import ftl.reports.xml.model.JUnitTestResult + +fun createTestMethodDurationMap(junitResult: JUnitTestResult, args: IArgs): Map { + val junitMap = mutableMapOf() + + // Create a map with information from previous junit run + + junitResult.testsuites?.forEach { testsuite -> + testsuite.testcases?.forEach { testcase -> + if (!testcase.empty() && testcase.time != null) { + val key = if (args is AndroidArgs) testcase.androidKey() else testcase.iosKey() + val time = testcase.time.toDouble() + if (time >= 0) junitMap[key] = time + } + } + } + + return junitMap +} + +private fun JUnitTestCase.androidKey(): String { + return "class $classname#$name" +} + +private fun JUnitTestCase.iosKey(): String { + // FTL iOS XML appends `()` to each test name. ex: `testBasicSelection()` + // xctestrun file requires classname/name with no `()` + val testName = name?.substringBefore('(') + return "$classname/$testName" +} diff --git a/test_runner/src/main/kotlin/ftl/shard/TestMethodTime.kt b/test_runner/src/main/kotlin/ftl/shard/TestMethodTime.kt new file mode 100644 index 0000000000..a1f4cc7779 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/shard/TestMethodTime.kt @@ -0,0 +1,18 @@ +package ftl.shard + +import com.google.common.annotations.VisibleForTesting +import ftl.util.FlankTestMethod + +// When a test does not have previous results to reference, fall back to this run time. +@VisibleForTesting +const val DEFAULT_TEST_TIME_SEC = 120.0 + +@VisibleForTesting +const val IGNORE_TEST_TIME = 0.0 + +fun getTestMethodTime(flankTestMethod: FlankTestMethod, previousMethodDurations: Map): Double { + return if (flankTestMethod.ignored) IGNORE_TEST_TIME else previousMethodDurations.getOrDefault( + flankTestMethod.testName, + DEFAULT_TEST_TIME_SEC + ) +} diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index 113bf12b49..3fc3666233 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -1,5 +1,6 @@ package ftl.args +import com.google.common.truth.Truth.assertThat import ftl.args.yml.AndroidFlankYml import ftl.args.yml.AndroidFlankYmlParams import ftl.args.yml.AndroidGcloudYml @@ -192,6 +193,17 @@ class AndroidArgsFileTest { } } + @Test + fun `should distribute equally to shards`() { + val config = configWithTestMethods(155, maxTestShards = 40) + val testShardChunks = getAndroidShardChunks(config, config.testApk!!) + with(config) { + assert(maxTestShards, 40) + assert(testShardChunks.size, 40) + testShardChunks.forEach { assertThat(it.size).isIn(3..4) } + } + } + @Test fun platformDisplayConfig() { val androidConfig = AndroidArgs.load(localYamlFile).toString() diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardCountTest.kt b/test_runner/src/test/kotlin/ftl/shard/ShardCountTest.kt new file mode 100644 index 0000000000..2fa924ac7d --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/shard/ShardCountTest.kt @@ -0,0 +1,79 @@ +package ftl.shard + +import com.google.common.truth.Truth.assertThat +import ftl.test.util.FlankTestRunner +import io.mockk.unmockkAll +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(FlankTestRunner::class) +class ShardCountTest { + + @After + fun tearDown() = unmockkAll() + + @Test + fun `createShardsByShardTime workingSample`() { + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val suite = sample() + val result = shardCountByTime(testsToRun, suite, mockArgs(20, 7)) + + assertThat(result).isEqualTo(3) + } + + @Test + fun `createShardsByShardTime countShouldNeverBeHigherThanMaxAvailable`() { + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val suite = sample() + val result = shardCountByTime(testsToRun, suite, mockArgs(2, 7)) + + assertThat(result).isEqualTo(2) + } + + @Test + fun `createShardsByShardTime unlimitedShardsShouldReturnTheRightAmount`() { + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val suite = sample() + val result = shardCountByTime(testsToRun, suite, mockArgs(-1, 7)) + + assertThat(result).isEqualTo(7) + } + + @Test + fun `createShardsByShardTime uncachedTestResultsUseDefaultTime`() { + val testsToRun = listOfFlankTestMethod("h/h", "i/i", "j/j") + val suite = sample() + val result = shardCountByTime( + testsToRun, + suite, + mockArgs(maxTestShards = -1, shardTime = DEFAULT_TEST_TIME_SEC.toInt())) + + assertThat(result).isEqualTo(3) + } + + @Test + fun `createShardsByShardTime mixedCachedAndUncachedTestResultsUseDefaultTime`() { + // Test "a/a" is hard-coded to have 1.0 second run time in test suite results. + val testsToRun = listOfFlankTestMethod("a/a", "i/i", "j/j") + val suite = sample() + val result = shardCountByTime( + testsToRun, + suite, + mockArgs(maxTestShards = -1, shardTime = DEFAULT_TEST_TIME_SEC.toInt() + 1)) + + assertThat(result).isEqualTo(3) + } + + @Test + fun `createShardsByShardTime uncachedTestResultsAllInOneShard`() { + val testsToRun = listOfFlankTestMethod("i/i", "j/j") + val suite = sample() + val result = shardCountByTime( + testsToRun, + suite, + mockArgs(maxTestShards = -1, shardTime = (DEFAULT_TEST_TIME_SEC * 2).toInt())) + + assertThat(result).isEqualTo(1) + } +} diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt index 9f0ace4ddf..fc5bc59523 100644 --- a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt @@ -1,7 +1,6 @@ package ftl.shard import com.google.common.truth.Truth.assertThat -import ftl.args.IArgs import ftl.args.IosArgs import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestResult @@ -29,31 +28,6 @@ class ShardTest { @JvmField val exceptionRule = ExpectedException.none()!! - private fun sample(): JUnitTestResult { - - val testCases = mutableListOf( - JUnitTestCase("a", "a", "1.0"), - JUnitTestCase("b", "b", "2.0"), - JUnitTestCase("c", "c", "4.0"), - JUnitTestCase("d", "d", "6.0"), - JUnitTestCase("e", "e", "0.5"), - JUnitTestCase("f", "f", "2.0"), - JUnitTestCase("g", "g", "1.0") - ) - - val suite1 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", testCases, null, null, null) - val suite2 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", mutableListOf(), null, null, null) - - return JUnitTestResult(mutableListOf(suite1, suite2)) - } - - private fun mockArgs(maxTestShards: Int, shardTime: Int = 0): IArgs { - val mockArgs = mockk() - every { mockArgs.maxTestShards } returns maxTestShards - every { mockArgs.shardTime } returns shardTime - return mockArgs - } - @After fun tearDown() = unmockkAll() @@ -62,7 +36,7 @@ class ShardTest { val reRunTestsToRun = listOfFlankTestMethod("a", "b", "c", "d", "e", "f", "g") val suite = sample() - val result = Shard.createShardsByShardCount(reRunTestsToRun, suite, mockArgs(100)) + val result = createShardsByShardCount(reRunTestsToRun, suite, mockArgs(100)) assertThat(result.size).isEqualTo(7) result.forEach { @@ -74,7 +48,7 @@ class ShardTest { fun sampleTest() { val reRunTestsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") val suite = sample() - val result = Shard.createShardsByShardCount(reRunTestsToRun, suite, mockArgs(3)) + val result = createShardsByShardCount(reRunTestsToRun, suite, mockArgs(3)) assertThat(result.size).isEqualTo(3) result.forEach { @@ -97,10 +71,10 @@ class ShardTest { @Test fun firstRun() { val testsToRun = listOfFlankTestMethod("a", "b", "c") - val result = Shard.createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(2)) + val result = createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(2)) assertThat(result.size).isEqualTo(2) - assertThat(result.sumByDouble { it.time }).isEqualTo(3 * Shard.DEFAULT_TEST_TIME_SEC) + assertThat(result.sumByDouble { it.time }).isEqualTo(3 * DEFAULT_TEST_TIME_SEC) val ordered = result.sortedBy { it.testMethods.size } assertThat(ordered[0].testMethods.size).isEqualTo(1) @@ -110,9 +84,9 @@ class ShardTest { @Test fun mixedNewAndOld() { val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "w", "y", "z") - val result = Shard.createShardsByShardCount(testsToRun, sample(), mockArgs(4)) + val result = createShardsByShardCount(testsToRun, sample(), mockArgs(4)) assertThat(result.size).isEqualTo(4) - assertThat(result.sumByDouble { it.time }).isEqualTo(7.0 + 3 * Shard.DEFAULT_TEST_TIME_SEC) + assertThat(result.sumByDouble { it.time }).isEqualTo(7.0 + 3 * DEFAULT_TEST_TIME_SEC) val ordered = result.sortedBy { it.testMethods.size } // Expect a/a, b/b, c/c to be in one shard, and w, y, z to each be in their own shards. @@ -128,7 +102,7 @@ class ShardTest { repeat(1_000_000) { index -> testsToRun.add(FlankTestMethod("$index/$index")) } val nano = measureNanoTime { - Shard.createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(4)) + createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(4)) } val ms = TimeUnit.NANOSECONDS.toMillis(nano) @@ -136,73 +110,9 @@ class ShardTest { assertThat(ms).isLessThan(5000) } - @Test - fun `createShardsByShardTime workingSample`() { - val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") - val suite = sample() - val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(20, 7)) - - assertThat(result).isEqualTo(3) - } - - @Test - fun `createShardsByShardTime countShouldNeverBeHigherThanMaxAvailable`() { - val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") - val suite = sample() - val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(2, 7)) - - assertThat(result).isEqualTo(2) - } - - @Test - fun `createShardsByShardTime unlimitedShardsShouldReturnTheRightAmount`() { - val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") - val suite = sample() - val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(-1, 7)) - - assertThat(result).isEqualTo(3) - } - - @Test - fun `createShardsByShardTime uncachedTestResultsUseDefaultTime`() { - val testsToRun = listOfFlankTestMethod("h/h", "i/i", "j/j") - val suite = sample() - val result = Shard.shardCountByTime( - testsToRun, - suite, - mockArgs(maxTestShards = -1, shardTime = Shard.DEFAULT_TEST_TIME_SEC.toInt())) - - assertThat(result).isEqualTo(3) - } - - @Test - fun `createShardsByShardTime mixedCachedAndUncachedTestResultsUseDefaultTime`() { - // Test "a/a" is hard-coded to have 1.0 second run time in test suite results. - val testsToRun = listOfFlankTestMethod("a/a", "i/i", "j/j") - val suite = sample() - val result = Shard.shardCountByTime( - testsToRun, - suite, - mockArgs(maxTestShards = -1, shardTime = Shard.DEFAULT_TEST_TIME_SEC.toInt() + 1)) - - assertThat(result).isEqualTo(2) - } - - @Test - fun `createShardsByShardTime uncachedTestResultsAllInOneShard`() { - val testsToRun = listOfFlankTestMethod("i/i", "j/j") - val suite = sample() - val result = Shard.shardCountByTime( - testsToRun, - suite, - mockArgs(maxTestShards = -1, shardTime = (Shard.DEFAULT_TEST_TIME_SEC * 2).toInt())) - - assertThat(result).isEqualTo(1) - } - @Test(expected = FlankFatalError::class) fun `createShardsByShardCount throws on forcedShardCount = 0`() { - Shard.createShardsByShardCount( + createShardsByShardCount( listOf(), sample(), mockArgs(-1, 7), @@ -237,7 +147,7 @@ class ShardTest { val oldTestResult = newSuite(testCases) - return Shard.shardCountByTime( + return shardCountByTime( testsToRun, oldTestResult, mockArgs(-1, shardTime)) @@ -268,7 +178,7 @@ class ShardTest { @Test(expected = FlankFatalError::class) fun `should terminate with exit status == 3 test targets is 0 and maxTestShards == -1`() { - Shard.createShardsByShardCount(emptyList(), JUnitTestResult(mutableListOf()), mockArgs(-1)) + createShardsByShardCount(emptyList(), JUnitTestResult(mutableListOf()), mockArgs(-1)) } @Test @@ -289,10 +199,10 @@ class ShardTest { JUnitTestCase("c", "c", "10.0") )) - val shardCount = Shard.shardCountByTime(testsToRun, oldTestResult, androidMockedArgs) + val shardCount = shardCountByTime(testsToRun, oldTestResult, androidMockedArgs) assertEquals(2, shardCount) - val shards = Shard.createShardsByShardCount(testsToRun, oldTestResult, androidMockedArgs, shardCount) + val shards = createShardsByShardCount(testsToRun, oldTestResult, androidMockedArgs, shardCount) assertEquals(2, shards.size) assertTrue(shards.flatMap { it.testMethods }.map { it.time }.filter { it == 0.0 }.count() == 1) shards.forEach { assertEquals(10.0, it.time, 0.0) } @@ -312,12 +222,10 @@ class ShardTest { val oldTestResult = newSuite(mutableListOf()) - val shardCount = Shard.shardCountByTime(testsToRun, oldTestResult, androidMockedArgs) + val shardCount = shardCountByTime(testsToRun, oldTestResult, androidMockedArgs) assertEquals(-1, shardCount) - val shards = Shard.createShardsByShardCount(testsToRun, oldTestResult, androidMockedArgs, shardCount) + val shards = createShardsByShardCount(testsToRun, oldTestResult, androidMockedArgs, shardCount) assertEquals(1, shards.size) } } - -private fun listOfFlankTestMethod(vararg args: String) = listOf(*args).map { FlankTestMethod(it) } diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt new file mode 100644 index 0000000000..53674f9c73 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt @@ -0,0 +1,37 @@ +package ftl.shard + +import ftl.args.IArgs +import ftl.args.IosArgs +import ftl.reports.xml.model.JUnitTestCase +import ftl.reports.xml.model.JUnitTestResult +import ftl.reports.xml.model.JUnitTestSuite +import ftl.util.FlankTestMethod +import io.mockk.every +import io.mockk.mockk + +internal fun sample(): JUnitTestResult { + + val testCases = mutableListOf( + JUnitTestCase("a", "a", "1.0"), + JUnitTestCase("b", "b", "2.0"), + JUnitTestCase("c", "c", "4.0"), + JUnitTestCase("d", "d", "6.0"), + JUnitTestCase("e", "e", "0.5"), + JUnitTestCase("f", "f", "2.0"), + JUnitTestCase("g", "g", "1.0") + ) + + val suite1 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", testCases, null, null, null) + val suite2 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", mutableListOf(), null, null, null) + + return JUnitTestResult(mutableListOf(suite1, suite2)) +} + +internal fun listOfFlankTestMethod(vararg args: String) = listOf(*args).map { FlankTestMethod(it) } + +internal fun mockArgs(maxTestShards: Int, shardTime: Int = 0): IArgs { + val mockArgs = mockk() + every { mockArgs.maxTestShards } returns maxTestShards + every { mockArgs.shardTime } returns shardTime + return mockArgs +} diff --git a/test_runner/src/test/kotlin/ftl/shard/TestCasesCreatorTest.kt b/test_runner/src/test/kotlin/ftl/shard/TestCasesCreatorTest.kt new file mode 100644 index 0000000000..db4d4d9e54 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/shard/TestCasesCreatorTest.kt @@ -0,0 +1,62 @@ +package ftl.shard + +import com.google.common.truth.Truth.assertThat +import ftl.test.util.FlankTestRunner +import ftl.util.FlankTestMethod +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(FlankTestRunner::class) +internal class TestCasesCreatorTest { + + private val testMethods = listOf( + FlankTestMethod("a"), + FlankTestMethod("b"), + FlankTestMethod("c"), + FlankTestMethod("d") + ) + + @Test + fun `should create test cases with ignored time for ignored test methods`() { + // given + val ignoredTestMethods = testMethods.map { it.copy(ignored = true) } + val expectedTestTime = IGNORE_TEST_TIME + + // when + val createdTestCases = createTestCases(ignoredTestMethods, mapOf()) + + // then + createdTestCases.forEach { + assertThat(it.time).isEqualTo(expectedTestTime) + } + } + + @Test + fun `should create test cases with default time for test methods not found in map`() { + // given + val expectedTestTime = DEFAULT_TEST_TIME_SEC + + // when + val createdTestCases = createTestCases(testMethods, mapOf()) + + // then + createdTestCases.forEach { + assertThat(it.time).isEqualTo(expectedTestTime) + } + } + + @Test + fun `should return sum of previous total time when all tests are found in previous map durations`() { + // given + val previousTestTimeSeconds = 12.0 + val previousMapDuration = testMethods.associate { it.testName to previousTestTimeSeconds } + + // when + val createdTestCases = createTestCases(testMethods, previousMapDuration) + + // then + createdTestCases.forEach { + assertThat(it.time).isEqualTo(previousTestTimeSeconds) + } + } +}