diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt index d082ff854a..b6f2de0396 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidTestShard.kt @@ -3,8 +3,10 @@ package ftl.args import com.linkedin.dex.parser.DexParser import com.linkedin.dex.parser.TestMethod import ftl.config.FtlConstants +import ftl.filter.TestFilter import ftl.filter.TestFilters import ftl.gc.GcStorage +import ftl.util.FlankTestMethod import ftl.util.Utils import kotlinx.coroutines.runBlocking @@ -24,7 +26,7 @@ object AndroidTestShard { return ArgsHelper.calculateShards(filteredTests, args) } - private fun getTestMethods(args: AndroidArgs, testLocalApk: String): List { + private fun getTestMethods(args: AndroidArgs, testLocalApk: String): List { val allTestMethods = DexParser.findTestMethods(testLocalApk) val shouldIgnoreMissingTests = allTestMethods.isEmpty() && args.disableSharding val shouldThrowErrorIfMissingTests = allTestMethods.isEmpty() && !args.disableSharding @@ -33,14 +35,18 @@ object AndroidTestShard { shouldThrowErrorIfMissingTests -> throw IllegalStateException(Utils.fatalError("Test APK has no tests")) } val testFilter = TestFilters.fromTestTargets(args.testTargets) - val filteredTests = allTestMethods - .asSequence() - .distinct() - .filter(testFilter.shouldRun) - .map(TestMethod::testName) - .map { "class $it" } - .toList() - require(FtlConstants.useMock || filteredTests.isNotEmpty()) { Utils.fatalError("All tests filtered out") } - return filteredTests + return allTestMethods filterWith testFilter } + + private infix fun List.filterWith(filter: TestFilter) = asSequence() + .distinct() + .filter(filter.shouldRun) + .map { FlankTestMethod("class ${it.testName}", it.isIgnored) } + .toList() + .also { + require(FtlConstants.useMock || it.isNotEmpty()) { Utils.fatalError("All tests filtered out") } + } } + +private val TestMethod.isIgnored: Boolean + get() = annotations.map { it.name }.contains("org.junit.Ignore") diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index 905c544ac8..5afec4e20a 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -23,6 +23,7 @@ import ftl.reports.xml.model.JUnitTestResult import ftl.shard.Shard import ftl.shard.StringShards import ftl.shard.stringShards +import ftl.util.FlankTestMethod import ftl.util.Utils import java.io.File import java.net.URI @@ -55,7 +56,7 @@ object ArgsHelper { fun assertCommonProps(args: IArgs) { Utils.assertNotEmpty( args.project, "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + - "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + "or save service account credential to ${defaultCredentialPath}\n" + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" ) @@ -114,7 +115,7 @@ object ArgsHelper { testTargets: List, validTestMethods: Collection, from: String, - skipValidation: Boolean = FtlConstants.useMock + skipValidation: Boolean = useMock ) { val missingMethods = testTargets - validTestMethods @@ -123,7 +124,7 @@ object ArgsHelper { } fun createJunitBucket(projectId: String, junitGcsPath: String) { - if (FtlConstants.useMock || junitGcsPath.isEmpty()) return + if (useMock || junitGcsPath.isEmpty()) return val bucket = junitGcsPath.drop(GCS_PREFIX.length).substringBefore('/') createGcsBucket(projectId, bucket) } @@ -190,7 +191,7 @@ object ArgsHelper { } fun getDefaultProjectId(): String? { - if (FtlConstants.useMock) return "mockProjectId" + if (useMock) return "mockProjectId" // Allow users control over project by checking using Google's logic first before falling back to JSON. return ServiceOptions.getDefaultProjectId() ?: serviceAccountProjectId() @@ -220,9 +221,9 @@ object ArgsHelper { return ArgsFileVisitor("glob:$filePath").walk(searchDir) } - fun calculateShards(filteredTests: List, args: IArgs): List> { + fun calculateShards(filteredTests: List, args: IArgs): List> { val shards = if (args.disableSharding) { - mutableListOf(filteredTests as MutableList) + mutableListOf(filteredTests.map { it.testName } as MutableList) } else { val oldTestResult = GcStorage.downloadJunitXml(args) ?: JUnitTestResult(mutableListOf()) val shardCount = Shard.shardCountByTime(filteredTests, oldTestResult, args) diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index cb3cf9416d..3ad4e2b258 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -21,6 +21,7 @@ import ftl.config.Device import ftl.config.FtlConstants import ftl.ios.IosCatalog import ftl.ios.Xctestrun +import ftl.util.FlankTestMethod import ftl.util.Utils.fatalError import java.nio.file.Files import java.nio.file.Path @@ -69,7 +70,7 @@ class IosArgs( if (disableSharding) return@lazy listOf(emptyList()) val validTestMethods = Xctestrun.findTestNames(xctestrunFile) - val testsToShard = filterTests(validTestMethods, testTargets).distinct() + val testsToShard = filterTests(validTestMethods, testTargets).distinct().map { FlankTestMethod(it, ignored = false) } ArgsHelper.calculateShards(testsToShard, this) } diff --git a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt index 01111e30e0..14ffb79561 100644 --- a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt +++ b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt @@ -43,9 +43,6 @@ object TestFilters { private const val ARGUMENT_TEST_FILE = "testFile" private const val ARGUMENT_NOT_TEST_FILE = "notTestFile" - // JUnit @Ignore tests are removed. - private const val ANNOTATION_IGNORE = "org.junit.Ignore" - private val FILTER_ARGUMENT by lazy { val pattern = listOf( @@ -75,24 +72,20 @@ object TestFilters { } fun fromTestTargets(targets: List): TestFilter { - return if (targets.isEmpty()) { - notIgnored() - } else { - val parsedFilters = - targets - .asSequence() - .map(String::trim) - .map(TestFilters::parseSingleFilter) - .toList() - - // select test method name filters and short circuit if they match ex: class a.b#c - val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray() - val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray() - - val result = allOf(notIgnored(), *annotationFilters, anyOf(*otherFilters)) - if (FtlConstants.useMock) println(result.describe) - result - } + val parsedFilters = + targets + .asSequence() + .map(String::trim) + .map(TestFilters::parseSingleFilter) + .toList() + + // select test method name filters and short circuit if they match ex: class a.b#c + val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray() + val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray() + + val result = allOf(*annotationFilters, anyOf(*otherFilters)) + if (FtlConstants.useMock) println(result.describe) + return result } private fun parseSingleFilter(target: String): TestFilter { @@ -156,14 +149,6 @@ object TestFilters { isAnnotation = true ) - private fun notIgnored(): TestFilter = TestFilter( - describe = "notIgnored", - shouldRun = { testMethod -> - withAnnotation(listOf(ANNOTATION_IGNORE)).shouldRun(testMethod).not() - }, - isAnnotation = true - ) - private fun not(filter: TestFilter): TestFilter = TestFilter( describe = "not ${filter.describe}", shouldRun = { testMethod -> diff --git a/test_runner/src/main/kotlin/ftl/shard/Shard.kt b/test_runner/src/main/kotlin/ftl/shard/Shard.kt index d320bb31c3..8ccb89ca7c 100644 --- a/test_runner/src/main/kotlin/ftl/shard/Shard.kt +++ b/test_runner/src/main/kotlin/ftl/shard/Shard.kt @@ -6,6 +6,7 @@ import ftl.args.IArgs import ftl.args.IosArgs import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestResult +import ftl.util.FlankTestMethod import ftl.util.Utils.fatalError import kotlin.math.ceil import kotlin.math.min @@ -47,7 +48,8 @@ class com.foo.ClassName#testMethodToSkip object Shard { // When a test does not have previous results to reference, fall back to this run time. @VisibleForTesting - const val DEFAULT_TEST_TIME_SEC: Double = 120.0 + const val DEFAULT_TEST_TIME_SEC = 120.0 + private const val IGNORE_TEST_TIME = 0.0 private fun JUnitTestCase.androidKey(): String { return "class $classname#$name" @@ -62,7 +64,7 @@ object Shard { // take in the XML with timing info then return the shard count based on execution time fun shardCountByTime( - testsToRun: List, + testsToRun: List, oldTestResult: JUnitTestResult, args: IArgs ): Int { @@ -70,7 +72,7 @@ object Shard { if (args.shardTime < -1 || args.shardTime == 0) fatalError("Invalid shard time ${args.shardTime}") val oldDurations = createTestMethodDurationMap(oldTestResult, args) - val testsTotalTime = testsToRun.sumByDouble { oldDurations[it] ?: DEFAULT_TEST_TIME_SEC } + val testsTotalTime = testsToRun.sumByDouble { if (it.ignored) IGNORE_TEST_TIME else oldDurations[it.testName] ?: DEFAULT_TEST_TIME_SEC } val shardsByTime = ceil(testsTotalTime / args.shardTime).toInt() @@ -93,7 +95,7 @@ object Shard { // take in the XML with timing info then return list of shards based on the amount of shards to use fun createShardsByShardCount( - testsToRun: List, + testsToRun: List, oldTestResult: JUnitTestResult, args: IArgs, forcedShardCount: Int = -1 @@ -104,11 +106,11 @@ object Shard { val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args) var cacheMiss = 0 - val testcases: List = testsToRun - .map { methodName -> + val testCases: List = testsToRun + .map { TestMethod( - name = methodName, - time = previousMethodDurations[methodName] ?: DEFAULT_TEST_TIME_SEC.also { + name = it.testName, + time = if (it.ignored) IGNORE_TEST_TIME else previousMethodDurations[it.testName] ?: DEFAULT_TEST_TIME_SEC.also { cacheMiss += 1 } ) @@ -116,7 +118,7 @@ object Shard { // We want to iterate over testcase going from slowest to fastest .sortedByDescending(TestMethod::time) - val testCount = testcases.size + val testCount = testCases.size // If maxShards is infinite or we have more shards than tests, let's match it val shardsCount = if (maxShards == -1 || maxShards > testCount) testCount else maxShards @@ -135,7 +137,7 @@ object Shard { } var shards = List(shardsCount) { TestShard(0.0, mutableListOf()) } - testcases.forEach { testMethod -> + testCases.forEach { testMethod -> val shard = shards.first() shard.testMethods.add(testMethod) diff --git a/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt b/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt new file mode 100644 index 0000000000..b35fe7162b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/util/FlankTestMethod.kt @@ -0,0 +1,3 @@ +package ftl.util + +data class FlankTestMethod(val testName: String, val ignored: Boolean = false) diff --git a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt index a95296c1ca..909c47a0ca 100644 --- a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt +++ b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt @@ -6,6 +6,7 @@ import com.linkedin.dex.parser.TestMethod import ftl.filter.TestFilters.fromTestTargets import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper +import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith @@ -14,31 +15,56 @@ val BAR_PACKAGE = TestMethod("bar.ClassName#testName", emptyList()) val FOO_CLASSNAME = TestMethod("whatever.Foo#testName", emptyList()) val BAR_CLASSNAME = TestMethod("whatever.Bar#testName", emptyList()) val WITHOUT_IGNORE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) -val WITH_IGNORE_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("org.junit.Ignore", emptyMap(), false))) +val WITH_IGNORE_ANNOTATION = + TestMethod("whatever.Foo#testName", listOf(TestAnnotation("org.junit.Ignore", emptyMap(), false))) val WITH_FOO_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("Foo", emptyMap(), false))) val WITH_BAR_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("Bar", emptyMap(), false))) val WITHOUT_FOO_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITH_FOO_ANNOTATION_AND_PACKAGE = TestMethod("foo.Bar#testName", listOf(TestAnnotation("Foo", emptyMap(), false))) val WITH_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("LargeTest", emptyMap(), false))) -val WITH_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("MediumTest", emptyMap(), false))) +val WITH_MEDIUM_ANNOTATION = + TestMethod("whatever.Foo#testName", listOf(TestAnnotation("MediumTest", emptyMap(), false))) val WITH_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", listOf(TestAnnotation("SmallTest", emptyMap(), false))) val WITHOUT_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) const val TEST_FILE = "src/test/kotlin/ftl/filter/fixtures/dummy-tests-file.txt" +private const val IGNORE_ANNOTATION = "org.junit.Ignore" @RunWith(FlankTestRunner::class) class TestFiltersTest { + private val targets = listOf( + TargetsHelper( + pack = "anyPackage_1", + cl = "anyClass_1", + m = "anyMethod_1", + annotation = IGNORE_ANNOTATION + ), + TargetsHelper(pack = "anyPackage_2", cl = "anyClass_2", m = "anyMethod_2", annotation = "Foo"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_3", m = "anyMethod_3", annotation = "Bar"), + TargetsHelper( + pack = "anyPackage_4", + cl = "anyClass_4", + m = "anyMethod_4", + annotation = IGNORE_ANNOTATION + ) + ) + + private val testMethodSet = targets.map { getDefaultTestMethod(it.fullView, it.annotation) } + private val commonExpected = targets.map { it.fullView } + @Test fun testIgnoreMultipleAnnotations() { - val m1 = TestMethod("com.example.app.ExampleUiTest#testFails", listOf( - TestAnnotation("org.junit.runner.RunWith", emptyMap(), false), - TestAnnotation("org.junit.Ignore", emptyMap(), false), - TestAnnotation("org.junit.Test", emptyMap(), false) - )) + val m1 = TestMethod( + "com.example.app.ExampleUiTest#testFails", listOf( + TestAnnotation("org.junit.runner.RunWith", emptyMap(), false), + TestAnnotation("org.junit.Ignore", emptyMap(), false), + TestAnnotation("org.junit.Test", emptyMap(), false) + ) + ) - val filter = fromTestTargets(listOf("class com.example.app.ExampleUiTest#testFails")) + val filter = fromTestTargets(listOf("notAnnotation org.junit.Ignore")) assertThat(filter.shouldRun(m1)).isFalse() } @@ -76,10 +102,10 @@ class TestFiltersTest { } @Test - fun emptyTargetsShouldFilterTestsWithTheIgnoreAnnotation() { + fun `empty targets should not filter @Ignore annotated tests`() { val filter = fromTestTargets(listOf()) - assertThat(filter.shouldRun(WITH_IGNORE_ANNOTATION)).isFalse() + assertThat(filter.shouldRun(WITH_IGNORE_ANNOTATION)).isTrue() assertThat(filter.shouldRun(WITHOUT_IGNORE_ANNOTATION)).isTrue() } @@ -165,7 +191,7 @@ class TestFiltersTest { @Test fun allOfProperlyChecksAllFilters() { - val filter = TestFilters.fromTestTargets(listOf("package foo,bar", "annotation Foo")) + val filter = fromTestTargets(listOf("package foo,bar", "annotation Foo")) assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() assertThat(filter.shouldRun(BAR_PACKAGE)).isFalse() @@ -205,77 +231,127 @@ class TestFiltersTest { fromTestTargets(listOf("invalidCommand com.my.package")) } - private fun getTestMethodSet(): List { - val m1 = TestMethod("a.b#c", listOf(TestAnnotation("org.junit.Ignore", emptyMap(), false))) - val m2 = TestMethod("d.e#f", listOf(TestAnnotation("Foo", emptyMap(), false))) - val m3 = TestMethod("h.i#j", listOf(TestAnnotation("Bar", emptyMap(), false))) - return listOf(m1, m2, m3) - } - @Test fun classFilterOverridesNotAnnotation() { - val testMethods = getTestMethodSet() - val filter = fromTestTargets(listOf("notAnnotation Foo", "class d.e#f", "class h.i#j")) + val filter = fromTestTargets( + listOf( + "notAnnotation Foo", + "class anyPackage_2.anyClass_2#anyMethod_2", + "class anyPackage_3.anyClass_3#anyMethod_3" + ) + ) val output = mutableListOf() - val filtered = testMethods.asSequence().filter { test -> + val filtered = testMethodSet.asSequence().filter { test -> val result = filter.shouldRun(test) output.add("""$result ${test.testName} [${filter.describe}]""") result }.map { "class ${it.testName}" }.toList() - // @Ignore a.b#c - // @Foo d.e#f - // @Bar h.i#j val expected = listOf( - "false a.b#c [allOf [notIgnored, not withAnnotation Foo, anyOf [withClassName d.e#f, withClassName h.i#j]]]", - "false d.e#f [allOf [notIgnored, not withAnnotation Foo, anyOf [withClassName d.e#f, withClassName h.i#j]]]", - "true h.i#j [allOf [notIgnored, not withAnnotation Foo, anyOf [withClassName d.e#f, withClassName h.i#j]]]" + "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", + "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", + "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", + "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]" ) assertThat(output).isEqualTo(expected) - assertThat(filtered).isEqualTo(listOf("class h.i#j")) + assertThat(filtered).isEqualTo(listOf("class anyPackage_3.anyClass_3#anyMethod_3")) } @Test fun notAnnotationFiltersWithClass() { - val testMethods = getTestMethodSet() - val filter = fromTestTargets(listOf("notAnnotation Foo", "class h.i#j")) + val filter = fromTestTargets(listOf("notAnnotation Foo", "class anyPackage_1.anyClass_1#anyMethod_1")) + val filtered = testMethodSet.withFilter(filter) - val filtered = testMethods.asSequence().filter(filter.shouldRun).map { it.testName }.toList() - assertThat(filtered).isEqualTo(listOf("h.i#j")) + assertEquals(listOf("anyPackage_1.anyClass_1#anyMethod_1"), filtered) } @Test fun notAnnotationFilters() { - val testMethods = getTestMethodSet() val filter = fromTestTargets(listOf("notAnnotation Moo")) + val filtered = testMethodSet.withFilter(filter, enrich = false) - val filtered = testMethods.asSequence().filter(filter.shouldRun).map { it.testName }.toList() - assertThat(filtered).isEqualTo(listOf("d.e#f", "h.i#j")) + assertEquals(commonExpected, filtered) } @Test fun methodOverrideIgnored() { - val filter = fromTestTargets(listOf("class a.b#c", "class d.e#f", "class h.i#j")) + val filter = fromTestTargets(targets.map { it.methodView }) + val filtered = testMethodSet.withFilter(filter) - val filtered = getTestMethodSet().asSequence().filter(filter.shouldRun).map { it.testName }.toList() - assertThat(filtered).isEqualTo(listOf("d.e#f", "h.i#j")) + assertEquals(commonExpected, filtered) } @Test fun multipleClassesResolveToMethods() { - val filter = fromTestTargets(listOf("class a.b", "class d.e", "class h.i")) + val filter = fromTestTargets(targets.map { it.classView }) + val filtered = testMethodSet.withFilter(filter) - val filtered = getTestMethodSet().asSequence().filter(filter.shouldRun).map { it.testName }.toList() - assertThat(filtered).isEqualTo(listOf("d.e#f", "h.i#j")) + assertEquals(commonExpected, filtered) } @Test fun multiplePackagesResolveToMethods() { - val filter = fromTestTargets(listOf("package a", "package d", "package h")) + val filter = fromTestTargets(targets.map { it.packageView }) + val filtered = testMethodSet.withFilter(filter) + + assertEquals(commonExpected, filtered) + } + + @Test + fun `by default - should contain method annotated with @Ignore`() { + val byPackageFilter = fromTestTargets(targets.map { it.packageView }) + val byClassFilter = fromTestTargets(targets.map { it.classView }) + val byNotAnnotationFilter = fromTestTargets(listOf("notAnnotation ThereIsNoSuchAnnotation")) + + val byPackage = testMethodSet.withFilter(byPackageFilter) + val byClass = testMethodSet.withFilter(byClassFilter) + val byNotAnnotation = testMethodSet.withFilter(byNotAnnotationFilter, enrich = false) + + assertEquals(commonExpected, byPackage) + assertEquals(commonExpected, byClass) + assertEquals(commonExpected, byNotAnnotation) + } + + @Test + fun `should filter tests annotated with @Ignore if user explicitly want to do so`() { + val byNotAnnotationFilter = fromTestTargets(listOf("notAnnotation $IGNORE_ANNOTATION")) + val byNotAnnotation = testMethodSet.withFilter(byNotAnnotationFilter, enrich = false) + val expected = targets.filterNot { it.annotation == IGNORE_ANNOTATION }.map { it.fullView } - val filtered = getTestMethodSet().asSequence().filter(filter.shouldRun).map { it.testName }.toList() - assertThat(filtered).isEqualTo(listOf("d.e#f", "h.i#j")) + assertEquals(byNotAnnotation, expected) } } + +private fun getDefaultTestMethod(testName: String, annotation: String) = + TestMethod(testName, listOf(TestAnnotation(annotation, emptyMap(), false))) + +private fun List.add(method: TestMethod) = listOf(*this.toTypedArray() + method) + +private fun List.withFilter(filter: TestFilter, enrich: Boolean = true) = + if (enrich) + add(getDefaultTestMethod("should.be#filtered", "AnyAnnotation")) + .filter(filter.shouldRun) + .map { it.testName } + else + filter(filter.shouldRun).map { it.testName } + +private class TargetsHelper( + private val pack: String, + private val cl: String, + private val m: String, + val annotation: String +) { + val classView: String + get() = "class $pack.$cl" + + val packageView: String + get() = "package $pack" + + val methodView: String + get() = "class $fullView" + + val fullView: String + get() = "$pack.$cl#$m" +} diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt index 715541b93b..4a8ebcc3e1 100644 --- a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt @@ -7,6 +7,9 @@ import ftl.reports.xml.model.JUnitTestCase import ftl.reports.xml.model.JUnitTestResult import ftl.reports.xml.model.JUnitTestSuite import ftl.test.util.FlankTestRunner +import ftl.util.FlankTestMethod +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test import org.junit.rules.ExpectedException @@ -19,7 +22,8 @@ import kotlin.system.measureNanoTime @RunWith(FlankTestRunner::class) class ShardTest { - @Rule @JvmField + @Rule + @JvmField val exceptionRule = ExpectedException.none()!! private fun sample(): JUnitTestResult { @@ -50,7 +54,7 @@ class ShardTest { @Test fun oneTestPerShard() { - val reRunTestsToRun = listOf("a", "b", "c", "d", "e", "f", "g") + val reRunTestsToRun = listOfFlankTestMethod("a", "b", "c", "d", "e", "f", "g") val suite = sample() val result = Shard.createShardsByShardCount(reRunTestsToRun, suite, mockArgs(100)) @@ -63,7 +67,7 @@ class ShardTest { @Test fun sampleTest() { - val reRunTestsToRun = listOf("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val reRunTestsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") val suite = sample() val result = Shard.createShardsByShardCount(reRunTestsToRun, suite, mockArgs(3)) @@ -87,7 +91,7 @@ class ShardTest { @Test fun firstRun() { - val testsToRun = listOf("a", "b", "c") + val testsToRun = listOfFlankTestMethod("a", "b", "c") val result = Shard.createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(2)) assertThat(result.size).isEqualTo(2) @@ -100,7 +104,7 @@ class ShardTest { @Test fun mixedNewAndOld() { - val testsToRun = listOf("a/a", "b/b", "c/c", "w", "y", "z") + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "w", "y", "z") val result = Shard.createShardsByShardCount(testsToRun, sample(), mockArgs(4)) assertThat(result.size).isEqualTo(4) assertThat(result.sumByDouble { it.time }).isEqualTo(7.0 + 3 * Shard.DEFAULT_TEST_TIME_SEC) @@ -115,8 +119,8 @@ class ShardTest { @Test fun `performance calculateShardsByTime`() { - val testsToRun = mutableListOf() - repeat(1_000_000) { index -> testsToRun.add("$index/$index") } + val testsToRun = mutableListOf() + repeat(1_000_000) { index -> testsToRun.add(FlankTestMethod("$index/$index")) } val nano = measureNanoTime { Shard.createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(4)) @@ -129,7 +133,7 @@ class ShardTest { @Test fun `createShardsByShardTime workingSample`() { - val testsToRun = listOf("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") val suite = sample() val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(20, 7)) @@ -138,7 +142,7 @@ class ShardTest { @Test fun `createShardsByShardTime countShouldNeverBeHigherThanMaxAvailable`() { - val testsToRun = listOf("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") val suite = sample() val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(2, 7)) @@ -147,7 +151,7 @@ class ShardTest { @Test fun `createShardsByShardTime unlimitedShardsShouldReturnTheRightAmount`() { - val testsToRun = listOf("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c", "d/d", "e/e", "f/f", "g/g") val suite = sample() val result = Shard.shardCountByTime(testsToRun, suite, mockArgs(-1, 7)) @@ -156,7 +160,7 @@ class ShardTest { @Test fun `createShardsByShardTime uncachedTestResultsUseDefaultTime`() { - val testsToRun = listOf("h/h", "i/i", "j/j") + val testsToRun = listOfFlankTestMethod("h/h", "i/i", "j/j") val suite = sample() val result = Shard.shardCountByTime( testsToRun, @@ -169,7 +173,7 @@ class ShardTest { @Test fun `createShardsByShardTime mixedCachedAndUncachedTestResultsUseDefaultTime`() { // Test "a/a" is hard-coded to have 1.0 second run time in test suite results. - val testsToRun = listOf("a/a", "i/i", "j/j") + val testsToRun = listOfFlankTestMethod("a/a", "i/i", "j/j") val suite = sample() val result = Shard.shardCountByTime( testsToRun, @@ -181,7 +185,7 @@ class ShardTest { @Test fun `createShardsByShardTime uncachedTestResultsAllInOneShard`() { - val testsToRun = listOf("i/i", "j/j") + val testsToRun = listOfFlankTestMethod("i/i", "j/j") val suite = sample() val result = Shard.shardCountByTime( testsToRun, @@ -218,7 +222,7 @@ class ShardTest { } private fun shardCountByTime(shardTime: Int): Int { - val testsToRun = listOf("a/a", "b/b", "c/c") + val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c") val testCases = mutableListOf( JUnitTestCase("a", "a", "0.001"), JUnitTestCase("b", "b", "0.0"), @@ -255,4 +259,38 @@ class ShardTest { assertThat(shardCountByTime(2)).isEqualTo(1) assertThat(shardCountByTime(3)).isEqualTo(1) } + + @Test(expected = RuntimeException::class) + fun `should terminate with exit status == 3 test targets is 0 and maxTestShards == -1`() { + Shard.createShardsByShardCount(emptyList(), JUnitTestResult(mutableListOf()), mockArgs(-1)) + } + + @Test + fun `tests annotated with @Ignore should have time 0 and do not hav impact on sharding`() { + val androidMockedArgs = mock(IosArgs::class.java) + `when`(androidMockedArgs.maxTestShards).thenReturn(2) + `when`(androidMockedArgs.shardTime).thenReturn(10) + + val testsToRun = listOf( + FlankTestMethod("a/a", ignored = true), + FlankTestMethod("b/b"), + FlankTestMethod("c/c") + ) + + val oldTestResult = newSuite(mutableListOf( + JUnitTestCase("a", "a", "5.0"), + JUnitTestCase("b", "b", "10.0"), + JUnitTestCase("c", "c", "10.0") + )) + + val shardCount = Shard.shardCountByTime(testsToRun, oldTestResult, androidMockedArgs) + assertEquals(2, shardCount) + + val shards = Shard.createShardsByShardCount(testsToRun, oldTestResult, androidMockedArgs, shardCount) + assertEquals(2, shards.size) + assertTrue(shards.flatMap { it.testMethods }.map { it.time }.filter { it == 0.0 }.count() == 1) + shards.forEach { assertEquals(10.0, it.time, 0.0) } + } } + +private fun listOfFlankTestMethod(vararg args: String) = listOf(*args).map { FlankTestMethod(it) }