diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt index f53e5f55ae..44ee9f84a5 100644 --- a/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt @@ -4,10 +4,13 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import org.junit.Test import org.junit.runner.RunWith +annotation class Annotation + @RunWith(AndroidJUnit4::class) class InstrumentedTest : BaseInstrumentedTest() { @Test + @Annotation fun test0() = testMethod() @Test diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt new file mode 100644 index 0000000000..d53e8b7be0 --- /dev/null +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt @@ -0,0 +1,13 @@ +package com.example.test_app.bar + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.test_app.BaseInstrumentedTest +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class BarInstrumentedTest : BaseInstrumentedTest() { + + @Test + fun testBar() = testMethod() +} diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt new file mode 100644 index 0000000000..5123161ba4 --- /dev/null +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt @@ -0,0 +1,13 @@ +package com.example.test_app.foo + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.test_app.BaseInstrumentedTest +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class FooInstrumentedTest : BaseInstrumentedTest() { + + @Test + fun testFoo() = testMethod() +} diff --git a/test_app/bash/test_filters.sh b/test_app/bash/test_filters.sh new file mode 100755 index 0000000000..c211467921 --- /dev/null +++ b/test_app/bash/test_filters.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Helper functions for testing AndroidJUnitRunner in action. +# Can be helpful for checking if flank test filters match AndroidJUnitRunner policy. + +PACKAGE="com.example.test_app" +PACKAGE_FOO="com.example.test_app.foo" +PACKAGE_BAR="com.example.test_app.bar" + +ANNOTATION="${PACKAGE}.Annotation" + +CLASS="${PACKAGE}.InstrumentedTest" +CLASS_FOO="${PACKAGE_FOO}.FooInstrumentedTest" +CLASS_BAR="${PACKAGE_BAR}.BarInstrumentedTest" + +METHOD_1="${CLASS}#test1" +METHOD_FOO="${CLASS_FOO}#testFoo" +METHOD_BAR="${CLASS_BAR}#testBar" + +RUNNER=com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner + +set -euxo pipefail + +# should run all tests +function run_instrument { + adb shell am instrument -r -w $@ ${RUNNER} +} + +# should run only method 1, last inclusion filter overrides the rest +function filter_package_bar_class_foo_method_1 { + run_instrument -e package ${PACKAGE_BAR} -e class ${CLASS_FOO} -e class ${METHOD_1} +} + +# should run nothing because of annotation intersect with other filters +function filter_annotation_method_foo { + run_instrument -e annotation ${ANNOTATION} -e class ${METHOD_FOO} +} + +# should exclude both +function filter_notPackage_foo_notClass_bar { + run_instrument -e notPackage ${PACKAGE_FOO} -e notClass ${CLASS_BAR} +} \ No newline at end of file diff --git a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt index 14ffb79561..d88866a1ac 100644 --- a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt +++ b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt @@ -81,9 +81,11 @@ object TestFilters { // select test method name filters and short circuit if they match ex: class a.b#c val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray() - val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray() + val otherFilters = parsedFilters.filterNot { it.isAnnotation } + val exclude = otherFilters.filter { it.describe.startsWith("not") }.toTypedArray() + val include = otherFilters.filterNot { it.describe.startsWith("not") }.toTypedArray() - val result = allOf(*annotationFilters, anyOf(*otherFilters)) + val result = allOf(*annotationFilters, *exclude, anyOf(*include)) if (FtlConstants.useMock) println(result.describe) return result } @@ -128,21 +130,23 @@ object TestFilters { } private fun withPackageName(packageNames: List): TestFilter = TestFilter( - describe = "withPackageName ${packageNames.joinToString(", ")}", + describe = "withPackageName (${packageNames.joinToString(", ")})", shouldRun = { testMethod -> - packageNames.any { packageName -> testMethod.testName.startsWith(packageName) } + packageNames.any { packageName -> + testMethod.testName.startsWith(packageName) + } } ) private fun withClassName(classNames: List): TestFilter = TestFilter( - describe = "withClassName ${classNames.joinToString(", ")}", + describe = "withClassName (${classNames.joinToString(", ")})", shouldRun = { testMethod -> withPackageName(classNames).shouldRun(testMethod) } ) private fun withAnnotation(annotations: List): TestFilter = TestFilter( - describe = "withAnnotation ${annotations.joinToString(", ")}", + describe = "withAnnotation (${annotations.joinToString(", ")})", shouldRun = { testMethod -> testMethod.annotationNames.any { annotations.contains(it) } }, @@ -150,7 +154,7 @@ object TestFilters { ) private fun not(filter: TestFilter): TestFilter = TestFilter( - describe = "not ${filter.describe}", + describe = "not (${filter.describe})", shouldRun = { testMethod -> filter.shouldRun(testMethod).not() }, @@ -169,9 +173,9 @@ object TestFilters { shouldRun = { testMethod -> if (FtlConstants.useMock) println(":: ${testMethod.testName} @${testMethod.annotations.firstOrNull()}") filters.isEmpty() || filters.all { filter -> - val result = filter.shouldRun(testMethod) - if (FtlConstants.useMock) println(" $result ${filter.describe}") - result + filter.shouldRun(testMethod).also { result -> + if (FtlConstants.useMock) println(" $result ${filter.describe}") + } } } ) diff --git a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt index 909c47a0ca..ae8a874281 100644 --- a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt +++ b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt @@ -29,8 +29,10 @@ val WITHOUT_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) const val TEST_FILE = "src/test/kotlin/ftl/filter/fixtures/dummy-tests-file.txt" +const val TEST_FILE_2 = "src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt" private const val IGNORE_ANNOTATION = "org.junit.Ignore" +@Suppress("TooManyFunctions") @RunWith(FlankTestRunner::class) class TestFiltersTest { @@ -249,10 +251,10 @@ class TestFiltersTest { }.map { "class ${it.testName}" }.toList() val expected = listOf( - "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]" + "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]" ) assertThat(output).isEqualTo(expected) @@ -322,6 +324,36 @@ class TestFiltersTest { assertEquals(byNotAnnotation, expected) } + + @Test + fun testFilteringClassAndPackageNegative() { + val filter = fromTestTargets(listOf("notPackage foo", "notClass whatever.Bar")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue() + } + + @Test + fun testFilteringClassAndPackageNegativeFromFile() { + val file = TestHelper.getPath(TEST_FILE_2) // contents: foo whatever.Bar + val filePath = file.toString() + + val filter = fromTestTargets(listOf("notTestFile $filePath")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue() + } + + @Test + fun `inclusion filter should override exclusion filter`() { + val filter = fromTestTargets(listOf("notPackage foo", "class whatever.Bar")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isTrue() + } } private fun getDefaultTestMethod(testName: String, annotation: String) = diff --git a/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt b/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt new file mode 100644 index 0000000000..bda41fba2f --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt @@ -0,0 +1,2 @@ +foo +whatever.Bar \ No newline at end of file