From 347ad98c9f25ffa841a2063b7b6bf2e8bfd92a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 5 Mar 2020 21:39:34 +0100 Subject: [PATCH 1/3] Fix test filters when using both notPackage and notClass #460 --- .../src/main/kotlin/ftl/filter/TestFilters.kt | 16 ++++++++----- .../test/kotlin/ftl/filter/TestFiltersTest.kt | 23 +++++++++++++++++++ .../ftl/filter/fixtures/exclude-tests.txt | 2 ++ 3 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt diff --git a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt index 14ffb79561..2702531403 100644 --- a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt +++ b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt @@ -81,9 +81,11 @@ object TestFilters { // select test method name filters and short circuit if they match ex: class a.b#c val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray() - val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray() + val otherFilters = parsedFilters.filterNot { it.isAnnotation } + val exclude = otherFilters.filter { it.describe.startsWith("not") }.toTypedArray() + val include = otherFilters.filterNot { it.describe.startsWith("not") }.toTypedArray() - val result = allOf(*annotationFilters, anyOf(*otherFilters)) + val result = allOf(*annotationFilters, *exclude, anyOf(*include)) if (FtlConstants.useMock) println(result.describe) return result } @@ -130,7 +132,9 @@ object TestFilters { private fun withPackageName(packageNames: List): TestFilter = TestFilter( describe = "withPackageName ${packageNames.joinToString(", ")}", shouldRun = { testMethod -> - packageNames.any { packageName -> testMethod.testName.startsWith(packageName) } + packageNames.any { packageName -> + testMethod.testName.startsWith(packageName) + } } ) @@ -169,9 +173,9 @@ object TestFilters { shouldRun = { testMethod -> if (FtlConstants.useMock) println(":: ${testMethod.testName} @${testMethod.annotations.firstOrNull()}") filters.isEmpty() || filters.all { filter -> - val result = filter.shouldRun(testMethod) - if (FtlConstants.useMock) println(" $result ${filter.describe}") - result + filter.shouldRun(testMethod).also { result -> + if (FtlConstants.useMock) println(" $result ${filter.describe}") + } } } ) diff --git a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt index 909c47a0ca..05570af5a2 100644 --- a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt +++ b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt @@ -29,8 +29,10 @@ val WITHOUT_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) val WITHOUT_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList()) const val TEST_FILE = "src/test/kotlin/ftl/filter/fixtures/dummy-tests-file.txt" +const val TEST_FILE_2 = "src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt" private const val IGNORE_ANNOTATION = "org.junit.Ignore" +@Suppress("TooManyFunctions") @RunWith(FlankTestRunner::class) class TestFiltersTest { @@ -322,6 +324,27 @@ class TestFiltersTest { assertEquals(byNotAnnotation, expected) } + + @Test + fun testFilteringClassAndPackageNegative() { + val filter = fromTestTargets(listOf("notPackage foo", "notClass whatever.Bar")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue() + } + + @Test + fun testFilteringClassAndPackageNegativeFromFile() { + val file = TestHelper.getPath(TEST_FILE_2) // contents: foo whatever.Bar + val filePath = file.toString() + + val filter = fromTestTargets(listOf("notTestFile $filePath")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue() + } } private fun getDefaultTestMethod(testName: String, annotation: String) = diff --git a/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt b/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt new file mode 100644 index 0000000000..bda41fba2f --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt @@ -0,0 +1,2 @@ +foo +whatever.Bar \ No newline at end of file From d93a143b1eb2b72b194d5fb40e380150401a4052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 6 Mar 2020 16:33:06 +0100 Subject: [PATCH 2/3] Improve TestFilters description readability --- test_runner/src/main/kotlin/ftl/filter/TestFilters.kt | 8 ++++---- test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt index 2702531403..d88866a1ac 100644 --- a/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt +++ b/test_runner/src/main/kotlin/ftl/filter/TestFilters.kt @@ -130,7 +130,7 @@ object TestFilters { } private fun withPackageName(packageNames: List): TestFilter = TestFilter( - describe = "withPackageName ${packageNames.joinToString(", ")}", + describe = "withPackageName (${packageNames.joinToString(", ")})", shouldRun = { testMethod -> packageNames.any { packageName -> testMethod.testName.startsWith(packageName) @@ -139,14 +139,14 @@ object TestFilters { ) private fun withClassName(classNames: List): TestFilter = TestFilter( - describe = "withClassName ${classNames.joinToString(", ")}", + describe = "withClassName (${classNames.joinToString(", ")})", shouldRun = { testMethod -> withPackageName(classNames).shouldRun(testMethod) } ) private fun withAnnotation(annotations: List): TestFilter = TestFilter( - describe = "withAnnotation ${annotations.joinToString(", ")}", + describe = "withAnnotation (${annotations.joinToString(", ")})", shouldRun = { testMethod -> testMethod.annotationNames.any { annotations.contains(it) } }, @@ -154,7 +154,7 @@ object TestFilters { ) private fun not(filter: TestFilter): TestFilter = TestFilter( - describe = "not ${filter.describe}", + describe = "not (${filter.describe})", shouldRun = { testMethod -> filter.shouldRun(testMethod).not() }, diff --git a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt index 05570af5a2..eba972b763 100644 --- a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt +++ b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt @@ -251,10 +251,10 @@ class TestFiltersTest { }.map { "class ${it.testName}" }.toList() val expected = listOf( - "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]", - "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]" + "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]" ) assertThat(output).isEqualTo(expected) From 128e8357a890990eab89e6ae568bc8f2f6a6c9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sat, 7 Mar 2020 00:53:07 +0100 Subject: [PATCH 3/3] Check if inclusion filter overrides exclusion filter --- .../com.example.test_app/InstrumentedTest.kt | 3 ++ .../bar/BarInstrumentedTest.kt | 13 ++++++ .../foo/FooInstrumentedTest.kt | 13 ++++++ test_app/bash/test_filters.sh | 42 +++++++++++++++++++ .../test/kotlin/ftl/filter/TestFiltersTest.kt | 9 ++++ 5 files changed, 80 insertions(+) create mode 100644 test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt create mode 100644 test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt create mode 100755 test_app/bash/test_filters.sh diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt index f53e5f55ae..44ee9f84a5 100644 --- a/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt @@ -4,10 +4,13 @@ import androidx.test.ext.junit.runners.AndroidJUnit4 import org.junit.Test import org.junit.runner.RunWith +annotation class Annotation + @RunWith(AndroidJUnit4::class) class InstrumentedTest : BaseInstrumentedTest() { @Test + @Annotation fun test0() = testMethod() @Test diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt new file mode 100644 index 0000000000..d53e8b7be0 --- /dev/null +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt @@ -0,0 +1,13 @@ +package com.example.test_app.bar + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.test_app.BaseInstrumentedTest +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class BarInstrumentedTest : BaseInstrumentedTest() { + + @Test + fun testBar() = testMethod() +} diff --git a/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt b/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt new file mode 100644 index 0000000000..5123161ba4 --- /dev/null +++ b/test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt @@ -0,0 +1,13 @@ +package com.example.test_app.foo + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.test_app.BaseInstrumentedTest +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class FooInstrumentedTest : BaseInstrumentedTest() { + + @Test + fun testFoo() = testMethod() +} diff --git a/test_app/bash/test_filters.sh b/test_app/bash/test_filters.sh new file mode 100755 index 0000000000..c211467921 --- /dev/null +++ b/test_app/bash/test_filters.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Helper functions for testing AndroidJUnitRunner in action. +# Can be helpful for checking if flank test filters match AndroidJUnitRunner policy. + +PACKAGE="com.example.test_app" +PACKAGE_FOO="com.example.test_app.foo" +PACKAGE_BAR="com.example.test_app.bar" + +ANNOTATION="${PACKAGE}.Annotation" + +CLASS="${PACKAGE}.InstrumentedTest" +CLASS_FOO="${PACKAGE_FOO}.FooInstrumentedTest" +CLASS_BAR="${PACKAGE_BAR}.BarInstrumentedTest" + +METHOD_1="${CLASS}#test1" +METHOD_FOO="${CLASS_FOO}#testFoo" +METHOD_BAR="${CLASS_BAR}#testBar" + +RUNNER=com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner + +set -euxo pipefail + +# should run all tests +function run_instrument { + adb shell am instrument -r -w $@ ${RUNNER} +} + +# should run only method 1, last inclusion filter overrides the rest +function filter_package_bar_class_foo_method_1 { + run_instrument -e package ${PACKAGE_BAR} -e class ${CLASS_FOO} -e class ${METHOD_1} +} + +# should run nothing because of annotation intersect with other filters +function filter_annotation_method_foo { + run_instrument -e annotation ${ANNOTATION} -e class ${METHOD_FOO} +} + +# should exclude both +function filter_notPackage_foo_notClass_bar { + run_instrument -e notPackage ${PACKAGE_FOO} -e notClass ${CLASS_BAR} +} \ No newline at end of file diff --git a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt index eba972b763..ae8a874281 100644 --- a/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt +++ b/test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt @@ -345,6 +345,15 @@ class TestFiltersTest { assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse() assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue() } + + @Test + fun `inclusion filter should override exclusion filter`() { + val filter = fromTestTargets(listOf("notPackage foo", "class whatever.Bar")) + + assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_PACKAGE)).isFalse() + assertThat(filter.shouldRun(BAR_CLASSNAME)).isTrue() + } } private fun getDefaultTestMethod(testName: String, annotation: String) =