Fix test filters when using both notPackage and notClass

test_app/app/src/androidTestMultiple/java/com.example.test_app/InstrumentedTest.kt

@@ -4,10 +4,13 @@ import androidx.test.ext.junit.runners.AndroidJUnit4
 import org.junit.Test
 import org.junit.runner.RunWith
 
+annotation class Annotation
+
 @RunWith(AndroidJUnit4::class)
 class InstrumentedTest : BaseInstrumentedTest() {
 
     @Test
+    @Annotation
     fun test0() = testMethod()
 
     @Test

test_app/app/src/androidTestMultiple/java/com.example.test_app/bar/BarInstrumentedTest.kt

@@ -0,0 +1,13 @@
+package com.example.test_app.bar
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.example.test_app.BaseInstrumentedTest
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class BarInstrumentedTest : BaseInstrumentedTest() {
+
+    @Test
+    fun testBar() = testMethod()
+}

test_app/app/src/androidTestMultiple/java/com.example.test_app/foo/FooInstrumentedTest.kt

@@ -0,0 +1,13 @@
+package com.example.test_app.foo
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.example.test_app.BaseInstrumentedTest
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class FooInstrumentedTest : BaseInstrumentedTest() {
+
+    @Test
+    fun testFoo() = testMethod()
+}

test_app/bash/test_filters.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Helper functions for testing AndroidJUnitRunner in action.
+# Can be helpful for checking if flank test filters match AndroidJUnitRunner policy.
+
+PACKAGE="com.example.test_app"
+PACKAGE_FOO="com.example.test_app.foo"
+PACKAGE_BAR="com.example.test_app.bar"
+
+ANNOTATION="${PACKAGE}.Annotation"
+
+CLASS="${PACKAGE}.InstrumentedTest"
+CLASS_FOO="${PACKAGE_FOO}.FooInstrumentedTest"
+CLASS_BAR="${PACKAGE_BAR}.BarInstrumentedTest"
+
+METHOD_1="${CLASS}#test1"
+METHOD_FOO="${CLASS_FOO}#testFoo"
+METHOD_BAR="${CLASS_BAR}#testBar"
+
+RUNNER=com.example.test_app.test/androidx.test.runner.AndroidJUnitRunner
+
+set -euxo pipefail
+
+# should run all tests
+function run_instrument {
+    adb shell am instrument -r -w $@ ${RUNNER}
+}
+
+# should run only method 1, last inclusion filter overrides the rest
+function filter_package_bar_class_foo_method_1 {
+    run_instrument -e package ${PACKAGE_BAR} -e class ${CLASS_FOO} -e class ${METHOD_1}
+}
+
+# should run nothing because of annotation intersect with other filters
+function filter_annotation_method_foo {
+    run_instrument -e annotation ${ANNOTATION} -e class ${METHOD_FOO}
+}
+
+# should exclude both
+function filter_notPackage_foo_notClass_bar {
+    run_instrument -e notPackage ${PACKAGE_FOO} -e notClass ${CLASS_BAR}
+}

test_runner/src/main/kotlin/ftl/filter/TestFilters.kt

@@ -81,9 +81,11 @@ object TestFilters {
 
         // select test method name filters and short circuit if they match ex: class a.b#c
         val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray()
-        val otherFilters = parsedFilters.filterNot { it.isAnnotation }.toTypedArray()
+        val otherFilters = parsedFilters.filterNot { it.isAnnotation }
+        val exclude = otherFilters.filter { it.describe.startsWith("not") }.toTypedArray()
+        val include = otherFilters.filterNot { it.describe.startsWith("not") }.toTypedArray()
 
-        val result = allOf(*annotationFilters, anyOf(*otherFilters))
+        val result = allOf(*annotationFilters, *exclude, anyOf(*include))
         if (FtlConstants.useMock) println(result.describe)
         return result
     }
@@ -128,29 +130,31 @@ object TestFilters {
     }
 
     private fun withPackageName(packageNames: List<String>): TestFilter = TestFilter(
-        describe = "withPackageName ${packageNames.joinToString(", ")}",
+        describe = "withPackageName (${packageNames.joinToString(", ")})",
         shouldRun = { testMethod ->
-            packageNames.any { packageName -> testMethod.testName.startsWith(packageName) }
+            packageNames.any { packageName ->
+                testMethod.testName.startsWith(packageName)
+            }
         }
     )
 
     private fun withClassName(classNames: List<String>): TestFilter = TestFilter(
-        describe = "withClassName ${classNames.joinToString(", ")}",
+        describe = "withClassName (${classNames.joinToString(", ")})",
         shouldRun = { testMethod ->
             withPackageName(classNames).shouldRun(testMethod)
         }
     )
 
     private fun withAnnotation(annotations: List<String>): TestFilter = TestFilter(
-        describe = "withAnnotation ${annotations.joinToString(", ")}",
+        describe = "withAnnotation (${annotations.joinToString(", ")})",
         shouldRun = { testMethod ->
             testMethod.annotationNames.any { annotations.contains(it) }
         },
         isAnnotation = true
     )
 
     private fun not(filter: TestFilter): TestFilter = TestFilter(
-        describe = "not ${filter.describe}",
+        describe = "not (${filter.describe})",
         shouldRun = { testMethod ->
             filter.shouldRun(testMethod).not()
         },
@@ -169,9 +173,9 @@ object TestFilters {
         shouldRun = { testMethod ->
             if (FtlConstants.useMock) println(":: ${testMethod.testName} @${testMethod.annotations.firstOrNull()}")
             filters.isEmpty() || filters.all { filter ->
-                val result = filter.shouldRun(testMethod)
-                if (FtlConstants.useMock) println("  $result ${filter.describe}")
-                result
+                filter.shouldRun(testMethod).also { result ->
+                    if (FtlConstants.useMock) println("  $result ${filter.describe}")
+                }
             }
         }
     )

test_runner/src/test/kotlin/ftl/filter/TestFiltersTest.kt

@@ -29,8 +29,10 @@ val WITHOUT_LARGE_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
 val WITHOUT_MEDIUM_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
 val WITHOUT_SMALL_ANNOTATION = TestMethod("whatever.Foo#testName", emptyList())
 const val TEST_FILE = "src/test/kotlin/ftl/filter/fixtures/dummy-tests-file.txt"
+const val TEST_FILE_2 = "src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt"
 private const val IGNORE_ANNOTATION = "org.junit.Ignore"
 
+@Suppress("TooManyFunctions")
 @RunWith(FlankTestRunner::class)
 class TestFiltersTest {
 
@@ -249,10 +251,10 @@ class TestFiltersTest {
         }.map { "class ${it.testName}" }.toList()
 
         val expected = listOf(
-            "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
-            "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
-            "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]",
-            "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not withAnnotation Foo, anyOf [withClassName anyPackage_2.anyClass_2#anyMethod_2, withClassName anyPackage_3.anyClass_3#anyMethod_3]]]"
+            "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
+            "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
+            "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]",
+            "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]"
         )
 
         assertThat(output).isEqualTo(expected)
@@ -322,6 +324,36 @@ class TestFiltersTest {
 
         assertEquals(byNotAnnotation, expected)
     }
+
+    @Test
+    fun testFilteringClassAndPackageNegative() {
+        val filter = fromTestTargets(listOf("notPackage foo", "notClass whatever.Bar"))
+
+        assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
+        assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse()
+        assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue()
+    }
+
+    @Test
+    fun testFilteringClassAndPackageNegativeFromFile() {
+        val file = TestHelper.getPath(TEST_FILE_2) // contents: foo whatever.Bar
+        val filePath = file.toString()
+
+        val filter = fromTestTargets(listOf("notTestFile $filePath"))
+
+        assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
+        assertThat(filter.shouldRun(BAR_CLASSNAME)).isFalse()
+        assertThat(filter.shouldRun(BAR_PACKAGE)).isTrue()
+    }
+
+    @Test
+    fun `inclusion filter should override exclusion filter`() {
+        val filter = fromTestTargets(listOf("notPackage foo", "class whatever.Bar"))
+
+        assertThat(filter.shouldRun(FOO_PACKAGE)).isFalse()
+        assertThat(filter.shouldRun(BAR_PACKAGE)).isFalse()
+        assertThat(filter.shouldRun(BAR_CLASSNAME)).isTrue()
+    }
 }
 
 private fun getDefaultTestMethod(testName: String, annotation: String) =

test_runner/src/test/kotlin/ftl/filter/fixtures/exclude-tests.txt

@@ -0,0 +1,2 @@
+foo
+whatever.Bar