fix: iOS nm global tests discovery (#2124)

* Add new flag for considering global tests

* iOS additions

test_runner/src/main/kotlin/ftl/args/CommonArgs.kt

@@ -49,5 +49,6 @@ data class CommonArgs(
     override val disableUsageStatistics: Boolean,
     override val outputReportType: OutputReportType,
     override val skipConfigValidation: Boolean,
-    override val customShardingJson: String
+    override val ignoreNonGlobalTests: Boolean,
+    override val customShardingJson: String,
 ) : IArgs

test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt

@@ -56,7 +56,8 @@ fun CommonConfig.createCommonArgs(
     disableUsageStatistics = flank.disableUsageStatistics ?: false,
     outputReportType = OutputReportType.fromName(flank.outputReport),
     skipConfigValidation = flank::skipConfigValidation.require(),
-    customShardingJson = flank::customShardingJson.require()
+    customShardingJson = flank::customShardingJson.require(),
+    ignoreNonGlobalTests = flank::ignoreNonGlobalTests.require(),
 ).apply {
     ArgsHelper.createJunitBucket(project, smartFlankGcsPath)
 }

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -101,6 +101,8 @@ interface IArgs {
     val shouldValidateConfig: Boolean
         get() = !skipConfigValidation
 
+    val ignoreNonGlobalTests: Boolean
+
     @AnonymizeInStatistics
     val customShardingJson: String
 

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -102,6 +102,7 @@ IosArgs
       output-report: $outputReportType
       skip-config-validation: $skipConfigValidation
       custom-sharding-json: $customShardingJson
+      ignore-global-tests: $ignoreNonGlobalTests
         """.trimIndent()
     }
 }

test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt

@@ -128,6 +128,18 @@ data class CommonFlankConfig @JsonIgnore constructor(
     @set:JsonProperty("ignore-failed-tests")
     var ignoreFailedTests: Boolean? by data
 
+    @set:CommandLine.Option(
+        names = ["--ignore-non-global-tests"],
+        description = [
+            "Test discovery for iOS considers by default global tests only. " +
+                "Disabling this flag will result in test discovery for private, public and global tests" +
+                "The technical difference is the flag used for nm test discovery. " +
+                "true: 'nm -gU' vs false: 'nm -U'"
+        ]
+    )
+    @set:JsonProperty("ignore-non-global-tests")
+    var ignoreNonGlobalTests: Boolean? by data
+
     @set:CommandLine.Option(
         names = ["--keep-file-path"],
         description = [
@@ -245,6 +257,7 @@ data class CommonFlankConfig @JsonIgnore constructor(
             outputReport = OutputReportType.NONE.name
             skipConfigValidation = false
             customShardingJson = ""
+            ignoreNonGlobalTests = true
         }
     }
 }

test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV1.kt

@@ -6,7 +6,8 @@ import ftl.ios.xctest.common.isMetadata
 
 internal fun findXcTestNamesV1(
     xcTestRoot: String,
-    xcTestNsDictionary: NSDictionary
+    xcTestNsDictionary: NSDictionary,
+    globalTestInclusion: Boolean = true,
 ): Map<String, List<String>> =
     xcTestNsDictionary
         .allKeys()
@@ -16,6 +17,7 @@ internal fun findXcTestNamesV1(
                 testRoot = xcTestRoot,
                 testTargetDict = xcTestNsDictionary[testTarget] as NSDictionary,
                 testTargetName = testTarget,
+                globalTestInclusion = globalTestInclusion
             )
         }
         .distinct()

test_runner/src/main/kotlin/ftl/ios/xctest/FindXcTestNamesV2.kt

@@ -8,16 +8,18 @@ import ftl.ios.xctest.common.getTestTargets
 
 internal fun findXcTestNamesV2(
     xcTestRoot: String,
-    xcTestNsDictionary: NSDictionary
+    xcTestNsDictionary: NSDictionary,
+    globalTestInclusion: Boolean
 ): Map<String, Map<String, List<String>>> =
     xcTestNsDictionary
         .getTestConfigurations()
         .mapValues { (_, configDict: NSDictionary) ->
-            configDict.findTestTargetMethods(xcTestRoot)
+            configDict.findTestTargetMethods(xcTestRoot, globalTestInclusion)
         }
 
 private fun NSDictionary.findTestTargetMethods(
-    xcTestRoot: String
+    xcTestRoot: String,
+    globalTestInclusion: Boolean
 ): Map<String, List<String>> =
     getTestTargets()
         .associateBy { targetDict -> targetDict.getBlueprintName() }
@@ -26,5 +28,6 @@ private fun NSDictionary.findTestTargetMethods(
                 testRoot = xcTestRoot,
                 testTargetName = name,
                 testTargetDict = dict,
+                globalTestInclusion = globalTestInclusion
             )
         }

test_runner/src/main/kotlin/ftl/ios/xctest/XcTestData.kt

@@ -109,19 +109,21 @@ private fun IosArgs.calculateConfigurationShards(
     filterTestConfigurationsIfNeeded(
         findXcTestNames(
             xcTestRoot = xcTestRoot,
-            xcTestNsDictionary = xcTestNsDictionary
+            xcTestNsDictionary = xcTestNsDictionary,
+            globalTestInclusion = ignoreNonGlobalTests
         )
     ).mapValues { (_, targets: Map<String, List<String>>) ->
         calculateConfigurationShards(targets, regexList)
     }
 
 private fun findXcTestNames(
     xcTestRoot: String,
-    xcTestNsDictionary: NSDictionary
+    xcTestNsDictionary: NSDictionary,
+    globalTestInclusion: Boolean
 ): Map<String, Map<String, List<String>>> =
     when (xcTestNsDictionary.getXcTestRunVersion()) {
-        V1 -> mapOf("" to findXcTestNamesV1(xcTestRoot, xcTestNsDictionary))
-        V2 -> findXcTestNamesV2(xcTestRoot, xcTestNsDictionary)
+        V1 -> mapOf("" to findXcTestNamesV1(xcTestRoot, xcTestNsDictionary, globalTestInclusion))
+        V2 -> findXcTestNamesV2(xcTestRoot, xcTestNsDictionary, globalTestInclusion)
     }
 
 private fun IosArgs.calculateConfigurationShards(

test_runner/src/main/kotlin/ftl/ios/xctest/common/FindTestsForTarget.kt

@@ -14,9 +14,10 @@ internal fun findTestsForTarget(
     testRoot: String,
     testTargetDict: NSDictionary,
     testTargetName: String,
+    globalTestInclusion: Boolean,
 ): List<String> = testTargetDict
     .findXcTestTargets(testTargetName)
-    .findBinaryTests(testRoot) - testTargetDict
+    .findBinaryTests(testRoot, globalTestInclusion) - testTargetDict
     .findTestsToSkip()
 
 private fun NSDictionary.findXcTestTargets(
@@ -27,14 +28,14 @@ private fun NSDictionary.findXcTestTargets(
         ?.first { product -> product.toString().contains("/$testTarget.xctest") }
         ?: throw FlankGeneralError("Test target $testTarget doesn't exist")
 
-private fun NSObject.findBinaryTests(testRoot: String): List<String> {
+private fun NSObject.findBinaryTests(testRoot: String, globalTestInclusion: Boolean): List<String> {
     val binaryRoot = toString().replace("__TESTROOT__/", testRoot)
     logLn("Found xctest: $binaryRoot")
 
     val binaryName = File(binaryRoot).nameWithoutExtension
     val binaryPath = Paths.get(binaryRoot, binaryName).toString()
 
-    return (parseObjcTests(binaryPath) + parseSwiftTests(binaryPath)).distinct()
+    return (parseObjcTests(binaryPath) + parseSwiftTests(binaryPath, globalTestInclusion)).distinct()
 }
 
 private fun NSDictionary.findTestsToSkip(): List<String> =
@@ -44,8 +45,8 @@ private fun NSDictionary.findTestsToSkip(): List<String> =
         ?: emptyList()
 
 @VisibleForTesting // TODO Remove it, cause is used only in tests
-internal fun findTestsForTestTarget(testTarget: String, xctestrun: File): List<String> =
+internal fun findTestsForTestTarget(testTarget: String, xctestrun: File, globalTestInclusion: Boolean): List<String> =
     parseToNSDictionary(xctestrun)[testTarget]
         ?.let { it as? NSDictionary }
-        ?.let { findTestsForTarget(xctestrun.parent + "/", it, testTarget) }
+        ?.let { findTestsForTarget(xctestrun.parent + "/", it, testTarget, globalTestInclusion) }
         ?: throw FlankGeneralError("Test target $testTarget doesn't exist")

test_runner/src/main/kotlin/ftl/ios/xctest/common/ParseSwiftTests.kt

@@ -7,7 +7,10 @@ import flank.common.isWindows
 import ftl.api.Command
 import ftl.api.runCommand
 
-internal fun parseSwiftTests(binary: String): List<String> {
+internal fun parseSwiftTests(
+    binary: String,
+    globalTestInclusion: Boolean
+): List<String> {
     installBinaries
     validateIsFile(binary)
     val results = mutableListOf<String>()
@@ -17,10 +20,11 @@ internal fun parseSwiftTests(binary: String): List<String> {
     // Windows has different limits
     val argMax = if (isWindows) 250_36 else 262_144
     val xargsOption = binary.quote().determineXArgsCommand(argMax)
+    val nmOption = determineNMOption(globalTestInclusion)
     val cmd = when {
-        isMacOS -> "nm -gU ${binary.quote()} | xargs $xargsOption xcrun swift-demangle"
-        isLinux -> "export LD_LIBRARY_PATH=~/.flank; export PATH=~/.flank:\$PATH; nm -gU ${binary.quote()} | xargs $xargsOption swift-demangle"
-        isWindows -> "llvm-nm.exe -gU ${binary.quote().replace("\\", "/")} | xargs.exe $xargsOption swift-demangle.exe"
+        isMacOS -> "nm $nmOption ${binary.quote()} | xargs $xargsOption xcrun swift-demangle"
+        isLinux -> "export LD_LIBRARY_PATH=~/.flank; export PATH=~/.flank:\$PATH; nm $nmOption ${binary.quote()} | xargs $xargsOption swift-demangle"
+        isWindows -> "llvm-nm.exe $nmOption ${binary.quote().replace("\\", "/")} | xargs.exe $xargsOption swift-demangle.exe"
         else -> throw RuntimeException("Unsupported OS for Integration Tests")
     }
 
@@ -45,6 +49,8 @@ internal fun parseSwiftTests(binary: String): List<String> {
     return results.distinct()
 }
 
+private fun determineNMOption(globalTestInclusion: Boolean) = if (globalTestInclusion) "-gU" else "-U"
+
 private fun String.determineXArgsCommand(argsMax: Int) = if (this.length > argsMax) {
     println("WARNING: The amount of characters has exceeded the OS threshold for xArgs. -n1 will be used to ensure the command completes successfully. It may increase completion time substantially.")
     "-n1"

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -290,6 +290,7 @@ IosArgs
       output-report: json
       skip-config-validation: false
       custom-sharding-json: 
+      ignore-global-tests: true
             """.trimIndent()
         )
     }
@@ -356,6 +357,7 @@ IosArgs
       output-report: none
       skip-config-validation: false
       custom-sharding-json: 
+      ignore-global-tests: true
             """.trimIndent(),
             args.toString()
         )

test_runner/src/test/kotlin/ftl/ios/xctest/FindXcTestNamesV2KtTest.kt

@@ -19,7 +19,8 @@ class FindXcTestNamesV2KtTest {
         // when
         val names = findXcTestNamesV2(
             xcTestRoot = xctestrun.parent + "/",
-            xcTestNsDictionary = parseToNSDictionary(xctestrun)
+            xcTestNsDictionary = parseToNSDictionary(xctestrun),
+            globalTestInclusion = true
         )
 
         // then

test_runner/src/test/kotlin/ftl/ios/xctest/common/FindTestsForTargetKtTest.kt

@@ -59,30 +59,30 @@ class FindTestsForTargetKtTest {
         Files.write(tmpXml, inputXml.toByteArray())
         tmpXml.toFile().deleteOnExit()
 
-        val actualTests = findTestsForTestTarget("EarlGreyExampleSwiftTests", tmpXml.toFile()).sorted()
+        val actualTests = findTestsForTestTarget("EarlGreyExampleSwiftTests", tmpXml.toFile(), true).sorted()
         assertThat(actualTests).isEqualTo(listOf("EarlGreyExampleSwiftTests/testBasicSelection"))
     }
 
     @Test
     fun findTestNamesForTestTarget() {
         assumeFalse(isWindows)
-        val names = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1)).sorted()
+        val names = findTestsForTestTarget(testTarget = "EarlGreyExampleSwiftTests", xctestrun = File(swiftXcTestRunV1), true).sorted()
         assertThat(swiftTestsV1).isEqualTo(names)
     }
 
     @Test(expected = FlankGeneralError::class)
     fun `findTestNames for nonexisting test target`() {
         assumeFalse(isWindows)
-        findTestsForTestTarget(testTarget = "Incorrect", xctestrun = File(swiftXcTestRunV1)).sorted()
+        findTestsForTestTarget(testTarget = "Incorrect", xctestrun = File(swiftXcTestRunV1), true).sorted()
     }
 
     @Test
     fun `find test names for xctestrun file containing multiple test targets`() {
         assumeFalse(isWindows)
-        val names = findTestsForTestTarget(testTarget = "FlankExampleTests", xctestrun = File(multiTargetsSwiftXcTestRunV1)).sorted()
+        val names = findTestsForTestTarget(testTarget = "FlankExampleTests", xctestrun = File(multiTargetsSwiftXcTestRunV1), true).sorted()
         assertThat(names).isEqualTo(listOf("FlankExampleTests/test1", "FlankExampleTests/test2"))
 
-        val names2 = findTestsForTestTarget(testTarget = "FlankExampleSecondTests", xctestrun = File(multiTargetsSwiftXcTestRunV1)).sorted()
+        val names2 = findTestsForTestTarget(testTarget = "FlankExampleSecondTests", xctestrun = File(multiTargetsSwiftXcTestRunV1), true).sorted()
         assertThat(names2).isEqualTo(listOf("FlankExampleSecondTests/test3", "FlankExampleSecondTests/test4"))
     }
 }

test_runner/src/test/kotlin/ftl/ios/xctest/common/ParseSwiftTestsKtTest.kt

@@ -14,29 +14,29 @@ class ParseSwiftTestsKtTest {
     fun parseSwiftTests() {
         assumeFalse(isWindows)
 
-        val results = parseSwiftTests(swiftBinary).sorted()
+        val results = parseSwiftTests(swiftBinary, true).sorted()
         checkSwiftTests(results)
     }
 
     @Test(expected = FlankGeneralError::class)
     fun `parseSwiftTests fileNotFound`() {
         assumeFalse(isWindows)
 
-        parseSwiftTests("./BinaryThatDoesNotExist")
+        parseSwiftTests("./BinaryThatDoesNotExist", true)
     }
 
     @Test(expected = FlankGeneralError::class)
     fun `parseSwiftTests tmpFolder`() {
         assumeFalse(isWindows)
 
-        parseSwiftTests("/tmp")
+        parseSwiftTests("/tmp", true)
     }
 
     @Test
     fun `Parse Swift with space in path`() {
         assumeFalse(isWindows)
 
-        val results = parseSwiftTests("$FIXTURES_PATH/sp ace/swift/EarlGreyExampleSwiftTests").sorted()
+        val results = parseSwiftTests("$FIXTURES_PATH/sp ace/swift/EarlGreyExampleSwiftTests", true).sorted()
         checkSwiftTests(results)
     }
 }