diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 4e7c0f727d..504dbb1a95 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -6,7 +6,6 @@ import ftl.args.ArgsHelper.createGcsBucket import ftl.args.ArgsHelper.createJunitBucket import ftl.args.ArgsHelper.evaluateFilePath import ftl.args.ArgsHelper.mergeYmlMaps -import ftl.args.ArgsHelper.validateTestMethods import ftl.args.ArgsHelper.yamlMapper import ftl.args.ArgsToString.devicesToString import ftl.args.ArgsToString.listToString @@ -62,13 +61,10 @@ class IosArgs( // computed properties not specified in yaml override val testShardChunks: List> by lazy { + if (disableSharding) return@lazy listOf(emptyList()) + val validTestMethods = Xctestrun.findTestNames(xctestrunFile) - validateTestMethods(testTargets, validTestMethods, "xctest binary") - val testsToShard = if (testTargets.isEmpty()) { - validTestMethods - } else { - testTargets - }.distinct() + val testsToShard = filterTests(validTestMethods, testTargets).distinct() ArgsHelper.calculateShards(testsToShard, this) } @@ -162,3 +158,19 @@ ${listToString(testTargets)} } } } + +fun filterTests(validTestMethods: List, testTargets: List): List { + if (testTargets.isEmpty()) { + return validTestMethods + } + + return validTestMethods.filter { test -> + testTargets.forEach { target -> + if (test.matches(target.toRegex())) { + return@filter true + } + } + + return@filter false + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index df804a7ea2..4cce241e54 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -40,15 +40,16 @@ object GcIosTestMatrix { val gcsBucket = args.resultsBucket val matrixGcsSuffix = join(runGcsPath, shardCounter.next()) val matrixGcsPath = join(gcsBucket, matrixGcsSuffix) - val methods = args.testShardChunks.elementAt(testShardsIndex) // Parameterized tests on iOS don't shard correctly. // Avoid changing Xctestrun file when disableSharding is on. val generatedXctestrun = if (args.disableSharding) { xcTestParsed.toByteArray() } else { + val methods = args.testShardChunks.elementAt(testShardsIndex) Xctestrun.rewrite(xcTestParsed, methods) } + val xctestrunFileGcsPath = GcStorage.uploadXCTestFile(args, gcsBucket, matrixGcsSuffix, generatedXctestrun) val iOSXCTest = IosXcTest() diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt index 742e11e7e4..15b9d1e98a 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt @@ -27,7 +27,7 @@ class IosArgsFileTest { private val xctestrunZip = getPath("src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip") private val xctestrunFile = getPath("src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun") - private val testName = "EarlGreyExampleMixedTests/testBasicSelection" + private val testName = "EarlGreyExampleSwiftTests/testLayout" // NOTE: Change working dir to '%MODULE_WORKING_DIR%' in IntelliJ to match gradle for this test to pass. @Test fun configLoadsSuccessfully() { @@ -69,17 +69,16 @@ class IosArgsFileTest { val config = IosArgs.load(yamlFile2) val chunk0 = arrayListOf( - "EarlGreyExampleMixedTests/testGrantCameraPermission", - "EarlGreyExampleMixedTests/testGrantMicrophonePermission", - "EarlGreyExampleMixedTests/testBasicSelection1", - "EarlGreyExampleMixedTests/testBasicSelection4" + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testWithInRoot", + "EarlGreyExampleSwiftTests/testWithCondition", + "EarlGreyExampleSwiftTests/testWithCustomFailureHandler" ) val chunk1 = arrayListOf( - "EarlGreyExampleMixedTests/testGrantCameraPermission", - "EarlGreyExampleMixedTests/testGrantMicrophonePermission", - "EarlGreyExampleMixedTests/testBasicSelection2", - "EarlGreyExampleMixedTests/testBasicSelection3" + "EarlGreyExampleSwiftTests/testWithGreyAssertions", + "EarlGreyExampleSwiftTests/testWithCustomMatcher", + "EarlGreyExampleSwiftTests/testWithCustomAssertion" ) val testShardChunks = config.testShardChunks diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 4e6ad0b935..98873cab4e 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -26,6 +26,7 @@ class IosArgsTest { private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExample.zip" private val xctestrunFile = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun" + private val invalidApp = "../test_app/apks/invalid.apk" private val xctestrunFileAbsolutePath = xctestrunFile.absolutePath() private val testAbsolutePath = testPath.absolutePath() private val iosNonDefault = """ @@ -91,10 +92,10 @@ class IosArgsTest { fun iosArgs_invalidXcodeExits() { exceptionRule.expectMessage("Xcode 99.9 is not a supported Xcode version") IosArgs( - GcloudYml(), - IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, xcodeVersion = "99.9")), - FlankYml(), - IosFlankYml(), + GcloudYml(), + IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, xcodeVersion = "99.9")), + FlankYml(), + IosFlankYml(), "" ) } @@ -256,6 +257,30 @@ IosArgs } } + @Test + fun `disableSharding allows using invalid app`() { + val yaml = """ + gcloud: + test: $invalidApp + xctestrun-file: $invalidApp + flank: + disableSharding: true + """ + IosArgs.load(yaml).testShardChunks + } + + @Test(expected = RuntimeException::class) + fun `Invalid app throws`() { + val yaml = """ + gcloud: + test: $invalidApp + xctestrun-file: $invalidApp + flank: + disableSharding: false + """ + IosArgs.load(yaml).testShardChunks + } + // gcloudYml @Test @@ -591,4 +616,60 @@ IosArgs val androidArgs = IosArgs.load(yaml, cli) assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) } + + private fun getValidTestsSample() = listOf( + "ClassOneTest/testOne", + "ClassOneTest/testTwo", + "ClassOneScreenshots/testOne", + "ClassTwoScreenshots/testTwo", + "ClassThreeTest/testName", + "ClassFourTest/testFour" + ) + + @Test + fun filterTests_emptyFilter() { + val tests = getValidTestsSample() + val actual = filterTests(tests, emptyList()) + + assertThat(actual).containsExactlyElementsIn(tests) + } + + @Test + fun filterTests_regularFilter() { + val tests = getValidTestsSample() + val filter = listOf("ClassOneTest/testOne", "ClassFourTest/testFour") + val actual = filterTests(tests, filter) + + val expected = listOf("ClassOneTest/testOne", "ClassFourTest/testFour") + + assertThat(actual).containsExactlyElementsIn(expected) + } + + @Test + fun filterTests_starFilter() { + val tests = getValidTestsSample() + val filter = listOf(".*?Test/testOne", ".*?/testFour") + val actual = filterTests(tests, filter) + + val expected = listOf( + "ClassOneTest/testOne", + "ClassFourTest/testFour" + ) + + assertThat(actual).containsExactlyElementsIn(expected) + } + + @Test + fun filterTests_starAndRegularFilter() { + val tests = getValidTestsSample() + val filter = listOf(".*?Screenshots/testTwo", "ClassOneTest/testOne") + val actual = filterTests(tests, filter) + + val expected = listOf( + "ClassTwoScreenshots/testTwo", + "ClassOneTest/testOne" + ) + + assertThat(actual).containsExactlyElementsIn(expected) + } } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml index 390137da4e..f3fa391c87 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml @@ -16,6 +16,6 @@ gcloud: flank: test-targets: - - EarlGreyExampleMixedTests/testBasicSelection + - EarlGreyExampleSwiftTests/testLayout testShards: 1 repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml index 520c0b9b0f..1172a04774 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml @@ -13,6 +13,6 @@ gcloud: flank: test-targets: - - EarlGreyExampleMixedTests/testBasicSelection + - EarlGreyExampleSwiftTests/testLayout testShards: 1 repeatTests: 1 diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml index a6da78c387..1fff8c552c 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml @@ -15,12 +15,6 @@ flank: testShards: 2 repeatTests: 1 test-targets: - - EarlGreyExampleMixedTests/testGrantCameraPermission - - EarlGreyExampleMixedTests/testGrantMicrophonePermission - - EarlGreyExampleMixedTests/testBasicSelection1 - - EarlGreyExampleMixedTests/testBasicSelection2 - - EarlGreyExampleMixedTests/testBasicSelection3 - - EarlGreyExampleMixedTests/testBasicSelection4 + - EarlGreyExampleSwiftTests/testWith.*$ test-targets-always-run: - - EarlGreyExampleMixedTests/testGrantCameraPermission - - EarlGreyExampleMixedTests/testGrantMicrophonePermission + - EarlGreyExampleSwiftTests/testWithGreyAssertions