diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt index 5342d4377a..3051396a42 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt @@ -40,8 +40,5 @@ open class AndroidArgsCompanion : IArgs.ICompanion { config = defaultAndroidConfig() + loadAndroidConfig(reader = yamlReader) + cli?.config - ).apply { - commonArgs.validate() - this.validate() - } + ) } diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index d7f4023f92..9bcb4547ac 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -245,7 +245,10 @@ object ArgsHelper { } } -fun String.processFilePath(name: String): String = - if (startsWith(GCS_PREFIX)) - this.also { ArgsHelper.assertGcsFileExists(it) } else - ArgsHelper.evaluateFilePath(this).also { ArgsHelper.assertFileExists(it, name) } +fun String.normalizeFilePath(): String = + if (startsWith(GCS_PREFIX)) this + else try { + ArgsHelper.evaluateFilePath(this) + } catch (e: Throwable) { + this + } diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt index 3c39521d09..0b694c249f 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -13,26 +13,26 @@ fun createAndroidArgs( ) = AndroidArgs( commonArgs = commonArgs, // gcloud - appApk = gcloud.app?.processFilePath("from app"), - testApk = gcloud.test?.processFilePath("from test"), + appApk = gcloud.app?.normalizeFilePath(), + testApk = gcloud.test?.normalizeFilePath(), useOrchestrator = gcloud.useOrchestrator!!, testTargets = gcloud.testTargets!!.filterNotNull(), testRunnerClass = gcloud.testRunnerClass, roboDirectives = gcloud.roboDirectives!!.parseRoboDirectives(), performanceMetrics = gcloud.performanceMetrics!!, - otherFiles = gcloud.otherFiles!!.mapValues { (_, path) -> path.processFilePath("from otherFiles") }, + otherFiles = gcloud.otherFiles!!.mapValues { (_, path) -> path.normalizeFilePath() }, numUniformShards = gcloud.numUniformShards, environmentVariables = gcloud.environmentVariables!!, directoriesToPull = gcloud.directoriesToPull!!, autoGoogleLogin = gcloud.autoGoogleLogin!!, - additionalApks = gcloud.additionalApks!!.map { it.processFilePath("from additional-apks") }, - roboScript = gcloud.roboScript?.processFilePath("from roboScript"), + additionalApks = gcloud.additionalApks!!.map { it.normalizeFilePath() }, + roboScript = gcloud.roboScript?.normalizeFilePath(), // flank additionalAppTestApks = flank.additionalAppTestApks?.map { (app, test) -> AppTestPair( - app = app?.processFilePath("from additional-app-test-apks.app"), - test = test.processFilePath("from additional-app-test-apks.test") + app = app?.normalizeFilePath(), + test = test.normalizeFilePath() ) } ?: emptyList(), useLegacyJUnitResult = flank.useLegacyJUnitResult!! diff --git a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt index c3a01e34eb..bf7bf170de 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt @@ -18,10 +18,10 @@ private fun createIosArgs( commonArgs: CommonArgs ) = IosArgs( commonArgs = commonArgs.copy(maxTestShards = convertToShardCount(commonArgs.maxTestShards)), - xctestrunZip = gcloud.test?.processFilePath("from test").orEmpty(), - xctestrunFile = gcloud.xctestrunFile?.processFilePath("from xctestrun-file").orEmpty(), + xctestrunZip = gcloud.test?.normalizeFilePath().orEmpty(), + xctestrunFile = gcloud.xctestrunFile?.normalizeFilePath().orEmpty(), xcodeVersion = gcloud.xcodeVersion, - testTargets = flank.testTargets!!.filterNotNull() + testTargets = flank.testTargets?.filterNotNull().orEmpty() ) private fun convertToShardCount(inputValue: Int) = diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt index b710084d48..d5cdca044c 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt @@ -40,8 +40,5 @@ open class IosArgsCompanion : IArgs.ICompanion { config = defaultIosConfig() + loadIosConfig(reader = yamlReader) + cli?.config - ).apply { - commonArgs.validate() - this.validate() - } + ) } diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 02455300f5..8164c7bf0b 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -11,25 +11,16 @@ import ftl.run.exception.FlankConfigurationError import ftl.run.exception.IncompatibleTestDimensionError import java.io.File -fun AndroidArgs.validate() { - assertAdditionalAppTestApks() +fun AndroidArgs.validate() = apply { + commonArgs.validate() assertDevicesSupported() assertShards() assertTestTypes() - assertRoboTest() assertDirectoriesToPull() assertMaxTestShardsByDeviceType() assertParametersConflict() -} - -private fun AndroidArgs.assertAdditionalAppTestApks() { - if (appApk == null) additionalAppTestApks - .filter { (app, _) -> app == null } - .map { File(it.test).name } - .run { - if (isNotEmpty()) - throw FlankConfigurationError("Cannot resolve app apk pair for $this") - } + assertTestFiles() + assertOtherFiles() } private fun AndroidArgs.assertDevicesSupported() = devices @@ -57,13 +48,6 @@ private fun AndroidArgs.assertTestTypes() { ) } -private fun AndroidArgs.assertRoboTest() { - // Using both roboDirectives and roboScript may hang test execution on FTL - if (roboDirectives.isNotEmpty() && roboScript != null) throw FlankConfigurationError( - "Options robo-directives and robo-script are mutually exclusive, use only one of them." - ) -} - // Validation is done according to https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run#--directories-to-pull private fun AndroidArgs.assertDirectoriesToPull() { val correctNameRegex = "(/[a-zA-Z0-9_\\-.+]+)+/?".toRegex() @@ -73,8 +57,8 @@ private fun AndroidArgs.assertDirectoriesToPull() { ?.also { throw FlankConfigurationError( "Invalid value for [directories-to-pull]: Invalid path $it.\n" + - "Path must be absolute paths under /sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2).\n" + - "Path names are restricted to the characters [a-zA-Z0-9_-./+]. " + "Path must be absolute paths under /sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2).\n" + + "Path names are restricted to the characters [a-zA-Z0-9_-./+]. " ) } } @@ -105,3 +89,53 @@ private fun AndroidArgs.assertParametersConflict() { if (useLegacyJUnitResult && fullJUnitResult) throw FlankConfigurationError("Parameters conflict, you cannot set: `--legacy-junit-result` and `--full-junit-result` at the same time.") } + +private fun AndroidArgs.assertTestFiles() { + if (isInstrumentationTest) assertInstrumentationTest() + if (isRoboTest) assertRoboTest() +} + +private fun AndroidArgs.assertInstrumentationTest() { + assertAdditionalAppTestApks() + assertApkFilePaths() +} + +private fun AndroidArgs.assertAdditionalAppTestApks() { + if (appApk == null) additionalAppTestApks + .filter { (app, _) -> app == null } + .map { File(it.test).name } + .run { if (isNotEmpty()) throw FlankConfigurationError("Cannot resolve app apk pair for $this") } +} + +private fun AndroidArgs.assertApkFilePaths() { + appApkPath().forEach { (file, comment) -> + ArgsHelper.assertFileExists(file, comment) + } +} + +private fun AndroidArgs.appApkPath(): Map = + mapOf( + appApk to "from app", + testApk to "from test" + ).filterNotNull() + additionalAppTestApks.fold(emptyMap()) { acc, pair -> + acc + mapOf( + pair.app to "from additional-app-test-apks.app", + pair.test to "from additional-app-test-apks.test" + ).filterNotNull() + } + +private fun Map.filterNotNull() = filter { it.key != null }.mapKeys { it.key!! } + +private fun AndroidArgs.assertRoboTest() { + // Using both roboDirectives and roboScript may hang test execution on FTL + if (roboDirectives.isNotEmpty() && roboScript != null) throw FlankConfigurationError( + "Options robo-directives and robo-script are mutually exclusive, use only one of them." + ) + if (roboScript != null) + ArgsHelper.assertFileExists(roboScript.toString(), "from roboScript") + ArgsHelper.assertFileExists(appApk.toString(), "from app") +} + +private fun AndroidArgs.assertOtherFiles() { + otherFiles.forEach { (_, path) -> ArgsHelper.assertFileExists(path, "from otherFiles") } +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt index fffdd91c30..1eb9b3d21b 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -4,12 +4,23 @@ import ftl.ios.IosCatalog import ftl.run.exception.FlankConfigurationError import ftl.run.exception.IncompatibleTestDimensionError -fun IosArgs.validate() { +fun IosArgs.validate() = apply { + commonArgs.validate() assertXcodeSupported() assertDevicesSupported() assertTestTypes() assertMaxTestShards() + assertTestFiles() } + +fun IosArgs.validateRefresh() = apply { + commonArgs.validate() + assertXcodeSupported() + assertDevicesSupported() + assertTestTypes() + assertMaxTestShards() +} + private fun IosArgs.assertMaxTestShards() { this.maxTestShards if ( maxTestShards !in IArgs.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE && @@ -32,3 +43,8 @@ private fun IosArgs.assertDevicesSupported() = devices.forEach { device -> if (!IosCatalog.supportedDevice(device.model, device.version, this.project)) throw IncompatibleTestDimensionError("iOS ${device.version} on ${device.model} is not a supported device") } + +private fun IosArgs.assertTestFiles() { + ArgsHelper.assertFileExists(xctestrunFile, "from test") + ArgsHelper.assertFileExists(xctestrunZip, "from xctestrun-file") +} diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 971330c289..a17e15e2d8 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -1,6 +1,7 @@ package ftl.cli.firebase.test.android import ftl.args.AndroidArgs +import ftl.args.validate import ftl.cli.firebase.test.CommonRunCommand import ftl.config.FtlConstants import ftl.config.emptyAndroidConfig @@ -43,7 +44,7 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = AndroidArgs.load(Paths.get(configPath), cli = this) + val config = AndroidArgs.load(Paths.get(configPath), cli = this).validate() runBlocking { if (dumpShards) dumpShards(args = config, obfuscatedOutput = obfuscate) else newTestRun(config) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 8c109c1926..6603b34365 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -1,6 +1,7 @@ package ftl.cli.firebase.test.ios import ftl.args.IosArgs +import ftl.args.validate import ftl.cli.firebase.test.CommonRunCommand import ftl.config.FtlConstants import ftl.config.emptyIosConfig @@ -43,7 +44,7 @@ class IosRunCommand : CommonRunCommand(), Runnable { MockServer.start() } - val config = IosArgs.load(Paths.get(configPath), cli = this) + val config = IosArgs.load(Paths.get(configPath), cli = this).validate() if (dumpShards) { dumpShards(args = config, obfuscatedOutput = obfuscate) diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt index 11ecdf0d5a..4ba7c69abc 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastArgs.kt @@ -3,6 +3,8 @@ package ftl.run.common import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs +import ftl.args.validate +import ftl.args.validateRefresh import ftl.config.FtlConstants import ftl.run.exception.FlankGeneralError import java.nio.file.Paths @@ -15,8 +17,8 @@ internal fun getLastArgs(args: IArgs): IArgs { val androidConfig = Paths.get(args.localResultDir, lastRun, FtlConstants.defaultAndroidConfig) return when { - iosConfig.toFile().exists() -> IosArgs.load(iosConfig) - androidConfig.toFile().exists() -> AndroidArgs.load(androidConfig) + iosConfig.toFile().exists() -> IosArgs.load(iosConfig).validateRefresh() + androidConfig.toFile().exists() -> AndroidArgs.load(androidConfig).validate() else -> throw FlankGeneralError("No config file found in the last run folder: $lastRun") } } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 2e8f58bdde..121388c4eb 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -47,6 +47,7 @@ class AndroidArgsTest { private val empty = emptyList() private val appApk = "../test_projects/android/apks/app-debug.apk" private val invalidApk = "../test_projects/android/apks/invalid.apk" + private val nonExistingApk = "../test_projects/android/apks/app-debug_non_existing.apk" private val testApk = "../test_projects/android/apks/app-debug-androidTest.apk" private val testErrorApk = "../test_projects/android/apks/error-androidTest.apk" private val testFlakyApk = "../test_projects/android/apks/flaky-androidTest.apk" @@ -157,7 +158,7 @@ class AndroidArgsTest { - model: no version: nope """ - ) + ).validate() } } @@ -173,7 +174,7 @@ class AndroidArgsTest { - model: NexusLowRes version: nope """ - ) + ).validate() } } @@ -189,7 +190,7 @@ class AndroidArgsTest { - model: shamu version: 18 """ - ) + ).validate() } } @@ -496,7 +497,7 @@ AndroidArgs flank: disable-sharding: true """ - val androidArgs = AndroidArgs.load(yaml) + val androidArgs = AndroidArgs.load(yaml).validate() val testShardChunks = runBlocking { androidArgs.createAndroidTestContexts() } assertThat(testShardChunks).hasSize(0) } @@ -508,7 +509,7 @@ AndroidArgs app: $invalidApk test: $invalidApk """ - val androidArgs = AndroidArgs.load(yaml) + val androidArgs = AndroidArgs.load(yaml).validate() val testShardChunks = runBlocking { androidArgs.createAndroidTestContexts() } assertThat(testShardChunks).hasSize(0) } @@ -523,9 +524,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).appApk).isEqualTo(appApkAbsolutePath) + assertThat(AndroidArgs.load(yaml).validate().appApk).isEqualTo(appApkAbsolutePath) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.appApk).isEqualTo(testApkAbsolutePath) } @@ -539,9 +540,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).testApk).isEqualTo(testApkAbsolutePath) + assertThat(AndroidArgs.load(yaml).validate().testApk).isEqualTo(testApkAbsolutePath) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.testApk).isEqualTo(appApkAbsolutePath) } @@ -561,9 +562,9 @@ AndroidArgs - class com.example.app.ExampleUiTest#testPasses - class com.example.app.ExampleUiTest#testFails """ - assertThat(AndroidArgs.load(yaml).testTargets.size).isEqualTo(3) + assertThat(AndroidArgs.load(yaml).validate().testTargets.size).isEqualTo(3) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.testTargets.size).isEqualTo(2) assertThat(androidArgs.testTargets).isEqualTo(listOf(testTarget, testTarget)) } @@ -579,9 +580,9 @@ AndroidArgs test: $testApk use-orchestrator: false """ - assertThat(AndroidArgs.load(yaml).useOrchestrator).isFalse() + assertThat(AndroidArgs.load(yaml).validate().useOrchestrator).isFalse() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.useOrchestrator).isTrue() } @@ -596,9 +597,9 @@ AndroidArgs test: $testApk use-orchestrator: true """ - assertThat(AndroidArgs.load(yaml).useOrchestrator).isTrue() + assertThat(AndroidArgs.load(yaml).validate().useOrchestrator).isTrue() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.useOrchestrator).isFalse() } @@ -613,9 +614,9 @@ AndroidArgs test: $testApk auto-google-login: false """ - assertThat(AndroidArgs.load(yaml).autoGoogleLogin).isFalse() + assertThat(AndroidArgs.load(yaml).validate().autoGoogleLogin).isFalse() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.autoGoogleLogin).isTrue() } @@ -630,9 +631,9 @@ AndroidArgs test: $testApk auto-google-login: true """ - assertThat(AndroidArgs.load(yaml).autoGoogleLogin).isTrue() + assertThat(AndroidArgs.load(yaml).validate().autoGoogleLogin).isTrue() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.autoGoogleLogin).isFalse() } @@ -647,9 +648,9 @@ AndroidArgs test: $testApk performance-metrics: false """ - assertThat(AndroidArgs.load(yaml).performanceMetrics).isFalse() + assertThat(AndroidArgs.load(yaml).validate().performanceMetrics).isFalse() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.performanceMetrics).isTrue() } @@ -664,9 +665,9 @@ AndroidArgs test: $testApk performance-metrics: true """ - assertThat(AndroidArgs.load(yaml).performanceMetrics).isTrue() + assertThat(AndroidArgs.load(yaml).validate().performanceMetrics).isTrue() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.performanceMetrics).isFalse() } @@ -681,9 +682,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).numUniformShards).isNull() + assertThat(AndroidArgs.load(yaml).validate().numUniformShards).isNull() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.numUniformShards).isEqualTo(expected) } @@ -697,7 +698,7 @@ AndroidArgs flank: max-test-shards: ${AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE.last} """ - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -710,9 +711,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).testRunnerClass).isNull() + assertThat(AndroidArgs.load(yaml).validate().testRunnerClass).isNull() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.testRunnerClass).isEqualTo("com.foo.bar.TestRunner") } @@ -726,9 +727,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).environmentVariables).isEmpty() + assertThat(AndroidArgs.load(yaml).validate().environmentVariables).isEmpty() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() val expectedMap = mapOf("a" to "1", "b" to "2") assertThat(androidArgs.environmentVariables).isEqualTo(expectedMap) } @@ -743,9 +744,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).directoriesToPull).isEmpty() + assertThat(AndroidArgs.load(yaml).validate().directoriesToPull).isEmpty() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.directoriesToPull).isEqualTo(listOf("/sdcard/test", "/data/local/tmp/test")) } @@ -759,9 +760,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).directoriesToPull).isEmpty() + assertThat(AndroidArgs.load(yaml).validate().directoriesToPull).isEmpty() - AndroidArgs.load(yaml, cli) + AndroidArgs.load(yaml, cli).validate() } @Test @@ -774,9 +775,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).filesToDownload).isEmpty() + assertThat(AndroidArgs.load(yaml).validate().filesToDownload).isEmpty() - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.filesToDownload).isEqualTo(listOf("a", "b")) } @@ -791,11 +792,11 @@ AndroidArgs test: $testApk """ val expectedDefaultDevice = Device(defaultAndroidModel, defaultAndroidVersion, isVirtual = true) - val defaultDevices = AndroidArgs.load(yaml).devices + val defaultDevices = AndroidArgs.load(yaml).validate().devices assertThat(defaultDevices.first()).isEqualTo(expectedDefaultDevice) assertThat(defaultDevices.size).isEqualTo(1) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() val expectedDevice = Device("shamu", "22", "zh_CN", "default", isVirtual = false) val actualDevices = androidArgs.devices assertThat(actualDevices.first()).isEqualTo(expectedDevice) @@ -813,7 +814,7 @@ AndroidArgs app: $appApk test: $testApk """ - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() val expectedDevice = Device("shamu", "22", "zh_CN", "default", false) val actualDevices = androidArgs.devices assertThat(actualDevices.size).isEqualTo(2) @@ -834,8 +835,8 @@ AndroidArgs test: $testApk results-bucket: b """ - assertThat(AndroidArgs.load(yaml).resultsBucket).isEqualTo("b") - assertThat(AndroidArgs.load(yaml, cli).resultsBucket).isEqualTo("a") + assertThat(AndroidArgs.load(yaml).validate().resultsBucket).isEqualTo("b") + assertThat(AndroidArgs.load(yaml, cli).validate().resultsBucket).isEqualTo("a") } @Test @@ -849,8 +850,8 @@ AndroidArgs test: $testApk record-video: false """ - assertThat(AndroidArgs.load(yaml).recordVideo).isFalse() - assertThat(AndroidArgs.load(yaml, cli).recordVideo).isTrue() + assertThat(AndroidArgs.load(yaml).validate().recordVideo).isFalse() + assertThat(AndroidArgs.load(yaml, cli).validate().recordVideo).isTrue() } @Test @@ -864,8 +865,8 @@ AndroidArgs test: $testApk record-video: true """ - assertThat(AndroidArgs.load(yaml).recordVideo).isTrue() - assertThat(AndroidArgs.load(yaml, cli).recordVideo).isFalse() + assertThat(AndroidArgs.load(yaml).validate().recordVideo).isTrue() + assertThat(AndroidArgs.load(yaml, cli).validate().recordVideo).isFalse() } @Test @@ -879,8 +880,8 @@ AndroidArgs test: $testApk timeout: 2m """ - assertThat(AndroidArgs.load(yaml).testTimeout).isEqualTo("2m") - assertThat(AndroidArgs.load(yaml, cli).testTimeout).isEqualTo("1m") + assertThat(AndroidArgs.load(yaml).validate().testTimeout).isEqualTo("2m") + assertThat(AndroidArgs.load(yaml, cli).validate().testTimeout).isEqualTo("1m") } @Test @@ -894,8 +895,8 @@ AndroidArgs test: $testApk async: false """ - assertThat(AndroidArgs.load(yaml).async).isEqualTo(false) - assertThat(AndroidArgs.load(yaml, cli).async).isEqualTo(true) + assertThat(AndroidArgs.load(yaml).validate().async).isEqualTo(false) + assertThat(AndroidArgs.load(yaml, cli).validate().async).isEqualTo(true) } @Test @@ -924,8 +925,8 @@ AndroidArgs test: $testApk results-history-name: a """ - assertThat(AndroidArgs.load(yaml).resultsHistoryName).isEqualTo("a") - assertThat(AndroidArgs.load(yaml, cli).resultsHistoryName).isEqualTo("b") + assertThat(AndroidArgs.load(yaml).validate().resultsHistoryName).isEqualTo("a") + assertThat(AndroidArgs.load(yaml, cli).validate().resultsHistoryName).isEqualTo("b") } @Test @@ -941,8 +942,8 @@ AndroidArgs flank: max-test-shards: 2 """ - assertThat(AndroidArgs.load(yaml).maxTestShards).isEqualTo(2) - assertThat(AndroidArgs.load(yaml, cli).maxTestShards).isEqualTo(3) + assertThat(AndroidArgs.load(yaml).validate().maxTestShards).isEqualTo(2) + assertThat(AndroidArgs.load(yaml, cli).validate().maxTestShards).isEqualTo(3) } @Test @@ -958,8 +959,8 @@ AndroidArgs flank: shard-time: 2 """ - assertThat(AndroidArgs.load(yaml).shardTime).isEqualTo(2) - assertThat(AndroidArgs.load(yaml, cli).shardTime).isEqualTo(3) + assertThat(AndroidArgs.load(yaml).validate().shardTime).isEqualTo(2) + assertThat(AndroidArgs.load(yaml, cli).validate().shardTime).isEqualTo(3) } @Test @@ -975,8 +976,8 @@ AndroidArgs flank: disable-sharding: false """ - assertThat(AndroidArgs.load(yaml).disableSharding).isEqualTo(false) - assertThat(AndroidArgs.load(yaml, cli).disableSharding).isEqualTo(true) + assertThat(AndroidArgs.load(yaml).validate().disableSharding).isEqualTo(false) + assertThat(AndroidArgs.load(yaml, cli).validate().disableSharding).isEqualTo(true) } @Test @@ -992,8 +993,8 @@ AndroidArgs flank: num-test-runs: 2 """ - assertThat(AndroidArgs.load(yaml).repeatTests).isEqualTo(2) - assertThat(AndroidArgs.load(yaml, cli).repeatTests).isEqualTo(3) + assertThat(AndroidArgs.load(yaml).validate().repeatTests).isEqualTo(2) + assertThat(AndroidArgs.load(yaml, cli).validate().repeatTests).isEqualTo(3) } @Test @@ -1006,7 +1007,7 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml, cli).testTargetsAlwaysRun).isEqualTo(arrayListOf("com.A", "com.B")) + assertThat(AndroidArgs.load(yaml, cli).validate().testTargetsAlwaysRun).isEqualTo(arrayListOf("com.A", "com.B")) } @Test @@ -1021,8 +1022,8 @@ AndroidArgs results-dir: results """ - assertThat(AndroidArgs.load(yaml).resultsDir).isEqualTo("results") - assertThat(AndroidArgs.load(yaml, cli).resultsDir).isEqualTo("build") + assertThat(AndroidArgs.load(yaml).validate().resultsDir).isEqualTo("results") + assertThat(AndroidArgs.load(yaml, cli).validate().resultsDir).isEqualTo("build") } @Test @@ -1035,9 +1036,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).flakyTestAttempts).isEqualTo(0) + assertThat(AndroidArgs.load(yaml).validate().flakyTestAttempts).isEqualTo(0) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) } @@ -1052,9 +1053,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).smartFlankGcsPath).isEqualTo("") + assertThat(AndroidArgs.load(yaml).validate().smartFlankGcsPath).isEqualTo("") - val args = AndroidArgs.load(yaml, cli) + val args = AndroidArgs.load(yaml, cli).validate() assertThat(args.smartFlankGcsPath).isEqualTo(xml) } @@ -1070,9 +1071,9 @@ AndroidArgs flank: local-result-dir: a """ - assertThat(AndroidArgs.load(yaml).localResultDir).isEqualTo("a") + assertThat(AndroidArgs.load(yaml).validate().localResultDir).isEqualTo("a") - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.localResultDir).isEqualTo("b") } @@ -1088,9 +1089,9 @@ AndroidArgs flank: smart-flank-disable-upload: false """ - assertThat(AndroidArgs.load(yaml).smartFlankDisableUpload).isEqualTo(false) + assertThat(AndroidArgs.load(yaml).validate().smartFlankDisableUpload).isEqualTo(false) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.smartFlankDisableUpload).isEqualTo(true) } @@ -1110,12 +1111,12 @@ AndroidArgs """ assertEquals( listOf(AppTestPair(appApkAbsolutePath, testErrorApkAbsolutePath)), - AndroidArgs.load(yaml).additionalAppTestApks + AndroidArgs.load(yaml).validate().additionalAppTestApks ) assertEquals( listOf(AppTestPair(appApkAbsolutePath, testFlakyApkAbsolutePath)), - AndroidArgs.load(yaml, cli).additionalAppTestApks + AndroidArgs.load(yaml, cli).validate().additionalAppTestApks ) } @@ -1131,9 +1132,9 @@ AndroidArgs flank: keep-file-path: false """ - assertThat(AndroidArgs.load(yaml).keepFilePath).isEqualTo(false) + assertThat(AndroidArgs.load(yaml).validate().keepFilePath).isEqualTo(false) - val androidArgs = AndroidArgs.load(yaml, cli) + val androidArgs = AndroidArgs.load(yaml, cli).validate() assertThat(androidArgs.keepFilePath).isEqualTo(true) } @@ -1147,9 +1148,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).parsedTimeout).isEqualTo(Long.MAX_VALUE) + assertThat(AndroidArgs.load(yaml).validate().parsedTimeout).isEqualTo(Long.MAX_VALUE) - val args = AndroidArgs.load(yaml, cli) + val args = AndroidArgs.load(yaml, cli).validate() assertThat(args.parsedTimeout).isEqualTo(20 * 60 * 1000L) } @@ -1163,9 +1164,9 @@ AndroidArgs app: $appApk test: $testApk """ - assertThat(AndroidArgs.load(yaml).outputStyle).isEqualTo(OutputStyle.Multi) + assertThat(AndroidArgs.load(yaml).validate().outputStyle).isEqualTo(OutputStyle.Multi) - val args = AndroidArgs.load(yaml, cli) + val args = AndroidArgs.load(yaml, cli).validate() assertThat(args.outputStyle).isEqualTo(OutputStyle.Verbose) } @@ -1178,7 +1179,7 @@ AndroidArgs flank: output-style: unknown """ - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1196,7 +1197,7 @@ AndroidArgs test: $testApk """.trimIndent() - val parsedYml = AndroidArgs.load(yaml) + val parsedYml = AndroidArgs.load(yaml).validate() val (matrixMap, chunks) = runBlocking { runAndroidTests(parsedYml) } assertEquals(4, matrixMap.map.size) assertEquals(4, chunks.size) @@ -1213,7 +1214,7 @@ AndroidArgs test: $testErrorApk """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1226,7 +1227,7 @@ AndroidArgs test: $testApk """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1259,7 +1260,7 @@ AndroidArgs robo-script: $appApk """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1270,7 +1271,7 @@ AndroidArgs robo-script: $appApk """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals( appApkAbsolutePath, @@ -1288,7 +1289,7 @@ AndroidArgs click:resource_name_2: "" """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals( args.roboDirectives, @@ -1308,7 +1309,7 @@ AndroidArgs app: $appApk """.trimIndent() - val args = AndroidArgs.load(yaml, cli) + val args = AndroidArgs.load(yaml, cli).validate() assertEquals( args.roboScript, @@ -1334,7 +1335,7 @@ AndroidArgs mockkStatic("ftl.run.platform.android.CreateAndroidTestContextKt") every { runBlocking { any().createAndroidTestContexts() } } returns listOf() - val parsedYml = AndroidArgs.load(yaml) + val parsedYml = AndroidArgs.load(yaml).validate() runBlocking { runAndroidTests(parsedYml) } } @@ -1351,7 +1352,7 @@ AndroidArgs test: $testApk results-dir: $resultsDir """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() if (Files.exists(directoryPath)) { fail("Test directory ($resultsDir) shouldn't be created! It's a remote directory on the cloud!") } @@ -1381,7 +1382,7 @@ AndroidArgs }.forEach { yaml -> assertEquals( OutputStyle.Multi, - AndroidArgs.load(yaml).defaultOutputStyle + AndroidArgs.load(yaml).validate().defaultOutputStyle ) } } @@ -1396,7 +1397,7 @@ AndroidArgs flank: disable-sharding: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() val androidTestConfig = args.createAndroidTestConfig( InstrumentationTestContext( app = "app".asFileReference(), @@ -1420,7 +1421,7 @@ AndroidArgs flank: disable-sharding: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() val androidTestConfig = args.createAndroidTestConfig( InstrumentationTestContext( app = "app".asFileReference(), @@ -1441,7 +1442,7 @@ AndroidArgs flank: max-test-shards: -1 """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE.last, args.maxTestShards) } @@ -1455,7 +1456,7 @@ AndroidArgs max-test-shards: -1 disable-results-upload: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertTrue(args.disableResultsUpload) } @@ -1468,7 +1469,7 @@ AndroidArgs flank: max-test-shards: -1 """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertFalse(args.disableResultsUpload) } @@ -1482,7 +1483,7 @@ AndroidArgs max-test-shards: -1 disable-results-upload: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE.last, args.maxTestShards) } @@ -1496,7 +1497,7 @@ AndroidArgs max-test-shards: 10 disable-results-upload: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(10, args.maxTestShards) } @@ -1510,7 +1511,7 @@ AndroidArgs max-test-shards: ${AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE.last + 1} disable-results-upload: true """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test(expected = FlankConfigurationError::class) @@ -1528,7 +1529,7 @@ AndroidArgs max-test-shards: ${AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE.last + 1} disable-results-upload: true """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1546,7 +1547,7 @@ AndroidArgs max-test-shards: -1 disable-results-upload: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE.last, args.maxTestShards) } @@ -1569,7 +1570,7 @@ AndroidArgs max-test-shards: -1 disable-results-upload: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE.last, args.maxTestShards) } @@ -1593,7 +1594,7 @@ AndroidArgs full-junit-result: true legacy-junit-result: true """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test fun `should not throw when --legacy-junit-result set and --full-junit-result not set`() { @@ -1615,7 +1616,7 @@ AndroidArgs full-junit-result: false legacy-junit-result: true """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1638,7 +1639,7 @@ AndroidArgs full-junit-result: true legacy-junit-result: false """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1663,7 +1664,7 @@ AndroidArgs flank: max-test-shards: -1 """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test(expected = FlankGeneralError::class) @@ -1680,7 +1681,7 @@ AndroidArgs flank: max-test-shards: -1 """.trimIndent() - AndroidArgs.load(yaml) + AndroidArgs.load(yaml).validate() } @Test @@ -1693,7 +1694,7 @@ AndroidArgs max-test-shards: -1 default-test-time: 15 """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(args.defaultTestTime, 15.0, 0.01) } @@ -1707,7 +1708,7 @@ AndroidArgs max-test-shards: -1 use-average-test-time-for-new-tests: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertEquals(args.defaultTestTime, 120.0, 0.01) } @@ -1721,12 +1722,12 @@ AndroidArgs max-test-shards: -1 use-average-test-time-for-new-tests: true """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertTrue(args.useAverageTestTimeForNewTests) } @Test - fun `should useAverageTestTimeForNewTests set to false by defaul`() { + fun `should useAverageTestTimeForNewTests set to false by default`() { val yaml = """ gcloud: app: $appApk @@ -1734,9 +1735,39 @@ AndroidArgs flank: max-test-shards: -1 """.trimIndent() - val args = AndroidArgs.load(yaml) + val args = AndroidArgs.load(yaml).validate() assertFalse(args.useAverageTestTimeForNewTests) } + + @Test + fun `should not throw when other files exists`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + other-files: + /sdcard/dir1/file1.txt: $appApk + /sdcard/dir2/file2.jpg: $testApk + flank: + max-test-shards: -1 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } + + @Test(expected = FlankGeneralError::class) + fun `should throw when other files don't exist`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + other-files: + /sdcard/dir1/file1.txt: $nonExistingApk + /sdcard/dir2/file2.jpg: $nonExistingApk + flank: + max-test-shards: -1 + """.trimIndent() + AndroidArgs.load(yaml).validate() + } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 8ec95219f6..6525ce2986 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -31,6 +31,8 @@ class IosArgsTest { private val empty = emptyList() private val simpleFlankPath = getPath("src/test/kotlin/ftl/fixtures/simple-ios-flank.yml") private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip" + private val nonExistingTestPath = "./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example_non_existing.zip" + private val nonExistingxctestrunFile = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e_non_exis.xctestrun" private val xctestrunFile = "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun" private val invalidApp = "../test_projects/android/apks/invalid.apk" @@ -931,7 +933,18 @@ IosArgs flank: max-test-shards: -1 """.trimIndent() - IosArgs.load(yaml) + IosArgs.load(yaml).validate() + } + + fun `verify no error message when test and xctestrun not set for refresh command`() { + val yaml = """ + gcloud: + test: $nonExistingTestPath + xctestrun-file: $nonExistingTestPath + flank: + max-test-shards: -1 + """.trimIndent() + IosArgs.load(yaml).validateRefresh() } @Test(expected = FlankConfigurationError::class) @@ -942,7 +955,7 @@ IosArgs flank: max-test-shards: -1 """.trimIndent() - IosArgs.load(yaml) + IosArgs.load(yaml).validate() } @Test(expected = FlankConfigurationError::class) @@ -953,7 +966,7 @@ IosArgs flank: max-test-shards: -1 """.trimIndent() - IosArgs.load(yaml) + IosArgs.load(yaml).validate() } @Test @@ -965,7 +978,19 @@ IosArgs flank: max-test-shards: -1 """.trimIndent() - IosArgs.load(yaml) + IosArgs.load(yaml).validate() + } + + @Test + fun `verify no error message when test and xctestrun-file not set and validation is for refresh command`() { + val yaml = """ + gcloud: + test: $nonExistingTestPath + xctestrun-file: $nonExistingxctestrunFile + flank: + max-test-shards: -1 + """.trimIndent() + IosArgs.load(yaml).validateRefresh() } @Test