diff --git a/release_notes.md b/release_notes.md index 91766b9c37..9d39f4fdee 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,6 @@ ## next (unreleased) +- [#831](https://github.com/Flank/flank/pull/831) Refactor config entities and arguments. ([jan-gogo](https://github.com/jan-gogo)) - [#817](https://github.com/Flank/flank/pull/817) Add AndroidTestContext as base data for dump shards & test execution. ([jan-gogo](https://github.com/jan-gogo)) - [#801](https://github.com/Flank/flank/pull/801) Omit missing app apk if additional-app-test-apks specified. ([jan-gogo](https://github.com/jan-gogo)) - [#784](https://github.com/Flank/flank/pull/784) Add output-style option. ([jan-gogo](https://github.com/jan-gogo)) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 7ed26d848b..faa66e1362 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -1,162 +1,28 @@ package ftl.args -import com.google.common.annotations.VisibleForTesting -import ftl.android.AndroidCatalog -import ftl.android.IncompatibleModelVersion -import ftl.android.SupportedDeviceConfig -import ftl.android.UnsupportedModelId -import ftl.android.UnsupportedVersionId -import ftl.args.ArgsHelper.assertCommonProps -import ftl.args.ArgsHelper.assertFileExists -import ftl.args.ArgsHelper.assertGcsFileExists -import ftl.args.ArgsHelper.createGcsBucket -import ftl.args.ArgsHelper.createJunitBucket -import ftl.args.ArgsHelper.evaluateFilePath -import ftl.args.ArgsHelper.mergeYmlMaps -import ftl.args.ArgsHelper.yamlMapper -import ftl.args.ArgsToString.apksToString -import ftl.args.ArgsToString.listToString -import ftl.args.ArgsToString.mapToString -import ftl.args.ArgsToString.objectsToString -import ftl.args.yml.AndroidFlankYml -import ftl.args.yml.AndroidGcloudYml -import ftl.args.yml.FlankYml -import ftl.args.yml.GcloudYml import ftl.args.yml.AppTestPair -import ftl.args.yml.AndroidGcloudYmlParams -import ftl.args.yml.YamlDeprecated -import ftl.util.loadFile -import ftl.cli.firebase.test.android.AndroidRunCommand -import ftl.config.Device -import ftl.config.FtlConstants -import ftl.config.parseRoboDirectives -import ftl.run.status.asOutputStyle -import ftl.util.FlankFatalError -import java.io.File -import ftl.util.uniqueObjectName -import java.io.Reader -import java.nio.file.Path -// set default values, init properties, etc. -class AndroidArgs( - gcloudYml: GcloudYml, - androidGcloudYml: AndroidGcloudYml, - flankYml: FlankYml, - androidFlankYml: AndroidFlankYml, - override val data: String, - val cli: AndroidRunCommand? = null -) : IArgs { +data class AndroidArgs( + val commonArgs: CommonArgs, + val appApk: String?, + val testApk: String?, + val additionalApks: List, + val autoGoogleLogin: Boolean, + val useOrchestrator: Boolean, + val roboDirectives: List, + val roboScript: String?, + val environmentVariables: Map, + val directoriesToPull: List, + val otherFiles: Map, + val performanceMetrics: Boolean, + val numUniformShards: Int?, + val testRunnerClass: String?, + val testTargets: List, + val additionalAppTestApks: List, + override val useLegacyJUnitResult: Boolean +) : IArgs by commonArgs { + companion object : AndroidArgsCompanion() - private val gcloud = gcloudYml.gcloud - override val resultsBucket: String - override val resultsDir = (cli?.resultsDir ?: gcloud.resultsDir) ?: uniqueObjectName() - override val recordVideo = cli?.recordVideo ?: cli?.noRecordVideo?.not() ?: gcloud.recordVideo - override val testTimeout = cli?.timeout ?: gcloud.timeout - override val async = cli?.async ?: gcloud.async - override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName - override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts - - private val androidGcloud = androidGcloudYml.gcloud - var appApk: String? = (cli?.app ?: androidGcloud.app)?.processFilePath("from app") - var testApk: String? = (cli?.test ?: androidGcloud.test)?.processFilePath("from test") - val additionalApks = (cli?.additionalApks ?: androidGcloud.additionalApks).map { it.processFilePath("from additional-apks") } - val autoGoogleLogin = cli?.autoGoogleLogin ?: cli?.noAutoGoogleLogin?.not() ?: androidGcloud.autoGoogleLogin - - // We use not() on noUseOrchestrator because if the flag is on, useOrchestrator needs to be false - val useOrchestrator = cli?.useOrchestrator ?: cli?.noUseOrchestrator?.not() ?: androidGcloud.useOrchestrator - val roboDirectives = cli?.roboDirectives?.parseRoboDirectives() ?: androidGcloud.roboDirectives.parseRoboDirectives() - val roboScript = (cli?.roboScript ?: androidGcloud.roboScript)?.processFilePath("from roboScript") - val environmentVariables = cli?.environmentVariables ?: androidGcloud.environmentVariables - val directoriesToPull = cli?.directoriesToPull ?: androidGcloud.directoriesToPull - val otherFiles = (cli?.otherFiles ?: androidGcloud.otherFiles).map { (devicePath, filePath) -> - devicePath to filePath.processFilePath("from otherFiles") - }.toMap() - val performanceMetrics = cli?.performanceMetrics ?: cli?.noPerformanceMetrics?.not() ?: androidGcloud.performanceMetrics - val numUniformShards = cli?.numUniformShards ?: androidGcloud.numUniformShards - val testRunnerClass = cli?.testRunnerClass ?: androidGcloud.testRunnerClass - val testTargets = cli?.testTargets ?: androidGcloud.testTargets.filterNotNull() - val devices = cli?.device ?: androidGcloud.device - - private val flank = flankYml.flank - override val maxTestShards = convertToShardCount(cli?.maxTestShards ?: flank.maxTestShards) - override val shardTime = cli?.shardTime ?: flank.shardTime - override val repeatTests = cli?.repeatTests ?: flank.repeatTests - override val smartFlankGcsPath = cli?.smartFlankGcsPath ?: flank.smartFlankGcsPath - override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload - override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun - override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload - override val disableSharding = cli?.disableSharding ?: flank.disableSharding - override val project = cli?.project ?: flank.project - override val localResultDir = cli?.localResultsDir ?: flank.localResultsDir - override val runTimeout = cli?.runTimeout ?: flank.runTimeout - override val useLegacyJUnitResult = cli?.useLegacyJUnitResult ?: flank.useLegacyJUnitResult - override val fullJUnitResult = cli?.fullJUnitResult ?: flank.fullJUnitResult - override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails - override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile - override val ignoreFailedTests = cli?.ignoreFailedTests ?: flank.ignoreFailedTests - override val keepFilePath = cli?.keepFilePath ?: flank.keepFilePath - override val outputStyle by lazy { (cli?.outputStyle ?: flank.outputStyle)?.asOutputStyle() ?: defaultOutputStyle } - - private val androidFlank = androidFlankYml.flank - val additionalAppTestApks = (cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks).map { (app, test) -> - AppTestPair( - app = app?.processFilePath("from additional-app-test-apks.app"), - test = test.processFilePath("from additional-app-test-apks.test") - ) - } - override val hasMultipleExecutions: Boolean get() = super.hasMultipleExecutions || additionalAppTestApks.isNotEmpty() - - init { - if (appApk == null) additionalAppTestApks - .filter { (app, _) -> app == null } - .map { File(it.test).name } - .run { - if (isNotEmpty()) throw FlankFatalError("Cannot resolve app apk pair for $this") - } - - resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket) - createJunitBucket(project, flank.smartFlankGcsPath) - - devices.forEach { device -> assertDeviceSupported(device) } - - if (numUniformShards != null && maxTestShards > 1) throw FlankFatalError( - "Option num-uniform-shards cannot be specified along with max-test-shards. Use only one of them." - ) - - if (!(isRoboTest or isInstrumentationTest)) throw FlankFatalError( - "One of following options must be specified [test, robo-directives, robo-script]." - ) - - // Using both roboDirectives and roboScript may hang test execution on FTL - if (roboDirectives.isNotEmpty() && roboScript != null) throw FlankFatalError( - "Options robo-directives and robo-script are mutually exclusive, use only one of them." - ) - - assertCommonProps(this) - - // Call lazy outputStyle inside init to get possible error ASAP - outputStyle - } - - val isInstrumentationTest - get() = appApk != null && testApk != null || - additionalAppTestApks.isNotEmpty() && - (appApk != null || additionalAppTestApks.all { (app, _) -> app != null }) - private val isRoboTest - get() = appApk != null && - (roboDirectives.isNotEmpty() || roboScript != null) - - private fun assertDeviceSupported(device: Device) { - when (val deviceConfigTest = AndroidCatalog.supportedDeviceConfig(device.model, device.version, this.project)) { - SupportedDeviceConfig -> { - } - UnsupportedModelId -> throw RuntimeException("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds(this.project)}") - UnsupportedVersionId -> throw RuntimeException("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds(this.project)}") - is IncompatibleModelVersion -> throw RuntimeException("Incompatible model, '${device.model}', and version, '${device.version}'\nSupported version ids for '${device.model}': $deviceConfigTest") - } - } - - // Note: environmentVariables may contain secrets and are not printed for security reasons. override fun toString(): String { return """ AndroidArgs @@ -166,24 +32,24 @@ AndroidArgs record-video: $recordVideo timeout: $testTimeout async: $async - client-details: ${mapToString(clientDetails)} + client-details: ${ArgsToString.mapToString(clientDetails)} network-profile: $networkProfile results-history-name: $resultsHistoryName # Android gcloud app: $appApk test: $testApk - additional-apks: ${listToString(additionalApks)} + additional-apks: ${ArgsToString.listToString(additionalApks)} auto-google-login: $autoGoogleLogin use-orchestrator: $useOrchestrator - directories-to-pull:${listToString(directoriesToPull)} - other-files:${mapToString(otherFiles)} + directories-to-pull:${ArgsToString.listToString(directoriesToPull)} + other-files:${ArgsToString.mapToString(otherFiles)} performance-metrics: $performanceMetrics num-uniform-shards: $numUniformShards test-runner-class: $testRunnerClass - test-targets:${listToString(testTargets)} - robo-directives:${objectsToString(roboDirectives)} + test-targets:${ArgsToString.listToString(testTargets)} + robo-directives:${ArgsToString.objectsToString(roboDirectives)} robo-script: $roboScript - device:${objectsToString(devices)} + device:${ArgsToString.objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: @@ -192,62 +58,28 @@ AndroidArgs num-test-runs: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath smart-flank-disable-upload: $smartFlankDisableUpload - files-to-download:${listToString(filesToDownload)} - test-targets-always-run:${listToString(testTargetsAlwaysRun)} + files-to-download:${ArgsToString.listToString(filesToDownload)} + test-targets-always-run:${ArgsToString.listToString(testTargetsAlwaysRun)} disable-sharding: $disableSharding project: $project local-result-dir: $localResultDir full-junit-result: $fullJUnitResult # Android Flank Yml keep-file-path: $keepFilePath - additional-app-test-apks:${apksToString(additionalAppTestApks)} + additional-app-test-apks:${ArgsToString.apksToString(additionalAppTestApks)} run-timeout: $runTimeout legacy-junit-result: $useLegacyJUnitResult ignore-failed-tests: $ignoreFailedTests output-style: ${outputStyle.name.toLowerCase()} """.trimIndent() } - - companion object : IArgsCompanion { - override val validArgs by lazy { - mergeYmlMaps(GcloudYml, AndroidGcloudYml, FlankYml, AndroidFlankYml) - } - - fun load(yamlPath: Path, cli: AndroidRunCommand? = null): AndroidArgs = load(loadFile(yamlPath), cli) - - @VisibleForTesting - internal fun load(yamlReader: Reader, cli: AndroidRunCommand? = null): AndroidArgs { - - val data = YamlDeprecated.modifyAndThrow(yamlReader, android = true) - val flankYml = yamlMapper.readValue(data, FlankYml::class.java) - val gcloudYml = yamlMapper.readValue(data, GcloudYml::class.java) - val androidGcloudYml = yamlMapper.readValue(data, AndroidGcloudYml::class.java) - val androidFlankYml = yamlMapper.readValue(data, AndroidFlankYml::class.java) - - return AndroidArgs( - gcloudYml, - androidGcloudYml, - flankYml, - androidFlankYml, - data, - cli - ) - } - - fun default(): AndroidArgs { - return AndroidArgs( - GcloudYml(), - AndroidGcloudYml(AndroidGcloudYmlParams(app = ".", test = ".")), - FlankYml(), - AndroidFlankYml(), - "", - AndroidRunCommand() - ) - } - } } -private fun String.processFilePath(name: String): String = - if (startsWith(FtlConstants.GCS_PREFIX)) - this.also { assertGcsFileExists(it) } else - evaluateFilePath(this).also { assertFileExists(it, name) } +val AndroidArgs.isInstrumentationTest + get() = appApk != null && testApk != null || + additionalAppTestApks.isNotEmpty() && + (appApk != null || additionalAppTestApks.all { (app, _) -> app != null }) + +val AndroidArgs.isRoboTest + get() = appApk != null && + (roboDirectives.isNotEmpty() || roboScript != null) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt new file mode 100644 index 0000000000..b8a4b8168e --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgsCompanion.kt @@ -0,0 +1,43 @@ +package ftl.args + +import com.google.common.annotations.VisibleForTesting +import ftl.args.yml.mergeYmlKeys +import ftl.cli.firebase.test.android.AndroidRunCommand +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.defaultAndroidConfig +import ftl.config.loadAndroidConfig +import ftl.config.plus +import ftl.util.loadFile +import java.io.Reader +import java.nio.file.Path + +open class AndroidArgsCompanion : IArgs.ICompanion { + override val validArgs by lazy { + mergeYmlKeys( + CommonGcloudConfig, + AndroidGcloudConfig, + CommonFlankConfig, + AndroidFlankConfig + ) + } + + fun default() = + createAndroidArgs(defaultAndroidConfig()) + + fun load(yamlPath: Path, cli: AndroidRunCommand? = null) = + load(loadFile(yamlPath), cli) + + @VisibleForTesting + internal fun load(yamlReader: Reader, cli: AndroidRunCommand? = null) = + createAndroidArgs( + config = defaultAndroidConfig() + + loadAndroidConfig(reader = yamlReader) + + cli?.config + ).apply { + commonArgs.validate() + this.validate() + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index d3163a4d57..967cf2dad2 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -11,7 +11,6 @@ import com.google.cloud.storage.Storage import com.google.cloud.storage.StorageClass import com.google.cloud.storage.StorageOptions import ftl.args.IArgs.Companion.AVAILABLE_SHARD_COUNT_RANGE -import ftl.args.yml.IYmlMap import ftl.args.yml.YamlObjectMapper import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX @@ -41,17 +40,6 @@ object ArgsHelper { YamlObjectMapper().registerKotlinModule() } - fun mergeYmlMaps(vararg ymlMaps: IYmlMap): Map> { - val result = mutableMapOf>() - ymlMaps.map { it.map } - .forEach { map -> - map.forEach { (k, v) -> - result.merge(k, v) { a, b -> a + b } - } - } - return result - } - fun assertFileExists(file: String, name: String) { if (!File(file).exists()) { throw FlankFatalError("'$file' $name doesn't exist") @@ -109,11 +97,7 @@ object ArgsHelper { val bucket = gcsURI.authority val path = gcsURI.path.drop(1) // Drop leading slash - val blob = GcStorage.storage.get(bucket, path) - - if (blob == null) { - throw FlankFatalError("The file at '$uri' does not exist") - } + GcStorage.storage.get(bucket, path) ?: throw FlankFatalError("The file at '$uri' does not exist") } fun validateTestMethods( @@ -258,3 +242,8 @@ object ArgsHelper { return shards } } + +fun String.processFilePath(name: String): String = + if (startsWith(GCS_PREFIX)) + this.also { ArgsHelper.assertGcsFileExists(it) } else + ArgsHelper.evaluateFilePath(this).also { ArgsHelper.assertFileExists(it, name) } diff --git a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt new file mode 100644 index 0000000000..165e173bda --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt @@ -0,0 +1,37 @@ +package ftl.args + +import ftl.config.Device +import ftl.run.status.OutputStyle + +data class CommonArgs( + override val data: String, + + // Gcloud + override val devices: List, + override val resultsBucket: String, + override val resultsDir: String, + override val recordVideo: Boolean, + override val testTimeout: String, + override val async: Boolean, + override val resultsHistoryName: String?, + override val flakyTestAttempts: Int, + override val clientDetails: Map?, + override val networkProfile: String?, + + // flank + override val project: String, + override val maxTestShards: Int, + override val shardTime: Int, + override val repeatTests: Int, + override val smartFlankGcsPath: String, + override val smartFlankDisableUpload: Boolean, + override val testTargetsAlwaysRun: List, + override val filesToDownload: List, + override val disableSharding: Boolean, + override val localResultDir: String, + override val runTimeout: String, + override val fullJUnitResult: Boolean, + override val ignoreFailedTests: Boolean, + override val keepFilePath: Boolean, + override val outputStyle: OutputStyle +) : IArgs diff --git a/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt new file mode 100644 index 0000000000..b3dabad09f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/CreateAndroidArgs.kt @@ -0,0 +1,40 @@ +package ftl.args + +import ftl.args.yml.AppTestPair +import ftl.config.AndroidConfig +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig + +fun createAndroidArgs( + config: AndroidConfig? = null, + gcloud: AndroidGcloudConfig = config!!.platform.gcloud, + flank: AndroidFlankConfig = config!!.platform.flank, + commonArgs: CommonArgs = config!!.common.createCommonArgs(config.data) +) = AndroidArgs( + commonArgs = commonArgs, + + // gcloud + appApk = gcloud.app?.processFilePath("from app"), + testApk = gcloud.test?.processFilePath("from test"), + useOrchestrator = gcloud.useOrchestrator!!, + testTargets = gcloud.testTargets!!.filterNotNull(), + testRunnerClass = gcloud.testRunnerClass, + roboDirectives = gcloud.roboDirectives!!.parseRoboDirectives(), + performanceMetrics = gcloud.performanceMetrics!!, + otherFiles = gcloud.otherFiles!!.mapValues { (_, path) -> path.processFilePath("from otherFiles") }, + numUniformShards = gcloud.numUniformShards, + environmentVariables = gcloud.environmentVariables!!, + directoriesToPull = gcloud.directoriesToPull!!, + autoGoogleLogin = gcloud.autoGoogleLogin!!, + additionalApks = gcloud.additionalApks!!.map { it.processFilePath("from additional-apks") }, + roboScript = gcloud.roboScript?.processFilePath("from roboScript"), + + // flank + additionalAppTestApks = flank.additionalAppTestApks?.map { (app, test) -> + AppTestPair( + app = app?.processFilePath("from additional-app-test-apks.app"), + test = test.processFilePath("from additional-app-test-apks.test") + ) + } ?: emptyList(), + useLegacyJUnitResult = flank.useLegacyJUnitResult!! +) diff --git a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt new file mode 100644 index 0000000000..226fcdb39f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt @@ -0,0 +1,65 @@ +package ftl.args + +import ftl.config.CommonConfig +import ftl.run.status.OutputStyle +import ftl.run.status.asOutputStyle +import ftl.util.uniqueObjectName + +fun CommonConfig.createCommonArgs( + data: String +) = CommonArgs( + data = data, + + // gcloud + devices = gcloud.devices!!, + resultsBucket = ArgsHelper.createGcsBucket( + projectId = flank.project!!, + bucket = gcloud.resultsBucket!! + ), + resultsDir = gcloud.resultsDir ?: uniqueObjectName(), + recordVideo = gcloud.recordVideo!!, + testTimeout = gcloud.timeout!!, + async = gcloud.async!!, + resultsHistoryName = gcloud.resultsHistoryName, + flakyTestAttempts = gcloud.flakyTestAttempts!!, + networkProfile = gcloud.networkProfile, + clientDetails = gcloud.clientDetails, + + // flank + maxTestShards = convertToShardCount(flank.maxTestShards!!), + shardTime = flank.shardTime!!, + repeatTests = flank.repeatTests!!, + smartFlankGcsPath = flank.smartFlankGcsPath!!, + smartFlankDisableUpload = flank.smartFlankDisableUpload!!, + testTargetsAlwaysRun = flank.testTargetsAlwaysRun!!, + runTimeout = flank.runTimeout!!, + fullJUnitResult = flank.fullJUnitResult!!, + project = flank.project!!, + outputStyle = outputStyle, + keepFilePath = flank.keepFilePath!!, + ignoreFailedTests = flank.ignoreFailedTests!!, + filesToDownload = flank.filesToDownload!!, + disableSharding = flank.disableSharding!!, + localResultDir = flank.localResultsDir!! +).apply { + ArgsHelper.createJunitBucket(project, smartFlankGcsPath) +} + +private val CommonConfig.outputStyle + get() = flank.outputStyle + ?.asOutputStyle() + ?: defaultOutputStyle + +private val CommonConfig.defaultOutputStyle + get() = if (hasMultipleExecutions) + OutputStyle.Multi else + OutputStyle.Verbose + +private val CommonConfig.hasMultipleExecutions + get() = gcloud.flakyTestAttempts!! > 0 || + (!flank.disableSharding!! && flank.maxTestShards!! > 0) + +private fun convertToShardCount(inputValue: Int): Int = + if (inputValue != -1) + inputValue else + IArgs.AVAILABLE_SHARD_COUNT_RANGE.last diff --git a/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt new file mode 100644 index 0000000000..b5b59df648 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/CreateIosArgs.kt @@ -0,0 +1,25 @@ +package ftl.args + +import ftl.config.IosConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig + +fun createIosArgs( + config: IosConfig +) = createIosArgs( + gcloud = config.platform.gcloud, + flank = config.platform.flank, + commonArgs = config.common.createCommonArgs(config.data) +) + +private fun createIosArgs( + gcloud: IosGcloudConfig, + flank: IosFlankConfig, + commonArgs: CommonArgs +) = IosArgs( + commonArgs = commonArgs, + xctestrunZip = gcloud.test!!.processFilePath("from test"), + xctestrunFile = gcloud.xctestrunFile!!.processFilePath("from xctestrun-file"), + xcodeVersion = gcloud.xcodeVersion, + testTargets = flank.testTargets!!.filterNotNull() +) diff --git a/test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt b/test_runner/src/main/kotlin/ftl/args/FlankRoboDirective.kt similarity index 97% rename from test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt rename to test_runner/src/main/kotlin/ftl/args/FlankRoboDirective.kt index 69cdbd8476..a17c36d3c0 100644 --- a/test_runner/src/main/kotlin/ftl/config/FlankRoboDirective.kt +++ b/test_runner/src/main/kotlin/ftl/args/FlankRoboDirective.kt @@ -1,4 +1,4 @@ -package ftl.config +package ftl.args data class FlankRoboDirective( val type: String, diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 0bd8a3a732..1821994b88 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -1,6 +1,7 @@ package ftl.args -import ftl.args.yml.FlankYmlParams +import ftl.config.Device +import ftl.config.common.CommonFlankConfig.Companion.defaultLocalResultsDir import ftl.run.status.OutputStyle import ftl.util.timeoutToMils @@ -10,6 +11,7 @@ interface IArgs { val data: String // GcloudYml + val devices: List val resultsBucket: String val resultsDir: String val recordVideo: Boolean @@ -49,16 +51,14 @@ interface IArgs { val hasMultipleExecutions get() = flakyTestAttempts > 0 || (!disableSharding && maxTestShards > 0) - fun useLocalResultDir() = localResultDir != FlankYmlParams.defaultLocalResultsDir - - fun convertToShardCount(inputValue: Int): Int = if (inputValue == -1) { - AVAILABLE_SHARD_COUNT_RANGE.last - } else { - inputValue - } + fun useLocalResultDir() = localResultDir != defaultLocalResultsDir companion object { // num_shards must be >= 1, and <= 50 val AVAILABLE_SHARD_COUNT_RANGE = 1..50 } + + interface ICompanion { + val validArgs: Map> + } } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/IArgsCompanion.kt deleted file mode 100644 index 4d5cd6cc57..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/IArgsCompanion.kt +++ /dev/null @@ -1,5 +0,0 @@ -package ftl.args - -interface IArgsCompanion { - val validArgs: Map> -} diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 8d87d34597..20ba24c3e3 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -1,124 +1,21 @@ package ftl.args import com.google.common.annotations.VisibleForTesting -import ftl.args.ArgsHelper.assertCommonProps -import ftl.args.ArgsHelper.assertFileExists -import ftl.args.ArgsHelper.assertGcsFileExists -import ftl.args.ArgsHelper.createGcsBucket -import ftl.args.ArgsHelper.createJunitBucket -import ftl.args.ArgsHelper.evaluateFilePath -import ftl.args.ArgsHelper.mergeYmlMaps -import ftl.args.ArgsHelper.yamlMapper -import ftl.args.ArgsToString.listToString -import ftl.args.ArgsToString.mapToString -import ftl.args.ArgsToString.objectsToString -import ftl.args.yml.FlankYml -import ftl.args.yml.GcloudYml -import ftl.args.yml.IosFlankYml -import ftl.args.yml.IosGcloudYml -import ftl.args.yml.IosGcloudYmlParams -import ftl.args.yml.YamlDeprecated -import ftl.cli.firebase.test.ios.IosRunCommand -import ftl.config.Device -import ftl.config.FtlConstants -import ftl.ios.IosCatalog -import ftl.ios.Xctestrun -import ftl.run.status.asOutputStyle -import ftl.util.FlankFatalError +import ftl.ios.Xctestrun.findTestNames import ftl.util.FlankTestMethod -import ftl.util.loadFile -import ftl.util.uniqueObjectName -import java.io.Reader -import java.nio.file.Path -class IosArgs( - gcloudYml: GcloudYml, - iosGcloudYml: IosGcloudYml, - flankYml: FlankYml, - iosFlankYml: IosFlankYml, - override val data: String, - val cli: IosRunCommand? = null -) : IArgs { +data class IosArgs( + val commonArgs: CommonArgs, + val xctestrunZip: String, + val xctestrunFile: String, + val xcodeVersion: String?, + val testTargets: List +) : IArgs by commonArgs { - private val gcloud = gcloudYml.gcloud - override val resultsBucket: String - override val resultsDir = (cli?.resultsDir ?: gcloud.resultsDir) ?: uniqueObjectName() - override val recordVideo = cli?.recordVideo ?: cli?.noRecordVideo?.not() ?: gcloud.recordVideo - override val testTimeout = cli?.timeout ?: gcloud.timeout - override val async = cli?.async ?: gcloud.async - override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName - override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts + override val useLegacyJUnitResult = true + val testShardChunks: ShardChunks by lazy { calculateShardChunks() } - private val iosGcloud = iosGcloudYml.gcloud - var xctestrunZip = cli?.test ?: iosGcloud.test ?: throw FlankFatalError("test is not set") - var xctestrunFile = cli?.xctestrunFile ?: iosGcloud.xctestrunFile ?: throw FlankFatalError("xctestrun-file is not set") - val xcodeVersion = cli?.xcodeVersion ?: iosGcloud.xcodeVersion - val devices = cli?.device ?: iosGcloud.device - - private val flank = flankYml.flank - override val maxTestShards = convertToShardCount(cli?.maxTestShards ?: flank.maxTestShards) - override val shardTime = cli?.shardTime ?: flank.shardTime - override val repeatTests = cli?.repeatTests ?: flank.repeatTests - override val smartFlankGcsPath = cli?.smartFlankGcsPath ?: flank.smartFlankGcsPath - override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload - override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun - override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload - override val disableSharding = cli?.disableSharding ?: flank.disableSharding - override val project = cli?.project ?: flank.project - override val localResultDir = cli?.localResultsDir ?: flank.localResultsDir - override val runTimeout = cli?.runTimeout ?: flank.runTimeout - override val useLegacyJUnitResult = true // currently, FTL does not provide API based results for iOS - override val fullJUnitResult = cli?.fullJUnitResult ?: flank.fullJUnitResult - override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails - override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile - override val ignoreFailedTests = cli?.ignoreFailedTests ?: flank.ignoreFailedTests - override val keepFilePath = cli?.keepFilePath ?: flank.keepFilePath - override val outputStyle = (cli?.outputStyle ?: flank.outputStyle)?.asOutputStyle() ?: defaultOutputStyle - - private val iosFlank = iosFlankYml.flank - val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull() - - // computed properties not specified in yaml - val testShardChunks: ShardChunks by lazy { - if (disableSharding) return@lazy listOf(emptyList()) - - val validTestMethods = Xctestrun.findTestNames(xctestrunFile) - val testsToShard = filterTests(validTestMethods, testTargets).distinct().map { FlankTestMethod(it, ignored = false) } - - ArgsHelper.calculateShards(testsToShard, this) - } - - init { - resultsBucket = createGcsBucket(project, cli?.resultsBucket ?: gcloud.resultsBucket) - createJunitBucket(project, flank.smartFlankGcsPath) - - if (xctestrunZip.startsWith(FtlConstants.GCS_PREFIX)) { - assertGcsFileExists(xctestrunZip) - } else { - xctestrunZip = evaluateFilePath(xctestrunZip) - assertFileExists(xctestrunZip, "xctestrunZip") - } - xctestrunFile = evaluateFilePath(xctestrunFile) - assertFileExists(xctestrunFile, "xctestrunFile") - - devices.forEach { device -> assertDeviceSupported(device) } - assertXcodeSupported(xcodeVersion) - - assertCommonProps(this) - } - - private fun assertXcodeSupported(xcodeVersion: String?) { - if (xcodeVersion == null) return - if (!IosCatalog.supportedXcode(xcodeVersion, this.project)) { - throw FlankFatalError(("Xcode $xcodeVersion is not a supported Xcode version")) - } - } - - private fun assertDeviceSupported(device: Device) { - if (!IosCatalog.supportedDevice(device.model, device.version, this.project)) { - throw FlankFatalError("iOS ${device.version} on ${device.model} is not a supported device") - } - } + companion object : IosArgsCompanion() override fun toString(): String { return """ @@ -129,14 +26,14 @@ IosArgs record-video: $recordVideo timeout: $testTimeout async: $async - client-details: ${mapToString(clientDetails)} + client-details: ${ArgsToString.mapToString(clientDetails)} network-profile: $networkProfile results-history-name: $resultsHistoryName # iOS gcloud test: $xctestrunZip xctestrun-file: $xctestrunFile xcode-version: $xcodeVersion - device:${objectsToString(devices)} + device:${ArgsToString.objectsToString(devices)} num-flaky-test-attempts: $flakyTestAttempts flank: @@ -145,12 +42,12 @@ IosArgs num-test-runs: $repeatTests smart-flank-gcs-path: $smartFlankGcsPath smart-flank-disable-upload: $smartFlankDisableUpload - test-targets-always-run:${listToString(testTargetsAlwaysRun)} - files-to-download:${listToString(filesToDownload)} + test-targets-always-run:${ArgsToString.listToString(testTargetsAlwaysRun)} + files-to-download:${ArgsToString.listToString(filesToDownload)} keep-file-path: $keepFilePath full-junit-result: $fullJUnitResult # iOS flank - test-targets:${listToString(testTargets)} + test-targets:${ArgsToString.listToString(testTargets)} disable-sharding: $disableSharding project: $project local-result-dir: $localResultDir @@ -159,47 +56,19 @@ IosArgs output-style: ${outputStyle.name.toLowerCase()} """.trimIndent() } - - companion object : IArgsCompanion { - override val validArgs by lazy { - mergeYmlMaps(GcloudYml, IosGcloudYml, FlankYml, IosFlankYml) - } - - fun load(yamlPath: Path, cli: IosRunCommand? = null): IosArgs = load(loadFile(yamlPath), cli) - - @VisibleForTesting - internal fun load(yamlReader: Reader, cli: IosRunCommand? = null): IosArgs { - val data = YamlDeprecated.modifyAndThrow(yamlReader, android = false) - - val flankYml = yamlMapper.readValue(data, FlankYml::class.java) - val iosFlankYml = yamlMapper.readValue(data, IosFlankYml::class.java) - val gcloudYml = yamlMapper.readValue(data, GcloudYml::class.java) - val iosGcloudYml = yamlMapper.readValue(data, IosGcloudYml::class.java) - - return IosArgs( - gcloudYml, - iosGcloudYml, - flankYml, - iosFlankYml, - data, - cli - ) - } - - fun default(): IosArgs { - return IosArgs( - GcloudYml(), - IosGcloudYml(IosGcloudYmlParams(test = ".", xctestrunFile = ".")), - FlankYml(), - IosFlankYml(), - "", - IosRunCommand() - ) - } - } } -fun filterTests(validTestMethods: List, testTargetsRgx: List): List { +private fun IosArgs.calculateShardChunks() = if (disableSharding) + listOf(emptyList()) else + ArgsHelper.calculateShards( + filteredTests = filterTests(findTestNames(xctestrunFile), testTargets) + .distinct() + .map { FlankTestMethod(it, ignored = false) }, + args = this + ) + +@VisibleForTesting +internal fun filterTests(validTestMethods: List, testTargetsRgx: List): List { if (testTargetsRgx.isEmpty()) { return validTestMethods } diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt new file mode 100644 index 0000000000..cab36b5b36 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/IosArgsCompanion.kt @@ -0,0 +1,43 @@ +package ftl.args + +import com.google.common.annotations.VisibleForTesting +import ftl.args.yml.mergeYmlKeys +import ftl.cli.firebase.test.ios.IosRunCommand +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.defaultIosConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig +import ftl.config.loadIosConfig +import ftl.config.plus +import ftl.util.loadFile +import java.io.Reader +import java.nio.file.Path + +open class IosArgsCompanion : IArgs.ICompanion { + override val validArgs by lazy { + mergeYmlKeys( + CommonGcloudConfig, + IosGcloudConfig, + CommonFlankConfig, + IosFlankConfig + ) + } + + fun default() = + createIosArgs(defaultIosConfig()) + + fun load(yamlPath: Path, cli: IosRunCommand? = null) = + load(loadFile(yamlPath), cli) + + @VisibleForTesting + internal fun load(yamlReader: Reader, cli: IosRunCommand? = null) = + createIosArgs( + config = defaultIosConfig() + + loadIosConfig(reader = yamlReader) + + cli?.config + ).apply { + commonArgs.validate() + this.validate() + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt new file mode 100644 index 0000000000..89ff24d1af --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -0,0 +1,59 @@ +package ftl.args + +import ftl.android.AndroidCatalog +import ftl.android.IncompatibleModelVersion +import ftl.android.SupportedDeviceConfig +import ftl.android.UnsupportedModelId +import ftl.android.UnsupportedVersionId +import ftl.util.FlankFatalError +import java.io.File + +fun AndroidArgs.validate() { + assertAdditionalAppTestApks() + assertDevicesSupported() + assertShards() + assertTestTypes() + assertRoboTest() +} + +private fun AndroidArgs.assertAdditionalAppTestApks() { + if (appApk == null) additionalAppTestApks + .filter { (app, _) -> app == null } + .map { File(it.test).name } + .run { + if (isNotEmpty()) + throw FlankFatalError("Cannot resolve app apk pair for $this") + } +} + +private fun AndroidArgs.assertDevicesSupported() = devices + .associateWith { device -> + AndroidCatalog.supportedDeviceConfig(device.model, device.version, project) + } + .forEach { (device, check) -> + when (check) { + SupportedDeviceConfig -> Unit + UnsupportedModelId -> throw FlankFatalError("Unsupported model id, '${device.model}'\nSupported model ids: ${AndroidCatalog.androidModelIds(project)}") + UnsupportedVersionId -> throw FlankFatalError("Unsupported version id, '${device.version}'\nSupported Version ids: ${AndroidCatalog.androidVersionIds(project)}") + IncompatibleModelVersion -> throw FlankFatalError("Incompatible model, '${device.model}', and version, '${device.version}'\nSupported version ids for '${device.model}': $check") + } + } + +private fun AndroidArgs.assertShards() { + if (numUniformShards != null && maxTestShards > 1) throw FlankFatalError( + "Option num-uniform-shards cannot be specified along with max-test-shards. Use only one of them." + ) +} + +private fun AndroidArgs.assertTestTypes() { + if (!(isRoboTest or isInstrumentationTest)) throw FlankFatalError( + "One of following options must be specified [test, robo-directives, robo-script]." + ) +} + +private fun AndroidArgs.assertRoboTest() { + // Using both roboDirectives and roboScript may hang test execution on FTL + if (roboDirectives.isNotEmpty() && roboScript != null) throw FlankFatalError( + "Options robo-directives and robo-script are mutually exclusive, use only one of them." + ) +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt new file mode 100644 index 0000000000..95ee078754 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt @@ -0,0 +1,55 @@ +package ftl.args + +import ftl.config.FtlConstants +import ftl.util.FlankFatalError + +fun CommonArgs.validate() { + assertProjectId() + assertMaxTestShards() + assertShardTime() + assertRepeatTests() + assertSmartFlankGcsPath() +} + +private fun CommonArgs.assertProjectId() { + if (project.isEmpty()) throw FlankFatalError( + "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + + "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" + ) +} + +private fun CommonArgs.assertMaxTestShards() { + if ( + maxTestShards !in IArgs.AVAILABLE_SHARD_COUNT_RANGE && + maxTestShards != -1 + ) throw FlankFatalError( + "max-test-shards must be >= ${IArgs.AVAILABLE_SHARD_COUNT_RANGE.first} and <= ${IArgs.AVAILABLE_SHARD_COUNT_RANGE.last}, or -1. But current is $maxTestShards" + ) +} + +private fun CommonArgs.assertShardTime() { + if (shardTime <= 0 && shardTime != -1) throw FlankFatalError( + "shard-time must be >= 1 or -1" + ) +} + +private fun CommonArgs.assertRepeatTests() { + if (repeatTests < 1) throw FlankFatalError( + "num-test-runs must be >= 1" + ) +} + +private fun CommonArgs.assertSmartFlankGcsPath() = with(smartFlankGcsPath) { + when { + isEmpty() -> Unit + + startsWith(FtlConstants.GCS_PREFIX).not() -> throw FlankFatalError( + "smart-flank-gcs-path must start with gs://" + ) + + count { it == '/' } <= 2 || endsWith(".xml").not() -> throw FlankFatalError( + "smart-flank-gcs-path must be in the format gs://bucket/foo.xml" + ) + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt new file mode 100644 index 0000000000..25c3bfaa3c --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -0,0 +1,20 @@ +package ftl.args + +import ftl.ios.IosCatalog +import ftl.util.FlankFatalError + +fun IosArgs.validate() { + assertXcodeSupported() + assertDevicesSupported() +} + +private fun IosArgs.assertXcodeSupported() = when { + xcodeVersion == null -> Unit + IosCatalog.supportedXcode(xcodeVersion, project) -> Unit + else -> throw FlankFatalError(("Xcode $xcodeVersion is not a supported Xcode version")) +} + +private fun IosArgs.assertDevicesSupported() = devices.forEach { device -> + if (!IosCatalog.supportedDevice(device.model, device.version, this.project)) + throw FlankFatalError("iOS ${device.version} on ${device.model} is not a supported device") +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt deleted file mode 100644 index 73d150526a..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidFlankYml.kt +++ /dev/null @@ -1,32 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty - -data class AppTestPair( - val app: String?, - val test: String -) - -/** Flank specific parameters for Android */ -@JsonIgnoreProperties(ignoreUnknown = true) -class AndroidFlankYmlParams( - @field:JsonProperty("additional-app-test-apks") - val additionalAppTestApks: List = emptyList() -) { - companion object : IYmlKeys { - override val keys = listOf("additional-app-test-apks") - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class AndroidFlankYml( - @field:JsonProperty("flank") - private val parsedFlank: AndroidFlankYmlParams? = AndroidFlankYmlParams() -) { - val flank = parsedFlank ?: AndroidFlankYmlParams() - - companion object : IYmlMap { - override val map = mapOf("flank" to AndroidFlankYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt deleted file mode 100644 index c8454e4a20..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt +++ /dev/null @@ -1,87 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import ftl.config.Device -import ftl.config.FlankDefaults -import ftl.config.FtlConstants.defaultAndroidModel -import ftl.config.FtlConstants.defaultAndroidVersion - -/** - * Android specific gcloud parameters - * - * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run - */ -@JsonIgnoreProperties(ignoreUnknown = true) -class AndroidGcloudYmlParams( - val app: String? = null, - val test: String? = null, - - @field:JsonProperty("additional-apks") - val additionalApks: List = emptyList(), - - @field:JsonProperty("auto-google-login") - val autoGoogleLogin: Boolean = FlankDefaults.DISABLE_AUTO_LOGIN, - - @field:JsonProperty("use-orchestrator") - val useOrchestrator: Boolean = true, - - @field:JsonProperty("environment-variables") - val environmentVariables: Map = emptyMap(), - - @field:JsonProperty("directories-to-pull") - val directoriesToPull: List = emptyList(), - - @field:JsonProperty("other-files") - val otherFiles: Map = emptyMap(), - - @field:JsonProperty("performance-metrics") - val performanceMetrics: Boolean = FlankDefaults.DISABLE_PERFORMANCE_METRICS, - - @field:JsonProperty("num-uniform-shards") - val numUniformShards: Int? = null, - - @field:JsonProperty("test-runner-class") - val testRunnerClass: String? = null, - - @field:JsonProperty("test-targets") - val testTargets: List = emptyList(), - - @field:JsonProperty("robo-directives") - val roboDirectives: Map = emptyMap(), - - @field:JsonProperty("robo-script") - val roboScript: String? = null, - - val device: List = listOf(Device(defaultAndroidModel, defaultAndroidVersion)) -) { - companion object : IYmlKeys { - override val keys = listOf( - "app", - "test", - "additional-apks", - "auto-google-login", - "use-orchestrator", - "environment-variables", - "directories-to-pull", - "other-files", - "performance-metrics", - "num-uniform-shards", - "test-runner-class", - "test-targets", - "robo-directives", - "robo-script", - "device" - ) - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class AndroidGcloudYml( - val gcloud: AndroidGcloudYmlParams = AndroidGcloudYmlParams() - -) { - companion object : IYmlMap { - override val map = mapOf("gcloud" to AndroidGcloudYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt new file mode 100644 index 0000000000..1fa447bd6f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/AppTestPair.kt @@ -0,0 +1,6 @@ +package ftl.args.yml + +data class AppTestPair( + val app: String?, + val test: String +) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt deleted file mode 100644 index 7d25ae0634..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt +++ /dev/null @@ -1,92 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import ftl.args.ArgsHelper -import ftl.config.FtlConstants - -/** Flank specific parameters for both iOS and Android */ -@JsonIgnoreProperties(ignoreUnknown = true) -class FlankYmlParams( - @field:JsonProperty("max-test-shards") - val maxTestShards: Int = 1, - - @field:JsonProperty("shard-time") - val shardTime: Int = -1, - - @field:JsonProperty("num-test-runs") - val repeatTests: Int = 1, - - @field:JsonProperty("smart-flank-gcs-path") - val smartFlankGcsPath: String = "", - - @field:JsonProperty("smart-flank-disable-upload") - val smartFlankDisableUpload: Boolean = false, - - @field:JsonProperty("disable-sharding") - val disableSharding: Boolean = false, - - @field:JsonProperty("test-targets-always-run") - val testTargetsAlwaysRun: List = emptyList(), - - @field:JsonProperty("files-to-download") - val filesToDownload: List = emptyList(), - - val project: String = ArgsHelper.getDefaultProjectId() ?: "", - - @field:JsonProperty("local-result-dir") - val localResultsDir: String = defaultLocalResultsDir, - - @field:JsonProperty("run-timeout") - val runTimeout: String = FtlConstants.runTimeout, - - @field:JsonProperty("legacy-junit-result") - val useLegacyJUnitResult: Boolean = false, - - @field:JsonProperty("full-junit-result") - val fullJUnitResult: Boolean = false, - - @field:JsonProperty("ignore-failed-tests") - val ignoreFailedTests: Boolean = false, - - @field:JsonProperty("keep-file-path") - val keepFilePath: Boolean = false, - - @field:JsonProperty("output-style") - val outputStyle: String? = null -) { - companion object : IYmlKeys { - override val keys = listOf( - "max-test-shards", - "shard-time", - "num-test-runs", - "smart-flank-gcs-path", - "smart-flank-disable-upload", - "disable-sharding", - "test-targets-always-run", - "files-to-download", - "project", - "local-result-dir", - "run-timeout", - "legacy-junit-result", - "ignore-failed-tests", - "keep-file-path", - "output-style" - ) - - const val defaultLocalResultsDir = "results" - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class FlankYml( - // when an empty 'flank:' is present in a yaml then parsedFlank will be parsed as null. - @field:JsonProperty("flank") - private val parsedFlank: FlankYmlParams? = FlankYmlParams() -) { - val flank = parsedFlank ?: FlankYmlParams() - - companion object : IYmlMap { - override val map = mapOf("flank" to FlankYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt deleted file mode 100644 index a73cf7807d..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt +++ /dev/null @@ -1,54 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import ftl.config.FlankDefaults - -/** - * Common Gcloud parameters shared between iOS and Android - * - * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run - * https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run - */ -@JsonIgnoreProperties(ignoreUnknown = true) -class GcloudYmlParams( - @field:JsonProperty("results-bucket") - var resultsBucket: String = "", - - @field:JsonProperty("results-dir") - var resultsDir: String? = null, - - @field:JsonProperty("record-video") - val recordVideo: Boolean = FlankDefaults.DISABLE_VIDEO_RECORDING, - - val timeout: String = "15m", - - val async: Boolean = false, - - @field:JsonProperty("client-details") - val clientDetails: Map? = null, - - @field:JsonProperty("network-profile") - val networkProfile: String? = null, - - @field:JsonProperty("results-history-name") - val resultsHistoryName: String? = null, - - @field:JsonProperty("num-flaky-test-attempts") - val flakyTestAttempts: Int = 0 -) { - companion object : IYmlKeys { - override val keys = - listOf("results-bucket", "results-dir", "record-video", "timeout", "async", - "results-history-name", "num-flaky-test-attempts") - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class GcloudYml( - val gcloud: GcloudYmlParams = GcloudYmlParams() -) { - companion object : IYmlMap { - override val map = mapOf("gcloud" to GcloudYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt index 0f0145f796..8ab4e83276 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/IYmlKeys.kt @@ -1,5 +1,17 @@ package ftl.args.yml interface IYmlKeys { + val group: String val keys: List + + object Group { + const val FLANK = "flank" + const val GCLOUD = "gcloud" + } } + +fun mergeYmlKeys( + vararg keys: IYmlKeys +): Map> = keys + .groupBy(IYmlKeys::group, IYmlKeys::keys) + .mapValues { (_, keys) -> keys.flatten() } diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IYmlMap.kt b/test_runner/src/main/kotlin/ftl/args/yml/IYmlMap.kt deleted file mode 100644 index 59b08918b9..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/IYmlMap.kt +++ /dev/null @@ -1,5 +0,0 @@ -package ftl.args.yml - -interface IYmlMap { - val map: Map> -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IosFlankYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/IosFlankYml.kt deleted file mode 100644 index f02560b3c0..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/IosFlankYml.kt +++ /dev/null @@ -1,28 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty - -/** Flank specific parameters for both iOS and Android */ -@JsonIgnoreProperties(ignoreUnknown = true) -class IosFlankYmlParams( - @field:JsonProperty("test-targets") - val testTargets: List = emptyList() -) { - companion object : IYmlKeys { - override val keys = listOf("test-targets") - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class IosFlankYml( - // when an empty 'flank:' is present in a yaml then parsedFlank will be parsed as null. - @field:JsonProperty("flank") - private val parsedFlank: IosFlankYmlParams? = IosFlankYmlParams() -) { - val flank = parsedFlank ?: IosFlankYmlParams() - - companion object : IYmlMap { - override val map = mapOf("flank" to IosFlankYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/IosGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/IosGcloudYml.kt deleted file mode 100644 index 323c34c05a..0000000000 --- a/test_runner/src/main/kotlin/ftl/args/yml/IosGcloudYml.kt +++ /dev/null @@ -1,39 +0,0 @@ -package ftl.args.yml - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.annotation.JsonProperty -import ftl.config.Device -import ftl.config.FtlConstants.defaultIosModel -import ftl.config.FtlConstants.defaultIosVersion - -/** - * iOS specific gcloud parameters - * - * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run - * https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run - */ -@JsonIgnoreProperties(ignoreUnknown = true) -class IosGcloudYmlParams( - val test: String? = null, - - @field:JsonProperty("xctestrun-file") - val xctestrunFile: String? = null, - - @field:JsonProperty("xcode-version") - val xcodeVersion: String? = null, - - val device: List = listOf(Device(defaultIosModel, defaultIosVersion)) -) { - companion object : IYmlKeys { - override val keys = listOf("test", "xctestrun-file", "xcode-version", "device") - } -} - -@JsonIgnoreProperties(ignoreUnknown = true) -class IosGcloudYml( - val gcloud: IosGcloudYmlParams = IosGcloudYmlParams() -) { - companion object : IYmlMap { - override val map = mapOf("gcloud" to IosGcloudYmlParams.keys) - } -} diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt index 22e1db7e6a..a9937b2239 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt @@ -1,15 +1,14 @@ package ftl.cli.firebase.test +import ftl.config.Config +import ftl.config.android.AndroidGcloudConfig +import ftl.config.asDevice +import ftl.config.common.addDevice import picocli.CommandLine abstract class CommonRunCommand { - // Flank debug - - @CommandLine.Option(names = ["--dry"], description = ["Dry run on mock server"]) - var dryRun: Boolean = false - - // Flank specific + abstract val config: Config.Platform<*, *> @CommandLine.Option( names = ["-h", "--help"], @@ -18,191 +17,32 @@ abstract class CommonRunCommand { ) var usageHelpRequested: Boolean = false + // Gcloud @CommandLine.Option( - names = ["--run-timeout"], - description = ["The max time this test run can execute before it is cancelled (default: unlimited)."] - ) - var runTimeout: String? = null - - // GcloudYml.kt - - @CommandLine.Option( - names = ["--results-bucket"], - description = ["The name of a Google Cloud Storage bucket where raw test " + - "results will be stored (default: \"test-lab-\"). Note that the bucket must be owned by a " + - "billing-enabled project, and that using a non-default bucket will result in billing charges for the " + - "storage used."] - ) - var resultsBucket: String? = null - - @CommandLine.Option( - names = ["--results-dir"], - description = ["The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + - "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + - "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + - "intermingled."] - ) - var resultsDir: String? = null - - @CommandLine.Option( - names = ["--record-video"], - description = ["Enable video recording during the test. " + - "Disabled by default."] - ) - var recordVideo: Boolean? = null - - @CommandLine.Option( - names = ["--no-record-video"], - description = ["Disable video recording during the test (default behavior). Use --record-video to enable."] - ) - var noRecordVideo: Boolean? = null - - @CommandLine.Option( - names = ["--timeout"], - description = ["The max time this test execution can run before it is cancelled " + - "(default: 15m). It does not include any time necessary to prepare and clean up the target device. The maximum " + - "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " + - "or s. If no unit is given, seconds are assumed. "] - ) - var timeout: String? = null - - @CommandLine.Option( - names = ["--async"], - description = ["Invoke a test asynchronously without waiting for test results."] - ) - var async: Boolean? = null - - @CommandLine.Option( - names = ["--client-details"], + names = ["--device"], split = ",", - description = ["Comma-separated, KEY=VALUE map of additional details to attach to the test matrix." + - "Arbitrary KEY=VALUE pairs may be attached to a test matrix to provide additional context about the tests being run." + - "When consuming the test results, such as in Cloud Functions or a CI system," + - "these details can add additional context such as a link to the corresponding pull request."] - ) - var clientDetails: Map? = null - - @CommandLine.Option( - names = ["--network-profile"], - description = ["The name of the network traffic profile, for example --network-profile=LTE, " + - "which consists of a set of parameters to emulate network conditions when running the test " + - "(default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). " + - "This feature only works on physical devices. "] - ) - var networkProfile: String? = null - - @CommandLine.Option( - names = ["--results-history-name"], - description = ["The history name for your test results " + - "(an arbitrary string label; default: the application's label from the APK manifest). All tests which use the " + - "same history name will have their results grouped together in the Firebase console in a time-ordered test " + - "history list."] - ) - var resultsHistoryName: String? = null - - @CommandLine.Option( - names = ["--num-flaky-test-attempts"], - description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + - "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] - ) - var flakyTestAttempts: Int? = null - - // FlankYml.kt - - @CommandLine.Option( - names = ["--max-test-shards"], - description = ["The amount of matrices to split the tests across."] - ) - var maxTestShards: Int? = null + description = ["A list of DIMENSION=VALUE pairs which specify a target " + + "device to test against. This flag may be repeated to specify multiple devices. The four device dimensions are: " + + "model, version, locale, and orientation. If any dimensions are omitted, they will use a default value. Omitting " + + "all of the preceding dimension-related flags will run tests against a single device using defaults for all four " + + "device dimensions."] + ) + fun device(map: Map?) { + config.common.gcloud.addDevice( + device = map?.asDevice( + android = config.platform.gcloud is AndroidGcloudConfig + ) + ) + } - @CommandLine.Option( - names = ["--shard-time"], - description = ["The max amount of seconds each shard should run."] - ) - var shardTime: Int? = null - - @CommandLine.Option( - names = ["--num-test-runs"], - description = ["The amount of times to run the test executions."] - ) - var repeatTests: Int? = null - - @CommandLine.Option( - names = ["--smart-flank-gcs-path"], - description = ["Google cloud storage path to save test timing data used by smart flank."] - ) - var smartFlankGcsPath: String? = null - - @CommandLine.Option( - names = ["--smart-flank-disable-upload"], - description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] - ) - var smartFlankDisableUpload: Boolean? = null - - @CommandLine.Option( - names = ["--disable-sharding"], - description = ["Disable sharding."] - ) - var disableSharding: Boolean? = null - - @CommandLine.Option( - names = ["--test-targets-always-run"], - split = ",", - description = [ - "A list of one or more test methods to always run first in every shard."] - ) - var testTargetsAlwaysRun: List? = null - - @CommandLine.Option( - names = ["--files-to-download"], - split = ",", - description = ["A list of paths that will be downloaded from the resulting bucket " + - "to the local results folder after the test is complete. These must be absolute paths " + - "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " + - "Path names are restricted to the characters a-zA-Z0-9_-./+."] - ) - var filesToDownload: List? = null - - @CommandLine.Option( - names = ["--project"], - description = ["The Google Cloud Platform project name to use for this invocation. " + - "If omitted, then the project from the service account credential is used"] - ) - var project: String? = null - - @CommandLine.Option( - names = ["--local-result-dir"], - description = ["Saves test result to this local folder. Deleted before each run."] - ) - var localResultsDir: String? = null - - @CommandLine.Option( - names = ["--ignore-failed-tests"], - description = ["Terminate with exit code 0 when there are failed tests. " + - "Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. " + - "The JUnit XML is used to determine failure. (default: false)"] - ) - var ignoreFailedTests: Boolean? = null - - @CommandLine.Option( - names = ["--keep-file-path"], - description = ["Keeps the full path of downloaded files. " + - "Required when file names are not unique."] - ) - var keepFilePath: Boolean? = null - - @CommandLine.Option( - names = ["--output-style"], - description = ["Output style of execution status. May be one of [verbose, multi, single]. " + - "For runs with only one test execution the default value is 'verbose', in other cases " + - "'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles " + - "which don't support ansi codes, to avoid corrupted output use `single` or `verbose`."] - ) - var outputStyle: String? = null + // Flank debug + @CommandLine.Option(names = ["--dry"], description = ["Dry run on mock server"]) + var dryRun: Boolean = false + // Flank specific @CommandLine.Option( - names = ["--full-junit-result"], - description = ["Enable create additional local junit result on local storage with failure nodes on passed flaky tests."] + names = ["-c", "--config"], + description = ["YAML config file path"] ) - var fullJUnitResult: Boolean? = null + var configPath: String = "" } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 2049ac3aed..261a950ebd 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -1,19 +1,15 @@ package ftl.cli.firebase.test.android import ftl.args.AndroidArgs -import ftl.args.yml.AppTestPair import ftl.cli.firebase.test.CommonRunCommand -import ftl.config.Device import ftl.config.FtlConstants -import ftl.config.FtlConstants.defaultAndroidModel -import ftl.config.FtlConstants.defaultAndroidVersion -import ftl.config.FtlConstants.defaultLocale -import ftl.config.FtlConstants.defaultOrientation +import ftl.config.emptyAndroidConfig import ftl.mock.MockServer import ftl.run.ANDROID_SHARD_FILE import ftl.run.dumpShards import ftl.run.newTestRun import kotlinx.coroutines.runBlocking +import picocli.CommandLine import picocli.CommandLine.Command import picocli.CommandLine.Option import java.nio.file.Paths @@ -35,6 +31,13 @@ Configuration is read from flank.yml ) class AndroidRunCommand : CommonRunCommand(), Runnable { + @CommandLine.Mixin + override val config = emptyAndroidConfig() + + init { + configPath = FtlConstants.defaultAndroidConfig + } + override fun run() { if (dryRun) { MockServer.start() @@ -49,227 +52,9 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { } } - // Flank debug - @Option( names = ["--dump-shards"], description = ["Measures test shards from given test apks and writes them into $ANDROID_SHARD_FILE file instead of executing."] ) var dumpShards: Boolean = false - - // Flank specific - - @Option( - names = ["-c", "--config"], - description = ["YAML config file path"] - ) - var configPath: String = FtlConstants.defaultAndroidConfig - - // AndroidGcloudYml.kt - - @Option( - names = ["--app"], - description = ["The path to the application binary file. " + - "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] - ) - var app: String? = null - - @Option( - names = ["--test"], - description = ["The path to the binary file containing instrumentation tests. " + - "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] - ) - var test: String? = null - - @Option( - names = ["--additional-apks"], - split = ",", - description = ["A list of up to 100 additional APKs to install, in addition to those being directly tested." + - "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "] - ) - var additionalApks: List? = null - - @Option( - names = ["--auto-google-login"], - description = ["Automatically log into the test device using a preconfigured " + - "Google account before beginning the test. Disabled by default."] - ) - var autoGoogleLogin: Boolean? = null - - @Option( - names = ["--no-auto-google-login"], - description = ["Google account not logged in (default behavior). Use --auto-google-login to enable"] - ) - var noAutoGoogleLogin: Boolean? = null - - @Option( - names = ["--use-orchestrator"], - description = ["Whether each test runs in its own Instrumentation instance " + - "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + - "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + - "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + - "information about Android Test Orchestrator."] - ) - var useOrchestrator: Boolean? = null - - @Option( - names = ["--no-use-orchestrator"], - description = ["Orchestrator is not used. See --use-orchestrator."] - ) - var noUseOrchestrator: Boolean? = null - - @Option( - names = ["--robo-directives"], - split = ",", - description = [ - "A comma-separated (:=) map of robo_directives that you can use to customize the behavior of Robo test.", - "The type specifies the action type of the directive, which may take on values click, text or ignore.", - "If no type is provided, text will be used by default.", - "Each key should be the Android resource name of a target UI element and each value should be the text input for that element.", - "Values are only permitted for text type elements, so no value should be specified for click and ignore type elements." - ] - ) - var roboDirectives: List? = null - - @Option( - names = ["--robo-script"], - description = [ - "The path to a Robo Script JSON file.", - "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.", - "You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.", - "Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. " - ] - ) - var roboScript: String? = null - - @Option( - names = ["--environment-variables"], - split = ",", - description = ["A comma-separated, key=value map of environment variables " + - "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + - "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + - "passed to your test runner (typically AndroidJUnitRunner)"] - ) - var environmentVariables: Map? = null - - @Option( - names = ["--directories-to-pull"], - split = ",", - description = ["A list of paths that will be copied from the device's " + - "storage to the designated results bucket after the test is complete. These must be absolute paths under " + - "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + - "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + - "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + - "storage, the system will replace it with the external storage path prefix for that device."] - ) - var directoriesToPull: List? = null - - @Option( - names = ["--other-files"], - split = ",", - description = ["A list of device-path=file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push." + - "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." + - "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "] - ) - var otherFiles: Map? = null - - @Option( - names = ["--performance-metrics"], - description = ["Monitor and record performance metrics: CPU, memory, " + - "network usage, and FPS (game-loop only). Disabled by default."] - ) - var performanceMetrics: Boolean? = null - - @Option( - names = ["--no-performance-metrics"], - description = ["Disables performance metrics (default behavior). Use --performance-metrics to enable."] - ) - var noPerformanceMetrics: Boolean? = null - - @Option( - names = ["--num-uniform-shards"], - description = ["Specifies the number of shards into which you want to evenly distribute test cases." + - "The shards are run in parallel on separate devices. For example," + - "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." + - "The number of shards should be less than the total number of test cases." + - "The number of shards specified must be >= 1 and <= 50." + - "This option cannot be used along max-test-shards and is not compatible with smart sharding." + - "If you want to take benefits of smart sharding use max-test-shards."] - ) - var numUniformShards: Int? = null - - @Option( - names = ["--test-runner-class"], - description = ["The fully-qualified Java class name of the instrumentation test runner (default: the last name extracted " + - "from the APK manifest)."] - ) - var testRunnerClass: String? = null - - @Option( - names = ["--test-targets"], - split = ",", - description = ["A list of one or more test target filters to apply " + - "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + - "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + - "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + - "information."] - ) - var testTargets: List? = null - - @Option( - names = ["--device"], - split = ",", - description = ["A list of DIMENSION=VALUE pairs which specify a target " + - "device to test against. This flag may be repeated to specify multiple devices. The four device dimensions are: " + - "model, version, locale, and orientation. If any dimensions are omitted, they will use a default value. Omitting " + - "all of the preceding dimension-related flags will run tests against a single device using defaults for all four " + - "device dimensions."] - ) - fun deviceMap(map: Map?) { - if (map.isNullOrEmpty()) return - val androidDevice = Device( - model = map.getOrDefault("model", defaultAndroidModel), - version = map.getOrDefault("version", defaultAndroidVersion), - locale = map.getOrDefault("locale", defaultLocale), - orientation = map.getOrDefault("orientation", defaultOrientation) - ) - - if (device == null) device = mutableListOf() - device?.add(androidDevice) - } - - var device: MutableList? = null - - // AndroidFlankYml - - @Option( - names = ["--additional-app-test-apks"], - split = ",", - description = ["A list of app & test apks to include in the run. " + - "Useful for running multiple module tests within a single Flank run."] - ) - fun apkMap(map: Map?) { - if (map.isNullOrEmpty()) return - if (additionalAppTestApks == null) additionalAppTestApks = mutableListOf() - - val appApk = map["app"] - val testApk = map["test"] - - if (appApk != null && testApk != null) { - additionalAppTestApks?.add( - AppTestPair( - app = appApk, - test = testApk - ) - ) - } - } - - var additionalAppTestApks: MutableList? = null - - @Option( - names = ["--legacy-junit-result"], - description = ["Fallback for legacy xml junit results parsing."] - ) - var useLegacyJUnitResult: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 52e362ca29..4dad71be83 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -2,15 +2,14 @@ package ftl.cli.firebase.test.ios import ftl.args.IosArgs import ftl.cli.firebase.test.CommonRunCommand -import ftl.config.Device import ftl.config.FtlConstants -import ftl.config.FtlConstants.defaultIosModel -import ftl.config.FtlConstants.defaultIosVersion +import ftl.config.emptyIosConfig import ftl.mock.MockServer import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.newTestRun import kotlinx.coroutines.runBlocking +import picocli.CommandLine import picocli.CommandLine.Command import picocli.CommandLine.Option import java.nio.file.Paths @@ -31,6 +30,14 @@ Configuration is read from flank.yml usageHelpAutoWidth = true ) class IosRunCommand : CommonRunCommand(), Runnable { + + @CommandLine.Mixin + override val config = emptyIosConfig() + + init { + configPath = FtlConstants.defaultIosConfig + } + override fun run() { if (dryRun) { MockServer.start() @@ -45,81 +52,9 @@ class IosRunCommand : CommonRunCommand(), Runnable { } } - // Flank debug - @Option( names = ["--dump-shards"], description = ["Measures test shards from given test apks and writes them into $IOS_SHARD_FILE file instead of executing."] ) var dumpShards: Boolean = false - - // Flank specific - - @Option( - names = ["-c", "--config"], - description = ["YAML config file path"] - ) - var configPath: String = FtlConstants.defaultIosConfig - - // IosGcloudYml.kt - - @Option( - names = ["--test"], - description = ["The path to the test package (a zip file containing the iOS app " + - "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + - "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + - "is specified."] - ) - var test: String? = null - - @Option( - names = ["--xctestrun-file"], - description = ["The path to an .xctestrun file that will override any " + - ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + - "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + - "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] - ) - var xctestrunFile: String? = null - - @Option( - names = ["--xcode-version"], - description = ["The version of Xcode that should be used to run an XCTest. " + - "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + - "all iOS versions selected in the test matrix."] - ) - var xcodeVersion: String? = null - - @Option( - names = ["--device"], - split = ",", - description = ["A list of DIMENSION=VALUE pairs which specify a target " + - "device to test against. This flag may be repeated to specify multiple devices. The four device dimensions are: " + - "model, version, locale, and orientation. If any dimensions are omitted, they will use a default value. Omitting " + - "all of the preceding dimension-related flags will run tests against a single device using defaults for all four " + - "device dimensions."] - ) - fun deviceMap(map: Map?) { - if (map.isNullOrEmpty()) return - val androidDevice = Device( - model = map.getOrDefault("model", defaultIosModel), - version = map.getOrDefault("version", defaultIosVersion), - locale = map.getOrDefault("locale", FtlConstants.defaultLocale), - orientation = map.getOrDefault("orientation", FtlConstants.defaultOrientation) - ) - - if (device == null) device = mutableListOf() - device?.add(androidDevice) - } - - var device: MutableList? = null - - // IosFlankYml.kt - - @Option( - names = ["--test-targets"], - split = ",", - description = ["A list of one or more test method " + - "names to run (default: run all test targets)."] - ) - var testTargets: List? = null } diff --git a/test_runner/src/main/kotlin/ftl/config/Config.kt b/test_runner/src/main/kotlin/ftl/config/Config.kt new file mode 100644 index 0000000000..88c150f92a --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/Config.kt @@ -0,0 +1,34 @@ +package ftl.config + +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig +import picocli.CommandLine + +interface Config { + val data: MutableMap + + @CommandLine.Command + data class Partial( + @field:CommandLine.Mixin + val gcloud: Gcloud, + @field:CommandLine.Mixin + val flank: Flank + ) + + @CommandLine.Command + data class Platform( + val data: String = "", + @field:CommandLine.Mixin + val common: Partial, + @field:CommandLine.Mixin + val platform: Partial + ) +} + +typealias CommonConfig = Config.Partial +typealias AndroidConfig = Config.Platform +typealias IosConfig = Config.Platform diff --git a/test_runner/src/main/kotlin/ftl/config/Create.kt b/test_runner/src/main/kotlin/ftl/config/Create.kt new file mode 100644 index 0000000000..ffb39052a3 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/Create.kt @@ -0,0 +1,52 @@ +package ftl.config + +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig + +fun defaultAndroidConfig() = AndroidConfig( + common = Config.Partial( + gcloud = CommonGcloudConfig.default(android = true), + flank = CommonFlankConfig.default() + ), + platform = Config.Partial( + gcloud = AndroidGcloudConfig.default(), + flank = AndroidFlankConfig.default() + ) +) + +fun defaultIosConfig() = IosConfig( + common = Config.Partial( + gcloud = CommonGcloudConfig.default(android = false), + flank = CommonFlankConfig.default() + ), + platform = Config.Partial( + gcloud = IosGcloudConfig.default(), + flank = IosFlankConfig.default() + ) +) + +fun emptyAndroidConfig() = AndroidConfig( + common = Config.Partial( + gcloud = CommonGcloudConfig(), + flank = CommonFlankConfig() + ), + platform = Config.Partial( + gcloud = AndroidGcloudConfig(), + flank = AndroidFlankConfig() + ) +) + +fun emptyIosConfig() = IosConfig( + common = Config.Partial( + gcloud = CommonGcloudConfig(), + flank = CommonFlankConfig() + ), + platform = Config.Partial( + gcloud = IosGcloudConfig(), + flank = IosFlankConfig() + ) +) diff --git a/test_runner/src/main/kotlin/ftl/config/Device.kt b/test_runner/src/main/kotlin/ftl/config/Device.kt index d7b7771348..122380676c 100644 --- a/test_runner/src/main/kotlin/ftl/config/Device.kt +++ b/test_runner/src/main/kotlin/ftl/config/Device.kt @@ -1,5 +1,9 @@ package ftl.config +import ftl.config.FtlConstants.defaultAndroidModel +import ftl.config.FtlConstants.defaultAndroidVersion +import ftl.config.FtlConstants.defaultIosModel +import ftl.config.FtlConstants.defaultIosVersion import ftl.config.FtlConstants.defaultLocale import ftl.config.FtlConstants.defaultOrientation import ftl.util.trimStartLine @@ -19,3 +23,19 @@ data class Device( orientation: $orientation""".trimStartLine() } } + +fun defaultDevice(android: Boolean) = Device( + model = if (android) defaultAndroidModel else defaultIosModel, + version = if (android) defaultAndroidVersion else defaultIosVersion +) + +fun Map.asDevice(android: Boolean) = + if (isEmpty()) null + else defaultDevice(android).run { + copy( + model = getOrDefault("model", model), + version = getOrDefault("version", version), + locale = getOrDefault("locale", version), + orientation = getOrDefault("orientation", version) + ) + } diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index c28cda6d5c..2277afb2fa 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -13,7 +13,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.config.BugsnagInitHelper.initBugsnag +import ftl.util.BugsnagInitHelper.initBugsnag import ftl.gc.UserAuth import ftl.http.HttpTimeoutIncrease import ftl.util.FlankFatalError diff --git a/test_runner/src/main/kotlin/ftl/config/Load.kt b/test_runner/src/main/kotlin/ftl/config/Load.kt new file mode 100644 index 0000000000..0120f0dcdd --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/Load.kt @@ -0,0 +1,58 @@ +@file:Suppress("UnusedPrivateClass") +package ftl.config + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import ftl.args.ArgsHelper +import ftl.args.yml.YamlDeprecated.modifyAndThrow +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig +import ftl.util.loadFile +import java.io.Reader +import java.nio.file.Path + +fun loadAndroidConfig( + path: Path? = null, + reader: Reader = loadFile(path!!) +): AndroidConfig = modifyAndThrow(reader, android = true).run { + AndroidConfig( + data = this, + common = parseYaml().run { Config.Partial(gcloud, flank) }, + platform = parseYaml().run { Config.Partial(gcloud, flank) } + ) +} + +fun loadIosConfig( + path: Path? = null, + reader: Reader = loadFile(path!!) +): IosConfig = modifyAndThrow(reader, android = false).run { + IosConfig( + data = this, + common = parseYaml().run { Config.Partial(gcloud, flank) }, + platform = parseYaml().run { Config.Partial(gcloud, flank) } + ) +} + +@JsonIgnoreProperties(ignoreUnknown = true) +private class CommonWrapper( + val flank: CommonFlankConfig = CommonFlankConfig(), + val gcloud: CommonGcloudConfig = CommonGcloudConfig() +) + +@JsonIgnoreProperties(ignoreUnknown = true) +private class AndroidWrapper( + val flank: AndroidFlankConfig = AndroidFlankConfig(), + val gcloud: AndroidGcloudConfig = AndroidGcloudConfig() +) + +@JsonIgnoreProperties(ignoreUnknown = true) +private class IosWrapper( + val flank: IosFlankConfig = IosFlankConfig(), + val gcloud: IosGcloudConfig = IosGcloudConfig() +) + +private inline fun String.parseYaml() = + ArgsHelper.yamlMapper.readValue(this, T::class.java) diff --git a/test_runner/src/main/kotlin/ftl/config/Merge.kt b/test_runner/src/main/kotlin/ftl/config/Merge.kt new file mode 100644 index 0000000000..e7e59d9733 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/Merge.kt @@ -0,0 +1,19 @@ +package ftl.config + +operator fun Config.Platform.plus(other: Config.Platform?) = copy( + data = if (other?.data?.isNotBlank() == true) other.data else data +).apply { + other?.let { + common + other.common + platform + other.platform + } +} + +private operator fun Config.Partial.plus(other: Config.Partial) = apply { + gcloud + other.gcloud + flank + other.flank +} + +private operator fun C.plus(other: Config) = apply { + data += other.data +} diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt new file mode 100644 index 0000000000..3fe2aafcb1 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidFlankConfig.kt @@ -0,0 +1,67 @@ +package ftl.config.android + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.config.Config +import ftl.args.yml.AppTestPair +import ftl.args.yml.IYmlKeys +import picocli.CommandLine + +/** Flank specific parameters for Android */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class AndroidFlankConfig @JsonIgnore constructor( + @JsonIgnore + override val data: MutableMap +) : Config { + + @CommandLine.Option( + names = ["--additional-app-test-apks"], + split = ",", + description = ["A list of app & test apks to include in the run. " + + "Useful for running multiple module tests within a single Flank run."] + ) + fun additionalAppTestApks(map: Map?) { + if (map.isNullOrEmpty()) return + if (additionalAppTestApks == null) additionalAppTestApks = mutableListOf() + + val appApk = map["app"] + val testApk = map["test"] + + if (testApk != null) { + additionalAppTestApks?.add( + AppTestPair( + app = appApk, + test = testApk + ) + ) + } + } + + @set:JsonProperty("additional-app-test-apks") + var additionalAppTestApks: MutableList? by data + + @set:CommandLine.Option( + names = ["--legacy-junit-result"], + description = ["Fallback for legacy xml junit results parsing."] + ) + @set:JsonProperty("legacy-junit-result") + var useLegacyJUnitResult: Boolean? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.FLANK + + override val keys = listOf( + "additional-app-test-apks", + "legacy-junit-result" + ) + + fun default() = AndroidFlankConfig().apply { + additionalAppTestApks = null + useLegacyJUnitResult = false + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt new file mode 100644 index 0000000000..71412c1d0c --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/android/AndroidGcloudConfig.kt @@ -0,0 +1,233 @@ +package ftl.config.android + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.args.yml.IYmlKeys +import ftl.config.Config +import ftl.config.FlankDefaults +import picocli.CommandLine + +/** + * Android specific gcloud parameters + * + * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run + */ +@CommandLine.Command +@JsonIgnoreProperties(ignoreUnknown = true) +data class AndroidGcloudConfig @JsonIgnore constructor( + @JsonIgnore + override val data: MutableMap +) : Config { + + @set:CommandLine.Option( + names = ["--app"], + description = ["The path to the application binary file. " + + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation."] + ) + var app: String? by data + + @set:CommandLine.Option( + names = ["--test"], + description = ["The path to the binary file containing instrumentation tests. " + + "The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] + ) + var test: String? by data + + @set:CommandLine.Option( + names = ["--additional-apks"], + split = ",", + description = ["A list of up to 100 additional APKs to install, in addition to those being directly tested." + + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. "] + ) + @set:JsonProperty("additional-apks") + var additionalApks: List? by data + + @set:CommandLine.Option( + names = ["--auto-google-login"], + description = ["Automatically log into the test device using a preconfigured " + + "Google account before beginning the test. Disabled by default."] + ) + @set:JsonProperty("auto-google-login") + var autoGoogleLogin: Boolean? by data + + @CommandLine.Option( + names = ["--no-auto-google-login"], + description = ["Google account not logged in (default behavior). Use --auto-google-login to enable"] + ) + @JsonIgnore + fun noAutoGoogleLogin(value: Boolean?) { + autoGoogleLogin = value?.not() ?: false + } + + @set:CommandLine.Option( + names = ["--use-orchestrator"], + description = ["Whether each test runs in its own Instrumentation instance " + + "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + + "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + + "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + + "information about Android Test Orchestrator."] + ) + @set:JsonProperty("use-orchestrator") + var useOrchestrator: Boolean? by data + + @CommandLine.Option( + names = ["--no-use-orchestrator"], + description = ["Orchestrator is not used. See --use-orchestrator."] + ) + fun noUseOrchestrator(value: Boolean?) { + useOrchestrator = value?.not() ?: false + } + + @set:CommandLine.Option( + names = ["--environment-variables"], + split = ",", + description = ["A comma-separated, key=value map of environment variables " + + "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + + "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + + "passed to your test runner (typically AndroidJUnitRunner)"] + ) + @set:JsonProperty("environment-variables") + var environmentVariables: Map? by data + + @set:CommandLine.Option( + names = ["--directories-to-pull"], + split = ",", + description = ["A list of paths that will be copied from the device's " + + "storage to the designated results bucket after the test is complete. These must be absolute paths under " + + "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + + "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + + "storage, the system will replace it with the external storage path prefix for that device."] + ) + @set:JsonProperty("directories-to-pull") + var directoriesToPull: List? by data + + @set:CommandLine.Option( + names = ["--other-files"], + split = ",", + description = ["A list of device-path=file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push." + + "Device paths must be under absolute, whitelisted paths (\${EXTERNAL_STORAGE}, or \${ANDROID_DATA}/local/tmp)." + + "Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). "] + ) + @set:JsonProperty("other-files") + var otherFiles: Map? by data + + @set:CommandLine.Option( + names = ["--performance-metrics"], + description = ["Monitor and record performance metrics: CPU, memory, " + + "network usage, and FPS (game-loop only). Disabled by default."] + ) + @set:JsonProperty("performance-metrics") + var performanceMetrics: Boolean? by data + + @CommandLine.Option( + names = ["--no-performance-metrics"], + description = ["Disables performance metrics (default behavior). Use --performance-metrics to enable."] + ) + @JsonIgnore + fun noPerformanceMetrics(value: Boolean?) { + performanceMetrics = value?.not() ?: false + } + + @set:CommandLine.Option( + names = ["--num-uniform-shards"], + description = ["Specifies the number of shards into which you want to evenly distribute test cases." + + "The shards are run in parallel on separate devices. For example," + + "if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases." + + "The number of shards should be less than the total number of test cases." + + "The number of shards specified must be >= 1 and <= 50." + + "This option cannot be used along max-test-shards and is not compatible with smart sharding." + + "If you want to take benefits of smart sharding use max-test-shards."] + ) + @set:JsonProperty("num-uniform-shards") + var numUniformShards: Int? by data + + @set:CommandLine.Option( + names = ["--test-runner-class"], + description = ["The fully-qualified Java class name of the instrumentation test runner (default: the last name extracted " + + "from the APK manifest)."] + ) + @set:JsonProperty("test-runner-class") + var testRunnerClass: String? by data + + @set:CommandLine.Option( + names = ["--test-targets"], + split = ",", + description = ["A list of one or more test target filters to apply " + + "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + + "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + + "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + + "information."] + ) + @set:JsonProperty("test-targets") + var testTargets: List? by data + + @set:CommandLine.Option( + names = ["--robo-directives"], + split = ",", + description = [ + "A comma-separated (:=) map of robo_directives that you can use to customize the behavior of Robo test.", + "The type specifies the action type of the directive, which may take on values click, text or ignore.", + "If no type is provided, text will be used by default.", + "Each key should be the Android resource name of a target UI element and each value should be the text input for that element.", + "Values are only permitted for text type elements, so no value should be specified for click and ignore type elements." + ] + ) + @set:JsonProperty("robo-directives") + var roboDirectives: Map? by data + + @set:CommandLine.Option( + names = ["--robo-script"], + description = [ + "The path to a Robo Script JSON file.", + "The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.", + "You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument.", + "Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. " + ] + ) + @set:JsonProperty("robo-script") + var roboScript: String? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.GCLOUD + + override val keys = listOf( + "app", + "test", + "additional-apks", + "auto-google-login", + "use-orchestrator", + "environment-variables", + "directories-to-pull", + "other-files", + "performance-metrics", + "num-uniform-shards", + "test-runner-class", + "test-targets", + "robo-directives", + "robo-script", + "device" + ) + + fun default() = AndroidGcloudConfig().apply { + app = null + test = null + additionalApks = emptyList() + autoGoogleLogin = FlankDefaults.DISABLE_AUTO_LOGIN + useOrchestrator = true + environmentVariables = emptyMap() + directoriesToPull = emptyList() + otherFiles = emptyMap() + performanceMetrics = FlankDefaults.DISABLE_PERFORMANCE_METRICS + numUniformShards = null + testRunnerClass = null + testTargets = emptyList() + roboDirectives = emptyMap() + roboScript = null + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt new file mode 100644 index 0000000000..153fac54fc --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt @@ -0,0 +1,179 @@ +package ftl.config.common + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.args.ArgsHelper +import ftl.config.Config +import ftl.args.yml.IYmlKeys +import ftl.config.FtlConstants +import picocli.CommandLine + +/** Flank specific parameters for both iOS and Android */ +@CommandLine.Command +@JsonIgnoreProperties(ignoreUnknown = true) +data class CommonFlankConfig @JsonIgnore constructor( + @field:JsonIgnore + override val data: MutableMap +) : Config { + @set:CommandLine.Option( + names = ["--max-test-shards"], + description = ["The amount of matrices to split the tests across."] + ) + @set:JsonProperty("max-test-shards") + var maxTestShards: Int? by data + + @set:CommandLine.Option( + names = ["--shard-time"], + description = ["The max amount of seconds each shard should run."] + ) + @set:JsonProperty("shard-time") + var shardTime: Int? by data + + @set:CommandLine.Option( + names = ["--num-test-runs"], + description = ["The amount of times to run the test executions."] + ) + @set:JsonProperty("num-test-runs") + var repeatTests: Int? by data + + @set:CommandLine.Option( + names = ["--smart-flank-gcs-path"], + description = ["Google cloud storage path to save test timing data used by smart flank."] + ) + @set:JsonProperty("smart-flank-gcs-path") + var smartFlankGcsPath: String? by data + + @set:CommandLine.Option( + names = ["--smart-flank-disable-upload"], + description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] + ) + @set:JsonProperty("smart-flank-disable-upload") + var smartFlankDisableUpload: Boolean? by data + + @set:CommandLine.Option( + names = ["--disable-sharding"], + description = ["Disable sharding."] + ) + @set:JsonProperty("disable-sharding") + var disableSharding: Boolean? by data + + @set:CommandLine.Option( + names = ["--test-targets-always-run"], + split = ",", + description = [ + "A list of one or more test methods to always run first in every shard."] + ) + @set:JsonProperty("test-targets-always-run") + var testTargetsAlwaysRun: List? by data + + @set:CommandLine.Option( + names = ["--files-to-download"], + split = ",", + description = ["A list of paths that will be downloaded from the resulting bucket " + + "to the local results folder after the test is complete. These must be absolute paths " + + "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+."] + ) + @set:JsonProperty("files-to-download") + var filesToDownload: List? by data + + @set:CommandLine.Option( + names = ["--project"], + description = ["The Google Cloud Platform project name to use for this invocation. " + + "If omitted, then the project from the service account credential is used"] + ) + var project: String? by data + + @set:CommandLine.Option( + names = ["--local-result-dir"], + description = ["Saves test result to this local folder. Deleted before each run."] + ) + @set:JsonProperty("local-result-dir") + var localResultsDir: String? by data + + @set:CommandLine.Option( + names = ["--run-timeout"], + description = ["The max time this test run can execute before it is cancelled (default: unlimited)."] + ) + @set:JsonProperty("run-timeout") + var runTimeout: String? by data + + @set:CommandLine.Option( + names = ["--full-junit-result"], + description = ["Enable create additional local junit result on local storage with failure nodes on passed flaky tests."] + ) + @set:JsonProperty("full-junit-result") + var fullJUnitResult: Boolean? by data + + @set:CommandLine.Option( + names = ["--ignore-failed-tests"], + description = ["Terminate with exit code 0 when there are failed tests. " + + "Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. " + + "The JUnit XML is used to determine failure. (default: false)"] + ) + @set:JsonProperty("ignore-failed-tests") + var ignoreFailedTests: Boolean? by data + + @set:CommandLine.Option( + names = ["--keep-file-path"], + description = ["Keeps the full path of downloaded files. " + + "Required when file names are not unique."] + ) + @set:JsonProperty("keep-file-path") + var keepFilePath: Boolean? by data + + @set:CommandLine.Option( + names = ["--output-style"], + description = ["Output style of execution status. May be one of [verbose, multi, single]. " + + "For runs with only one test execution the default value is 'verbose', in other cases " + + "'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles " + + "which don't support ansi codes, to avoid corrupted output use `single` or `verbose`."] + ) + @set:JsonProperty("output-style") + var outputStyle: String? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.FLANK + + override val keys = listOf( + "max-test-shards", + "shard-time", + "num-test-runs", + "smart-flank-gcs-path", + "smart-flank-disable-upload", + "disable-sharding", + "test-targets-always-run", + "files-to-download", + "project", + "run-timeout", + "legacy-junit-result", + "ignore-failed-tests", + "keep-file-path", + "output-style" + ) + + const val defaultLocalResultsDir = "results" + + fun default() = CommonFlankConfig().apply { + project = ArgsHelper.getDefaultProjectId() ?: "" + maxTestShards = 1 + shardTime = -1 + repeatTests = 1 + smartFlankGcsPath = "" + smartFlankDisableUpload = false + testTargetsAlwaysRun = emptyList() + filesToDownload = emptyList() + disableSharding = false + localResultsDir = defaultLocalResultsDir + runTimeout = FtlConstants.runTimeout + fullJUnitResult = false + ignoreFailedTests = false + keepFilePath = false + outputStyle = null + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt new file mode 100644 index 0000000000..9cfb573c51 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonGcloudConfig.kt @@ -0,0 +1,155 @@ +package ftl.config.common + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.args.ArgsHelper +import ftl.args.yml.IYmlKeys +import ftl.config.Config +import ftl.config.Device +import ftl.config.FlankDefaults +import ftl.config.defaultDevice +import picocli.CommandLine + +/** + * Common Gcloud parameters shared between iOS and Android + * + * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run + * https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run + */ +@CommandLine.Command +@JsonIgnoreProperties(ignoreUnknown = true) +data class CommonGcloudConfig @JsonIgnore constructor( + @JsonIgnore + override val data: MutableMap +) : Config { + + @set:JsonProperty("device") + var devices: List? by data + + @set:CommandLine.Option( + names = ["--results-bucket"], + description = ["The name of a Google Cloud Storage bucket where raw test " + + "results will be stored (default: \"test-lab-\"). Note that the bucket must be owned by a " + + "billing-enabled project, and that using a non-default bucket will result in billing charges for the " + + "storage used."] + ) + @set:JsonProperty("results-bucket") + var resultsBucket: String? by data + + @set:CommandLine.Option( + names = ["--results-dir"], + description = ["The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + + "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + + "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + + "intermingled."] + ) + @set:JsonProperty("results-dir") + var resultsDir: String? by data + + @set:CommandLine.Option( + names = ["--record-video"], + description = ["Enable video recording during the test. " + + "Disabled by default."] + ) + @set:JsonProperty("record-video") + var recordVideo: Boolean? by data + + @CommandLine.Option( + names = ["--no-record-video"], + description = ["Disable video recording during the test (default behavior). Use --record-video to enable."] + ) + fun noRecordVideo(value: Boolean) { + recordVideo = !value + } + + @set:CommandLine.Option( + names = ["--timeout"], + description = ["The max time this test execution can run before it is cancelled " + + "(default: 15m). It does not include any time necessary to prepare and clean up the target device. The maximum " + + "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " + + "or s. If no unit is given, seconds are assumed. "] + ) + var timeout: String? by data + + @set:CommandLine.Option( + names = ["--async"], + description = ["Invoke a test asynchronously without waiting for test results."] + ) + var async: Boolean? by data + + @set:CommandLine.Option( + names = ["--client-details"], + split = ",", + description = ["Comma-separated, KEY=VALUE map of additional details to attach to the test matrix." + + "Arbitrary KEY=VALUE pairs may be attached to a test matrix to provide additional context about the tests being run." + + "When consuming the test results, such as in Cloud Functions or a CI system," + + "these details can add additional context such as a link to the corresponding pull request."] + ) + @set:JsonProperty("client-details") + var clientDetails: Map? by data + + @set:CommandLine.Option( + names = ["--network-profile"], + description = ["The name of the network traffic profile, for example --network-profile=LTE, " + + "which consists of a set of parameters to emulate network conditions when running the test " + + "(default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). " + + "This feature only works on physical devices. "] + ) + @set:JsonProperty("network-profile") + var networkProfile: String? by data + + @set:CommandLine.Option( + names = ["--results-history-name"], + description = ["The history name for your test results " + + "(an arbitrary string label; default: the application's label from the APK manifest). All tests which use the " + + "same history name will have their results grouped together in the Firebase console in a time-ordered test " + + "history list."] + ) + @set:JsonProperty("results-history-name") + var resultsHistoryName: String? by data + + @set:CommandLine.Option( + names = ["--num-flaky-test-attempts"], + description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + + "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] + ) + @set:JsonProperty("num-flaky-test-attempts") + var flakyTestAttempts: Int? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.GCLOUD + + override val keys = listOf( + "results-bucket", + "results-dir", + "record-video", + "timeout", + "async", + "results-history-name", + "num-flaky-test-attempts" + ) + + fun default(android: Boolean) = CommonGcloudConfig().apply { + ArgsHelper.yamlMapper.readerFor(CommonGcloudConfig::class.java) + + resultsBucket = "" + resultsDir = null + recordVideo = FlankDefaults.DISABLE_VIDEO_RECORDING + timeout = "15m" + async = false + resultsHistoryName = null + flakyTestAttempts = 0 + clientDetails = null + networkProfile = null + devices = listOf(defaultDevice(android)) + } + } +} + +fun CommonGcloudConfig.addDevice(device: Device?) { + device?.let { devices = (devices ?: emptyList()) + device } +} diff --git a/test_runner/src/main/kotlin/ftl/config/ios/IosFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/ios/IosFlankConfig.kt new file mode 100644 index 0000000000..490c161f7b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/ios/IosFlankConfig.kt @@ -0,0 +1,38 @@ +package ftl.config.ios + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.args.yml.IYmlKeys +import ftl.config.Config +import picocli.CommandLine + +/** Flank specific parameters for iOS */ +@CommandLine.Command +@JsonIgnoreProperties(ignoreUnknown = true) +data class IosFlankConfig @JsonIgnore constructor( + @JsonIgnore + override val data: MutableMap +) : Config { + @set:CommandLine.Option( + names = ["--test-targets"], + split = ",", + description = ["A list of one or more test method " + + "names to run (default: run all test targets)."] + ) + @set:JsonProperty("test-targets") + var testTargets: List? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.FLANK + + override val keys = listOf("test-targets") + + fun default() = IosFlankConfig().apply { + testTargets = emptyList() + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt new file mode 100644 index 0000000000..cf4ce70438 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/config/ios/IosGcloudConfig.kt @@ -0,0 +1,69 @@ +package ftl.config.ios + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import ftl.config.Config +import ftl.args.yml.IYmlKeys +import picocli.CommandLine + +/** + * iOS specific gcloud parameters + * + * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run + * https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run + */ +@JsonIgnoreProperties(ignoreUnknown = true) +data class IosGcloudConfig @JsonIgnore constructor( + @JsonIgnore + override val data: MutableMap +) : Config { + + @set:CommandLine.Option( + names = ["--test"], + description = ["The path to the test package (a zip file containing the iOS app " + + "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + + "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + + "is specified."] + ) + var test: String? by data + + @set:CommandLine.Option( + names = ["--xctestrun-file"], + description = ["The path to an .xctestrun file that will override any " + + ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + + "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + + "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] + ) + @set:JsonProperty("xctestrun-file") + var xctestrunFile: String? by data + + @set:CommandLine.Option( + names = ["--xcode-version"], + description = ["The version of Xcode that should be used to run an XCTest. " + + "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + + "all iOS versions selected in the test matrix."] + ) + @set:JsonProperty("xcode-version") + var xcodeVersion: String? by data + + constructor() : this(mutableMapOf().withDefault { null }) + + companion object : IYmlKeys { + + override val group = IYmlKeys.Group.GCLOUD + + override val keys = listOf( + "test", + "xctestrun-file", + "xcode-version", + "device" + ) + + fun default() = IosGcloudConfig().apply { + test = null + xctestrunFile = null + xcodeVersion = null + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt b/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt index 9c79ad9af4..ad2689f80b 100644 --- a/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt +++ b/test_runner/src/main/kotlin/ftl/doctor/Doctor.kt @@ -2,19 +2,19 @@ package ftl.doctor import com.google.common.annotations.VisibleForTesting import ftl.args.ArgsHelper -import ftl.args.IArgsCompanion +import ftl.args.IArgs import ftl.util.loadFile import java.io.Reader import java.nio.file.Path object Doctor { - fun validateYaml(args: IArgsCompanion, data: Path): String { + fun validateYaml(args: IArgs.ICompanion, data: Path): String { if (!data.toFile().exists()) return "Skipping yaml validation. No file at path $data" return validateYaml(args, loadFile(data)) } @VisibleForTesting - internal fun validateYaml(args: IArgsCompanion, data: Reader): String { + internal fun validateYaml(args: IArgs.ICompanion, data: Reader): String { var result = "" val parsed = ArgsHelper.yamlMapper.readTree(data) diff --git a/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt index 9e5e684fae..840e597f6d 100644 --- a/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt +++ b/test_runner/src/main/kotlin/ftl/gc/android/CreateAndroidRobotTest.kt @@ -3,7 +3,7 @@ package ftl.gc.android import com.google.api.services.testing.model.AndroidRoboTest import com.google.api.services.testing.model.FileReference import com.google.api.services.testing.model.RoboDirective -import ftl.config.FlankRoboDirective +import ftl.args.FlankRoboDirective import ftl.run.platform.android.AndroidTestConfig internal fun createAndroidRoboTest( diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index 354e90b195..bcf7b70979 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -2,6 +2,7 @@ package ftl.run import ftl.args.AndroidArgs import ftl.args.IosArgs +import ftl.args.isInstrumentationTest import ftl.run.common.prettyPrint import ftl.run.model.AndroidMatrixTestShards import ftl.run.platform.android.getAndroidMatrixShards diff --git a/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt index 8e3a83b337..11d699b831 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/android/AndroidTestConfig.kt @@ -1,7 +1,7 @@ package ftl.run.platform.android import ftl.args.ShardChunks -import ftl.config.FlankRoboDirective +import ftl.args.FlankRoboDirective sealed class AndroidTestConfig { diff --git a/test_runner/src/main/kotlin/ftl/config/BugsnagInitHelper.kt b/test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt similarity index 97% rename from test_runner/src/main/kotlin/ftl/config/BugsnagInitHelper.kt rename to test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt index 599c919572..9869ed74da 100644 --- a/test_runner/src/main/kotlin/ftl/config/BugsnagInitHelper.kt +++ b/test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt @@ -1,4 +1,4 @@ -package ftl.config +package ftl.util import com.bugsnag.Bugsnag import java.nio.file.Paths diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index a32987d08f..c64a3ebd5a 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -20,8 +20,9 @@ fun main() { // "--debug", "firebase", "test", "android", "run", + "--results-dir=asd", // "--dry", -// "--dump-shards", + "--dump-shards", "--output-style=single", "--full-junit-result", "-c=src/test/kotlin/ftl/fixtures/test_app_cases/flank-$quantity-$type.yml", diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt index 695436a1d5..ff20cbc74b 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt @@ -1,16 +1,9 @@ package ftl.args import com.google.common.truth.Truth.assertThat -import ftl.args.yml.AndroidFlankYml -import ftl.args.yml.AndroidFlankYmlParams -import ftl.args.yml.AndroidGcloudYml -import ftl.args.yml.AndroidGcloudYmlParams import ftl.args.yml.AppTestPair -import ftl.args.yml.FlankYml -import ftl.args.yml.FlankYmlParams -import ftl.args.yml.GcloudYml -import ftl.args.yml.GcloudYmlParams import ftl.config.Device +import ftl.config.defaultAndroidConfig import ftl.run.platform.android.createAndroidTestContexts import ftl.run.platform.android.getAndroidMatrixShards import ftl.run.status.OutputStyle @@ -94,54 +87,43 @@ class AndroidArgsFileTest { } } - private fun configWithTestMethods(amount: Int, maxTestShards: Int = 1): AndroidArgs { - - return AndroidArgs( - GcloudYml(GcloudYmlParams()), - AndroidGcloudYml( - AndroidGcloudYmlParams( - app = appApkLocal, + private fun configWithTestMethods( + amount: Int, + maxTestShards: Int = 1 + ): AndroidArgs = createAndroidArgs( + defaultAndroidConfig().apply { + platform.apply { + gcloud.apply { + app = appApkLocal test = getString("src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_$amount.apk") - ) - ), - FlankYml( - FlankYmlParams( - maxTestShards = maxTestShards - ) - ), - AndroidFlankYml(), - "" - ) - } + } + } + common.flank.maxTestShards = maxTestShards + } + ) @Test fun `calculateShards additionalAppTestApks`() { val test1 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_1.apk" val test155 = "src/test/kotlin/ftl/fixtures/tmp/apk/app-debug-androidTest_155.apk" - val config = AndroidArgs( - GcloudYml(GcloudYmlParams()), - AndroidGcloudYml( - AndroidGcloudYmlParams( - app = appApkLocal, - test = getString(test1) - ) - ), - FlankYml( - FlankYmlParams( - maxTestShards = 3 - ) - ), - AndroidFlankYml( - AndroidFlankYmlParams( - additionalAppTestApks = listOf( - AppTestPair( - app = appApkLocal, - test = getString(test155) + val config = createAndroidArgs( + defaultAndroidConfig().apply { + platform.apply { + gcloud.apply { + app = appApkLocal + test = getString(test1) + } + flank.apply { + additionalAppTestApks = mutableListOf( + AppTestPair( + app = appApkLocal, + test = getString(test155) + ) ) - ) - ) - ), - "" + } + } + common.flank.maxTestShards = 3 + } ) with(runBlocking { config.getAndroidMatrixShards() }) { assertEquals(1, get("matrix-0")!!.shards["shard-0"]!!.size) @@ -216,27 +198,18 @@ class AndroidArgsFileTest { fun assertGcsBucket() { val oldConfig = AndroidArgs.load(localYamlFile) // Need to set the project id to get the bucket info from StorageOptions - val config = AndroidArgs( - GcloudYml( - GcloudYmlParams( - resultsBucket = oldConfig.resultsBucket - ) - ), - AndroidGcloudYml( - AndroidGcloudYmlParams( - app = oldConfig.appApk, + val config = createAndroidArgs( + defaultAndroidConfig().apply { + common.apply { + gcloud.resultsBucket = oldConfig.resultsBucket + flank.project = "flank-open-source" + } + platform.gcloud.apply { + app = oldConfig.appApk test = oldConfig.testApk - ) - ), - FlankYml( - FlankYmlParams( - project = "flank-open-source" - ) - ), - AndroidFlankYml(), - "" + } + } ) - assert(config.resultsBucket, "tmp_bucket_2") } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 3f37c1b462..6a8a049f89 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -6,7 +6,6 @@ import ftl.args.IArgs.Companion.AVAILABLE_SHARD_COUNT_RANGE import ftl.args.yml.AppTestPair import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device -import ftl.config.FlankRoboDirective import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion import ftl.gc.android.setupAndroidTest @@ -139,7 +138,7 @@ class AndroidArgsTest { app: $appApk test: $testApk test-targets: - - + - """.trimIndent() @@ -149,7 +148,7 @@ class AndroidArgsTest { @Test fun `androidArgs invalidModel`() { - expectedException.expect(RuntimeException::class.java) + expectedException.expect(FlankFatalError::class.java) expectedException.expectMessage("Unsupported model id") AndroidArgs.load( """ @@ -165,7 +164,7 @@ class AndroidArgsTest { @Test fun `androidArgs invalidVersion`() { - expectedException.expect(RuntimeException::class.java) + expectedException.expect(FlankFatalError::class.java) expectedException.expectMessage("Unsupported version id") AndroidArgs.load( """ @@ -181,7 +180,7 @@ class AndroidArgsTest { @Test fun `androidArgs incompatibleModel`() { - expectedException.expect(RuntimeException::class.java) + expectedException.expect(FlankFatalError::class.java) expectedException.expectMessage("Incompatible model") AndroidArgs.load( """ @@ -541,7 +540,6 @@ AndroidArgs val androidArgs = AndroidArgs.load(yaml, cli) assertThat(androidArgs.testApk).isEqualTo(appApkAbsolutePath) - assertThat(androidArgs.cli).isEqualTo(cli) } @Test @@ -1257,8 +1255,8 @@ AndroidArgs val args = AndroidArgs.load(yaml) assertEquals( - args.roboScript, - appApkAbsolutePath + appApkAbsolutePath, + args.roboScript ) } diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt index 6585d9a1e2..93f701207f 100644 --- a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt @@ -5,11 +5,11 @@ import ftl.args.ArgsHelper.assertCommonProps import ftl.args.ArgsHelper.assertFileExists import ftl.args.ArgsHelper.assertGcsFileExists import ftl.args.ArgsHelper.createGcsBucket -import ftl.args.ArgsHelper.mergeYmlMaps import ftl.args.ArgsHelper.validateTestMethods -import ftl.args.yml.GcloudYml -import ftl.args.yml.IosGcloudYml +import ftl.args.yml.mergeYmlKeys import ftl.config.FtlConstants +import ftl.config.common.CommonGcloudConfig +import ftl.config.ios.IosGcloudConfig import ftl.shard.TestMethod import ftl.shard.TestShard import ftl.shard.stringShards @@ -48,7 +48,7 @@ class ArgsHelperTest { @Test fun `mergeYmlMaps succeeds`() { - val merged = mergeYmlMaps(GcloudYml, IosGcloudYml) + val merged = mergeYmlKeys(CommonGcloudConfig, IosGcloudConfig) assertThat(merged.keys.size).isEqualTo(1) assertThat(merged["gcloud"]?.size).isEqualTo(11) } diff --git a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt b/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt deleted file mode 100644 index d92b0080ae..0000000000 --- a/test_runner/src/test/kotlin/ftl/args/FlankYmlTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -package ftl.args - -import com.google.common.truth.Truth.assertThat -import ftl.args.yml.FlankYml -import ftl.args.yml.FlankYmlParams -import ftl.test.util.FlankTestRunner -import org.junit.Rule -import org.junit.Test -import org.junit.contrib.java.lang.system.SystemErrRule -import org.junit.rules.ExpectedException -import org.junit.runner.RunWith - -@RunWith(FlankTestRunner::class) -class FlankYmlTest { - - @Rule - @JvmField - val exceptionRule = ExpectedException.none()!! - - @Rule - @JvmField - val systemErrRule: SystemErrRule = SystemErrRule().enableLog().muteForSuccessfulTests() - - @Test - fun testValidArgs() { - FlankYml() - FlankYml(FlankYmlParams(maxTestShards = -1)) - val yml = FlankYml(FlankYmlParams(maxTestShards = 1, repeatTests = 1, shardTime = 58)) - assertThat(yml.flank.repeatTests).isEqualTo(1) - assertThat(yml.flank.maxTestShards).isEqualTo(1) - assertThat(yml.flank.shardTime).isEqualTo(58) - assertThat(yml.flank.testTargetsAlwaysRun).isEqualTo(emptyList()) - assertThat(yml.flank.runTimeout).isEqualTo("-1") - assertThat(FlankYml.map).isNotEmpty() - } -} diff --git a/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt b/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt deleted file mode 100644 index b86798f031..0000000000 --- a/test_runner/src/test/kotlin/ftl/args/GcloudYmlTest.kt +++ /dev/null @@ -1,23 +0,0 @@ -package ftl.args - -import com.google.common.truth.Truth.assertThat -import ftl.args.yml.GcloudYml -import ftl.test.util.FlankTestRunner -import org.junit.Test -import org.junit.runner.RunWith - -@RunWith(FlankTestRunner::class) -class GcloudYmlTest { - - @Test - fun gcloudYml() { - val gcloud = GcloudYml().gcloud - gcloud.resultsBucket = "mockBucket" - assertThat(gcloud.resultsBucket) - .isEqualTo("mockBucket") - - gcloud.resultsBucket = "tmp" - assertThat(gcloud.resultsBucket) - .isEqualTo("tmp") - } -} diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 7e4f09512e..85b27e5081 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -1,16 +1,12 @@ package ftl.args import com.google.common.truth.Truth.assertThat -import ftl.args.yml.FlankYml -import ftl.args.yml.GcloudYml -import ftl.args.yml.IosFlankYml -import ftl.args.yml.IosGcloudYml -import ftl.args.yml.IosGcloudYmlParams import ftl.cli.firebase.test.ios.IosRunCommand import ftl.config.Device import ftl.config.FtlConstants import ftl.config.FtlConstants.defaultIosModel import ftl.config.FtlConstants.defaultIosVersion +import ftl.config.defaultIosConfig import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.absolutePath @@ -114,26 +110,30 @@ flank: @Test fun `args invalidDeviceExits`() { exceptionRule.expectMessage("iOS 99.9 on iphoneZ is not a supported device") - val invalidDevice = listOf(Device("iphoneZ", "99.9")) - IosArgs( - GcloudYml(), - IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, device = invalidDevice)), - FlankYml(), - IosFlankYml(), - "" - ) + val invalidDevice = mutableListOf(Device("iphoneZ", "99.9")) + createIosArgs( + config = defaultIosConfig().apply { + common.gcloud.devices = invalidDevice + platform.gcloud.also { + it.test = testPath + it.xctestrunFile = xctestrunFile + } + } + ).validate() } @Test fun `args invalidXcodeExits`() { exceptionRule.expectMessage("Xcode 99.9 is not a supported Xcode version") - IosArgs( - GcloudYml(), - IosGcloudYml(IosGcloudYmlParams(test = testPath, xctestrunFile = xctestrunFile, xcodeVersion = "99.9")), - FlankYml(), - IosFlankYml(), - "" - ) + createIosArgs( + config = defaultIosConfig().apply { + platform.gcloud.also { + it.test = testPath + it.xctestrunFile = xctestrunFile + it.xcodeVersion = "99.9" + } + } + ).validate() } @Test diff --git a/test_runner/src/test/kotlin/ftl/args/yml/ConfigTest.kt b/test_runner/src/test/kotlin/ftl/args/yml/ConfigTest.kt new file mode 100644 index 0000000000..c3dca4112c --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/args/yml/ConfigTest.kt @@ -0,0 +1,116 @@ +package ftl.args.yml + +import ftl.args.ArgsHelper.yamlMapper +import ftl.config.Config +import ftl.config.android.AndroidFlankConfig +import ftl.config.android.AndroidGcloudConfig +import ftl.config.common.CommonFlankConfig +import ftl.config.common.CommonGcloudConfig +import ftl.config.ios.IosFlankConfig +import ftl.config.ios.IosGcloudConfig +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import picocli.CommandLine +import kotlin.reflect.KClass +import kotlin.reflect.full.memberProperties + +@RunWith(Parameterized::class) +class ConfigTest( + param: Parameter +) { + companion object { + @JvmStatic + @Parameterized.Parameters + fun parameters() = listOf( + param( + new = ::CommonGcloudConfig, + default = { CommonGcloudConfig.default(true) } + ), + param( + new = ::CommonFlankConfig, + default = { CommonFlankConfig.default() } + ), + param( + new = ::AndroidGcloudConfig, + default = { AndroidGcloudConfig.default() } + ), + param( + new = ::AndroidFlankConfig, + default = { AndroidFlankConfig.default() } + ), + param( + new = ::IosGcloudConfig, + default = { IosGcloudConfig.default() } + ), + param( + new = ::IosFlankConfig, + default = { IosFlankConfig.default() } + ) + ) + } + + val default = param.default + val new = param.new + val type = param.type + + @Test + fun `validate all default values are set explicit`() { + assertEquals( + "Pls declare default values for all members explicit", + + type.memberProperties + .map { it.name } + .minus("data") + .sorted(), + + default().data.keys + .sorted() + ) + } + + @Test + fun `load empty config from empty data`() { + assertEquals( + new(), + yamlMapper.readValue("unknown-key: ", type.java) + ) + } + + @Test + fun `load empty config from empty cli`() { + assertEquals( + new(), + new().also { CommandLine(it).parseArgs() } + ) + } + + @Test + fun `reparse config`() { + assertEquals( + default(), + yamlMapper.run { + readValue( + writeValueAsString(default()), + type.java + ) + } + ) + } +} + +inline fun param( + noinline new: () -> T, + noinline default: () -> T +) = Parameter( + new = new, + default = default, + type = T::class +) + +data class Parameter( + val new: () -> T, + val default: () -> T, + val type: KClass +) diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index f8b7401412..cd5e43daa2 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -5,6 +5,7 @@ import ftl.args.yml.AppTestPair import ftl.config.Device import ftl.config.FtlConstants import ftl.test.util.FlankTestRunner +import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemOutRule @@ -58,66 +59,62 @@ class AndroidRunCommandTest { CommandLine(cmd).parseArgs() assertThat(cmd.dumpShards).isFalse() assertThat(cmd.dryRun).isFalse() - assertThat(cmd.app).isNull() - assertThat(cmd.test).isNull() - assertThat(cmd.additionalApks).isNull() - assertThat(cmd.testTargets).isNull() - assertThat(cmd.useOrchestrator).isNull() - assertThat(cmd.noUseOrchestrator).isNull() - assertThat(cmd.autoGoogleLogin).isNull() - assertThat(cmd.noUseOrchestrator).isNull() - assertThat(cmd.performanceMetrics).isNull() - assertThat(cmd.noPerformanceMetrics).isNull() - assertThat(cmd.numUniformShards).isNull() - assertThat(cmd.testRunnerClass).isNull() - assertThat(cmd.environmentVariables).isNull() - assertThat(cmd.directoriesToPull).isNull() - assertThat(cmd.otherFiles).isNull() - assertThat(cmd.device).isNull() - assertThat(cmd.resultsBucket).isNull() - assertThat(cmd.recordVideo).isNull() - assertThat(cmd.noRecordVideo).isNull() - assertThat(cmd.timeout).isNull() - assertThat(cmd.async).isNull() - assertThat(cmd.clientDetails).isNull() - assertThat(cmd.networkProfile).isNull() - assertThat(cmd.project).isNull() - assertThat(cmd.resultsHistoryName).isNull() - assertThat(cmd.maxTestShards).isNull() - assertThat(cmd.shardTime).isNull() - assertThat(cmd.repeatTests).isNull() - assertThat(cmd.testTargetsAlwaysRun).isNull() - assertThat(cmd.filesToDownload).isNull() - assertThat(cmd.resultsDir).isNull() - assertThat(cmd.flakyTestAttempts).isNull() - assertThat(cmd.disableSharding).isNull() - assertThat(cmd.localResultsDir).isNull() - assertThat(cmd.smartFlankDisableUpload).isNull() - assertThat(cmd.smartFlankGcsPath).isNull() - assertThat(cmd.additionalAppTestApks).isNull() - assertThat(cmd.keepFilePath).isNull() - assertThat(cmd.runTimeout).isNull() + assertThat(cmd.config.platform.gcloud.app).isNull() + assertThat(cmd.config.platform.gcloud.test).isNull() + assertThat(cmd.config.platform.gcloud.additionalApks).isNull() + assertThat(cmd.config.platform.gcloud.testTargets).isNull() + assertThat(cmd.config.platform.gcloud.useOrchestrator).isNull() + assertThat(cmd.config.platform.gcloud.autoGoogleLogin).isNull() + assertThat(cmd.config.platform.gcloud.performanceMetrics).isNull() + assertThat(cmd.config.platform.gcloud.numUniformShards).isNull() + assertThat(cmd.config.platform.gcloud.testRunnerClass).isNull() + assertThat(cmd.config.platform.gcloud.environmentVariables).isNull() + assertThat(cmd.config.platform.gcloud.directoriesToPull).isNull() + assertThat(cmd.config.platform.gcloud.otherFiles).isNull() + assertThat(cmd.config.common.gcloud.devices).isNull() + assertThat(cmd.config.common.gcloud.resultsBucket).isNull() + assertThat(cmd.config.common.gcloud.recordVideo).isNull() + assertThat(cmd.config.common.gcloud.timeout).isNull() + assertThat(cmd.config.common.gcloud.async).isNull() + assertThat(cmd.config.common.gcloud.clientDetails).isNull() + assertThat(cmd.config.common.gcloud.networkProfile).isNull() + assertThat(cmd.config.common.flank.project).isNull() + assertThat(cmd.config.common.gcloud.resultsHistoryName).isNull() + assertThat(cmd.config.common.flank.maxTestShards).isNull() + assertThat(cmd.config.common.flank.shardTime).isNull() + assertThat(cmd.config.common.flank.repeatTests).isNull() + assertThat(cmd.config.common.flank.testTargetsAlwaysRun).isNull() + assertThat(cmd.config.common.flank.filesToDownload).isNull() + assertThat(cmd.config.common.gcloud.resultsDir).isNull() + assertThat(cmd.config.common.gcloud.flakyTestAttempts).isNull() + assertThat(cmd.config.common.flank.disableSharding).isNull() + assertThat(cmd.config.common.flank.localResultsDir).isNull() + assertThat(cmd.config.common.flank.smartFlankDisableUpload).isNull() + assertThat(cmd.config.common.flank.smartFlankGcsPath).isNull() + assertThat(cmd.config.platform.flank.additionalAppTestApks).isNull() + assertThat(cmd.config.common.flank.keepFilePath).isNull() + assertThat(cmd.config.common.flank.runTimeout).isNull() } @Test fun `app parse`() { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--app", "myApp.apk") - assertThat(cmd.app).isEqualTo("myApp.apk") + assertThat(cmd.config.platform.gcloud.app).isEqualTo("myApp.apk") } @Test fun `test parse`() { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--test", "myTestApp.apk") - assertThat(cmd.test).isEqualTo("myTestApp.apk") + assertThat(cmd.config.platform.gcloud.test).isEqualTo("myTestApp.apk") } @Test fun `additionalApks parse`() { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--additional-apks=a.apk,b.apk") - assertThat(cmd.additionalApks).isEqualTo(listOf("a.apk", "b.apk")) + assertThat(cmd.config.platform.gcloud.additionalApks).isEqualTo(listOf("a.apk", "b.apk")) } @Test @@ -128,9 +125,9 @@ class AndroidRunCommandTest { CommandLine(cmd).parseArgs(*params) - assertThat(cmd.testTargets).isNotNull() - assertThat(cmd.testTargets?.size).isEqualTo(2) - assertThat(cmd.testTargets).isEqualTo(params.filter { it != testTargets }) + assertThat(cmd.config.platform.gcloud.testTargets).isNotNull() + assertThat(cmd.config.platform.gcloud.testTargets?.size).isEqualTo(2) + assertThat(cmd.config.platform.gcloud.testTargets).isEqualTo(params.filter { it != testTargets }) } @Test @@ -138,7 +135,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--use-orchestrator") - assertThat(cmd.useOrchestrator).isTrue() + assertThat(cmd.config.platform.gcloud.useOrchestrator).isTrue() } @Test @@ -146,7 +143,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--no-use-orchestrator") - assertThat(cmd.noUseOrchestrator).isTrue() + assertThat(cmd.config.platform.gcloud.useOrchestrator).isFalse() } @Test @@ -154,7 +151,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--auto-google-login") - assertThat(cmd.autoGoogleLogin).isTrue() + assertThat(cmd.config.platform.gcloud.autoGoogleLogin).isTrue() } @Test @@ -162,7 +159,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--no-auto-google-login") - assertThat(cmd.noAutoGoogleLogin).isTrue() + assertThat(cmd.config.platform.gcloud.autoGoogleLogin).isFalse() } @Test @@ -170,7 +167,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--performance-metrics") - assertThat(cmd.performanceMetrics).isTrue() + assertThat(cmd.config.platform.gcloud.performanceMetrics).isTrue() } @Test @@ -178,7 +175,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--no-performance-metrics") - assertThat(cmd.noPerformanceMetrics).isTrue() + assertThat(cmd.config.platform.gcloud.performanceMetrics).isFalse() } @Test @@ -187,7 +184,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--num-uniform-shards=$expected") - assertThat(cmd.numUniformShards).isEqualTo(expected) + assertThat(cmd.config.platform.gcloud.numUniformShards).isEqualTo(expected) } @Test @@ -195,7 +192,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--test-runner-class=com.foo.bar.TestRunner") - assertThat(cmd.testRunnerClass).isEqualTo("com.foo.bar.TestRunner") + assertThat(cmd.config.platform.gcloud.testRunnerClass).isEqualTo("com.foo.bar.TestRunner") } @Test @@ -203,7 +200,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--environment-variables=a=1,b=2") - assertThat(cmd.environmentVariables).hasSize(2) + assertThat(cmd.config.platform.gcloud.environmentVariables).hasSize(2) } @Test @@ -211,7 +208,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--directories-to-pull=a,b") - assertThat(cmd.directoriesToPull).hasSize(2) + assertThat(cmd.config.platform.gcloud.directoriesToPull).hasSize(2) } @Test @@ -219,7 +216,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--other-files=a=1,b=2") - assertThat(cmd.otherFiles).hasSize(2) + assertThat(cmd.config.platform.gcloud.otherFiles).hasSize(2) } @Test @@ -228,8 +225,8 @@ class AndroidRunCommandTest { CommandLine(cmd).parseArgs("--device=model=shamu,version=22,locale=zh_CN,orientation=default") val expectedDevice = Device("shamu", "22", "zh_CN", "default") - assertThat(cmd.device?.size).isEqualTo(1) - assertThat(cmd.device?.first()).isEqualTo(expectedDevice) + assertThat(cmd.config.common.gcloud.devices?.size).isEqualTo(1) + assertThat(cmd.config.common.gcloud.devices?.first()).isEqualTo(expectedDevice) } @Test @@ -237,7 +234,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--results-bucket=a") - assertThat(cmd.resultsBucket).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsBucket).isEqualTo("a") } @Test @@ -245,7 +242,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--record-video") - assertThat(cmd.recordVideo).isTrue() + assertThat(cmd.config.common.gcloud.recordVideo).isTrue() } @Test @@ -253,7 +250,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--no-record-video") - assertThat(cmd.noRecordVideo).isTrue() + assertThat(cmd.config.common.gcloud.recordVideo).isFalse() } @Test @@ -261,7 +258,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--timeout=1m") - assertThat(cmd.timeout).isEqualTo("1m") + assertThat(cmd.config.common.gcloud.timeout).isEqualTo("1m") } @Test @@ -269,7 +266,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--async") - assertThat(cmd.async).isTrue() + assertThat(cmd.config.common.gcloud.async).isTrue() } @Test @@ -277,7 +274,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--client-details=key1=value1,key2=value2") - assertThat(cmd.clientDetails).isEqualTo( + assertThat(cmd.config.common.gcloud.clientDetails).isEqualTo( mapOf( "key1" to "value1", "key2" to "value2" @@ -290,7 +287,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--network-profile=a") - assertThat(cmd.networkProfile).isEqualTo("a") + assertThat(cmd.config.common.gcloud.networkProfile).isEqualTo("a") } @Test @@ -298,7 +295,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--project=a") - assertThat(cmd.project).isEqualTo("a") + assertThat(cmd.config.common.flank.project).isEqualTo("a") } @Test @@ -306,7 +303,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--results-history-name=a") - assertThat(cmd.resultsHistoryName).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsHistoryName).isEqualTo("a") } // flankYml @@ -316,7 +313,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--max-test-shards=3") - assertThat(cmd.maxTestShards).isEqualTo(3) + assertThat(cmd.config.common.flank.maxTestShards).isEqualTo(3) } @Test @@ -324,7 +321,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--num-test-runs=3") - assertThat(cmd.repeatTests).isEqualTo(3) + assertThat(cmd.config.common.flank.repeatTests).isEqualTo(3) } @Test @@ -332,7 +329,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--test-targets-always-run=a,b,c") - assertThat(cmd.testTargetsAlwaysRun).isEqualTo(arrayListOf("a", "b", "c")) + assertThat(cmd.config.common.flank.testTargetsAlwaysRun).isEqualTo(arrayListOf("a", "b", "c")) } @Test @@ -340,7 +337,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--results-dir=a") - assertThat(cmd.resultsDir).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsDir).isEqualTo("a") } @Test @@ -348,7 +345,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--files-to-download=a,b") - assertThat(cmd.filesToDownload).isEqualTo(arrayListOf("a", "b")) + assertThat(cmd.config.common.flank.filesToDownload).isEqualTo(arrayListOf("a", "b")) } @Test @@ -356,7 +353,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--num-flaky-test-attempts=10") - assertThat(cmd.flakyTestAttempts).isEqualTo(10) + assertThat(cmd.config.common.gcloud.flakyTestAttempts).isEqualTo(10) } @Test @@ -364,7 +361,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--shard-time=99") - assertThat(cmd.shardTime).isEqualTo(99) + assertThat(cmd.config.common.flank.shardTime).isEqualTo(99) } @Test @@ -372,7 +369,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--disable-sharding") - assertThat(cmd.disableSharding).isEqualTo(true) + assertThat(cmd.config.common.flank.disableSharding).isEqualTo(true) } @Test @@ -380,7 +377,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--local-result-dir=a") - assertThat(cmd.localResultsDir).isEqualTo("a") + assertThat(cmd.config.common.flank.localResultsDir).isEqualTo("a") } @Test @@ -388,7 +385,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--smart-flank-disable-upload=true") - assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) + assertThat(cmd.config.common.flank.smartFlankDisableUpload).isEqualTo(true) } @Test @@ -396,7 +393,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--smart-flank-gcs-path=foo") - assertThat(cmd.smartFlankGcsPath).isEqualTo("foo") + assertThat(cmd.config.common.flank.smartFlankGcsPath).isEqualTo("foo") } @Test @@ -404,7 +401,7 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--keep-file-path=true") - assertThat(cmd.keepFilePath).isEqualTo(true) + assertThat(cmd.config.common.flank.keepFilePath).isEqualTo(true) } @Test @@ -413,7 +410,7 @@ class AndroidRunCommandTest { CommandLine(cmd).parseArgs("--additional-app-test-apks=app=a,test=b") val expected = AppTestPair(app = "a", test = "b") - assertThat(cmd.additionalAppTestApks).isEqualTo(listOf(expected)) + assertThat(cmd.config.platform.flank.additionalAppTestApks).isEqualTo(listOf(expected)) } @Test @@ -437,15 +434,21 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--run-timeout=20s") - assertThat(cmd.runTimeout).isEqualTo("20s") + assertThat(cmd.config.common.flank.runTimeout).isEqualTo("20s") } @Test fun `robo-directives parse`() { val cmd = AndroidRunCommand() - CommandLine(cmd).parseArgs("--robo-directives=text:a=b,click=c") + CommandLine(cmd).parseArgs("--robo-directives=text:a=b,click:c=") - assertThat(cmd.roboDirectives).hasSize(2) + assertEquals( + mapOf( + "text:a" to "b", + "click:c" to "" + ), + cmd.config.platform.gcloud.roboDirectives + ) } @Test @@ -453,6 +456,6 @@ class AndroidRunCommandTest { val cmd = AndroidRunCommand() CommandLine(cmd).parseArgs("--robo-script=a") - assertThat(cmd.roboScript).isEqualTo("a") + assertThat(cmd.config.platform.gcloud.roboScript).isEqualTo("a") } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 67bfc17663..da502ae9eb 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -1,7 +1,6 @@ package ftl.cli.firebase.test.ios import com.google.common.truth.Truth.assertThat -import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FtlConstants import ftl.config.FtlConstants.isWindows @@ -67,32 +66,31 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs() assertThat(cmd.dumpShards).isFalse() - assertThat(cmd.resultsBucket).isNull() - assertThat(cmd.recordVideo).isNull() - assertThat(cmd.noRecordVideo).isNull() - assertThat(cmd.timeout).isNull() - assertThat(cmd.async).isNull() - assertThat(cmd.clientDetails).isNull() - assertThat(cmd.networkProfile).isNull() - assertThat(cmd.project).isNull() - assertThat(cmd.resultsHistoryName).isNull() - assertThat(cmd.maxTestShards).isNull() - assertThat(cmd.shardTime).isNull() - assertThat(cmd.repeatTests).isNull() - assertThat(cmd.testTargetsAlwaysRun).isNull() - assertThat(cmd.testTargets).isNull() - assertThat(cmd.filesToDownload).isNull() - assertThat(cmd.disableSharding).isNull() - assertThat(cmd.test).isNull() - assertThat(cmd.xctestrunFile).isNull() - assertThat(cmd.xcodeVersion).isNull() - assertThat(cmd.device).isNull() - assertThat(cmd.resultsDir).isNull() - assertThat(cmd.flakyTestAttempts).isNull() - assertThat(cmd.localResultsDir).isNull() - assertThat(cmd.smartFlankDisableUpload).isNull() - assertThat(cmd.smartFlankGcsPath).isNull() - assertThat(cmd.runTimeout).isNull() + assertThat(cmd.config.common.gcloud.resultsBucket).isNull() + assertThat(cmd.config.common.gcloud.recordVideo).isNull() + assertThat(cmd.config.common.gcloud.timeout).isNull() + assertThat(cmd.config.common.gcloud.async).isNull() + assertThat(cmd.config.common.gcloud.clientDetails).isNull() + assertThat(cmd.config.common.gcloud.networkProfile).isNull() + assertThat(cmd.config.common.flank.project).isNull() + assertThat(cmd.config.common.gcloud.resultsHistoryName).isNull() + assertThat(cmd.config.common.flank.maxTestShards).isNull() + assertThat(cmd.config.common.flank.shardTime).isNull() + assertThat(cmd.config.common.flank.repeatTests).isNull() + assertThat(cmd.config.common.flank.testTargetsAlwaysRun).isNull() + assertThat(cmd.config.platform.flank.testTargets).isNull() + assertThat(cmd.config.common.flank.filesToDownload).isNull() + assertThat(cmd.config.common.flank.disableSharding).isNull() + assertThat(cmd.config.platform.gcloud.test).isNull() + assertThat(cmd.config.platform.gcloud.xctestrunFile).isNull() + assertThat(cmd.config.platform.gcloud.xcodeVersion).isNull() + assertThat(cmd.config.common.gcloud.devices).isNull() + assertThat(cmd.config.common.gcloud.resultsDir).isNull() + assertThat(cmd.config.common.gcloud.flakyTestAttempts).isNull() + assertThat(cmd.config.common.flank.localResultsDir).isNull() + assertThat(cmd.config.common.flank.smartFlankDisableUpload).isNull() + assertThat(cmd.config.common.flank.smartFlankGcsPath).isNull() + assertThat(cmd.config.common.flank.runTimeout).isNull() } @Test @@ -100,7 +98,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--results-bucket=a") - assertThat(cmd.resultsBucket).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsBucket).isEqualTo("a") } @Test @@ -108,7 +106,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--record-video") - assertThat(cmd.recordVideo).isTrue() + assertThat(cmd.config.common.gcloud.recordVideo).isTrue() } @Test @@ -116,7 +114,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--no-record-video") - assertThat(cmd.noRecordVideo).isTrue() + assertThat(cmd.config.common.gcloud.recordVideo).isFalse() } @Test @@ -124,7 +122,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--timeout=1m") - assertThat(cmd.timeout).isEqualTo("1m") + assertThat(cmd.config.common.gcloud.timeout).isEqualTo("1m") } @Test @@ -132,15 +130,15 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--async") - assertThat(cmd.async).isTrue() + assertThat(cmd.config.common.gcloud.async).isTrue() } @Test fun `clientDetails parse`() { - val cmd = AndroidRunCommand() + val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--client-details=key1=value1,key2=value2") - assertThat(cmd.clientDetails).isEqualTo( + assertThat(cmd.config.common.gcloud.clientDetails).isEqualTo( mapOf( "key1" to "value1", "key2" to "value2" @@ -150,10 +148,10 @@ class IosRunCommandTest { @Test fun `networkProfile parse`() { - val cmd = AndroidRunCommand() + val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--network-profile=a") - assertThat(cmd.networkProfile).isEqualTo("a") + assertThat(cmd.config.common.gcloud.networkProfile).isEqualTo("a") } @Test @@ -161,7 +159,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--project=a") - assertThat(cmd.project).isEqualTo("a") + assertThat(cmd.config.common.flank.project).isEqualTo("a") } @Test @@ -169,7 +167,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--results-history-name=a") - assertThat(cmd.resultsHistoryName).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsHistoryName).isEqualTo("a") } // flankYml @@ -179,7 +177,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--max-test-shards=3") - assertThat(cmd.maxTestShards).isEqualTo(3) + assertThat(cmd.config.common.flank.maxTestShards).isEqualTo(3) } @Test @@ -187,7 +185,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--num-test-runs=3") - assertThat(cmd.repeatTests).isEqualTo(3) + assertThat(cmd.config.common.flank.repeatTests).isEqualTo(3) } @Test @@ -195,7 +193,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--test-targets-always-run=a,b,c") - assertThat(cmd.testTargetsAlwaysRun).isEqualTo(arrayListOf("a", "b", "c")) + assertThat(cmd.config.common.flank.testTargetsAlwaysRun).isEqualTo(arrayListOf("a", "b", "c")) } @Test @@ -203,7 +201,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--test-targets=a,b,c") - assertThat(cmd.testTargets).isEqualTo(arrayListOf("a", "b", "c")) + assertThat(cmd.config.platform.flank.testTargets).isEqualTo(arrayListOf("a", "b", "c")) } @Test @@ -211,7 +209,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--test=a") - assertThat(cmd.test).isEqualTo("a") + assertThat(cmd.config.platform.gcloud.test).isEqualTo("a") } @Test @@ -219,7 +217,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--xctestrun-file=a") - assertThat(cmd.xctestrunFile).isEqualTo("a") + assertThat(cmd.config.platform.gcloud.xctestrunFile).isEqualTo("a") } @Test @@ -227,7 +225,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--xcode-version=999") - assertThat(cmd.xcodeVersion).isEqualTo("999") + assertThat(cmd.config.platform.gcloud.xcodeVersion).isEqualTo("999") } @Test @@ -236,8 +234,8 @@ class IosRunCommandTest { CommandLine(cmd).parseArgs("--device=model=iphone8,version=11.2,locale=zh_CN,orientation=default") val expectedDevice = Device("iphone8", "11.2", "zh_CN", "default") - assertThat(cmd.device?.size).isEqualTo(1) - assertThat(cmd.device?.first()).isEqualTo(expectedDevice) + assertThat(cmd.config.common.gcloud.devices?.size).isEqualTo(1) + assertThat(cmd.config.common.gcloud.devices?.first()).isEqualTo(expectedDevice) } @Test @@ -245,7 +243,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--results-dir=a") - assertThat(cmd.resultsDir).isEqualTo("a") + assertThat(cmd.config.common.gcloud.resultsDir).isEqualTo("a") } @Test @@ -253,7 +251,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--files-to-download=a,b") - assertThat(cmd.filesToDownload).isEqualTo(arrayListOf("a", "b")) + assertThat(cmd.config.common.flank.filesToDownload).isEqualTo(arrayListOf("a", "b")) } @Test @@ -261,7 +259,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--num-flaky-test-attempts=10") - assertThat(cmd.flakyTestAttempts).isEqualTo(10) + assertThat(cmd.config.common.gcloud.flakyTestAttempts).isEqualTo(10) } @Test @@ -269,7 +267,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--shard-time=99") - assertThat(cmd.shardTime).isEqualTo(99) + assertThat(cmd.config.common.flank.shardTime).isEqualTo(99) } @Test @@ -277,7 +275,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--disable-sharding") - assertThat(cmd.disableSharding).isEqualTo(true) + assertThat(cmd.config.common.flank.disableSharding).isEqualTo(true) } @Test @@ -285,7 +283,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--local-result-dir=a") - assertThat(cmd.localResultsDir).isEqualTo("a") + assertThat(cmd.config.common.flank.localResultsDir).isEqualTo("a") } @Test @@ -293,7 +291,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--smart-flank-disable-upload=true") - assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) + assertThat(cmd.config.common.flank.smartFlankDisableUpload).isEqualTo(true) } @Test @@ -301,7 +299,7 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--smart-flank-gcs-path=foo") - assertThat(cmd.smartFlankGcsPath).isEqualTo("foo") + assertThat(cmd.config.common.flank.smartFlankGcsPath).isEqualTo("foo") } @Test @@ -317,6 +315,6 @@ class IosRunCommandTest { val cmd = IosRunCommand() CommandLine(cmd).parseArgs("--run-timeout=20s") - assertThat(cmd.runTimeout).isEqualTo("20s") + assertThat(cmd.config.common.flank.runTimeout).isEqualTo("20s") } } diff --git a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt index 6ff6ccd918..a9e20265f4 100644 --- a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt +++ b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt @@ -2,13 +2,13 @@ package ftl.doctor import com.google.common.truth.Truth.assertThat import ftl.args.AndroidArgs -import ftl.args.IArgsCompanion +import ftl.args.IArgs import ftl.args.IosArgs import ftl.test.util.FlankTestRunner -import java.nio.file.Paths import org.junit.Test import org.junit.runner.RunWith import java.io.StringReader +import java.nio.file.Paths @RunWith(FlankTestRunner::class) class DoctorTest { @@ -148,4 +148,4 @@ flank: } } -private fun Doctor.validateYaml(args: IArgsCompanion, data: String): String = validateYaml(args, StringReader(data)) +private fun Doctor.validateYaml(args: IArgs.ICompanion, data: String): String = validateYaml(args, StringReader(data)) diff --git a/test_runner/src/test/kotlin/ftl/config/FlankBugsnagInitHelperTest.kt b/test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt similarity index 99% rename from test_runner/src/test/kotlin/ftl/config/FlankBugsnagInitHelperTest.kt rename to test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt index 2b158fd158..af9a1ac9ce 100644 --- a/test_runner/src/test/kotlin/ftl/config/FlankBugsnagInitHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt @@ -1,4 +1,4 @@ -package ftl.config +package ftl.util import ftl.log.LogbackLogger import io.mockk.every