diff --git a/release_notes.md b/release_notes.md index 04ebd77606..d9c76d07ed 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,5 @@ ## next (unreleased) -- +- [#541](https://github.com/TestArmada/flank/pull/541) Rename `--test-shards` CLI flag to `--max-test-shards`. Add `--smart-flank-gcs-path` CLI flag. ([bootstraponline](https://github.com/bootstraponline)) ## v5.1.0 - [#537](https://github.com/TestArmada/flank/pull/537) Add `smart-flank-disable-upload` yml option to prevent new results from overriding previous results. ([elihart](https://github.com/elihart)) diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index cd8b133249..ce0053aadc 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -68,7 +68,7 @@ class AndroidArgs( override val maxTestShards = cli?.maxTestShards ?: flank.maxTestShards override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests - override val smartFlankGcsPath = flank.smartFlankGcsPath + override val smartFlankGcsPath = cli?.smartFlankGcsPath ?: flank.smartFlankGcsPath override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 615615bb0f..b3ac05d0d2 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -51,7 +51,7 @@ class IosArgs( override val maxTestShards = cli?.maxTestShards ?: flank.maxTestShards override val shardTime = cli?.shardTime ?: flank.shardTime override val repeatTests = cli?.repeatTests ?: flank.repeatTests - override val smartFlankGcsPath = flank.smartFlankGcsPath + override val smartFlankGcsPath = cli?.smartFlankGcsPath ?: flank.smartFlankGcsPath override val smartFlankDisableUpload = cli?.smartFlankDisableUpload ?: flank.smartFlankDisableUpload override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 623f5ba044..b5af34ae10 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -36,6 +36,8 @@ class AndroidRunCommand : Runnable { } } + // Flank specific + @Option( names = ["-c", "--config"], description = ["YAML config file path"] @@ -49,6 +51,8 @@ class AndroidRunCommand : Runnable { ) var usageHelpRequested: Boolean = false + // AndroidGcloudYml.kt + @Option( names = ["--app"], description = ["The path to the application binary file. " + @@ -64,23 +68,25 @@ class AndroidRunCommand : Runnable { var test: String? = null @Option( - names = ["--test-targets"], - split = ",", - description = ["A list of one or more test target filters to apply " + - "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + - "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + - "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + - "information."] + names = ["--auto-google-login"], + description = ["Automatically log into the test device using a preconfigured " + + "Google account before beginning the test. Enabled by default, use --no-auto-google-login to disable."] ) - var testTargets: List? = null + var autoGoogleLogin: Boolean? = null + + @Option( + names = ["--no-auto-google-login"], + description = ["Google account not logged in. See --auto-google-login."] + ) + var noAutoGoogleLogin: Boolean? = null @Option( names = ["--use-orchestrator"], description = ["Whether each test runs in its own Instrumentation instance " + - "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + - "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + - "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + - "information about Android Test Orchestrator."] + "with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). " + + "Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See " + + "https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more " + + "information about Android Test Orchestrator."] ) var useOrchestrator: Boolean? = null @@ -91,22 +97,31 @@ class AndroidRunCommand : Runnable { var noUseOrchestrator: Boolean? = null @Option( - names = ["--auto-google-login"], - description = ["Automatically log into the test device using a preconfigured " + - "Google account before beginning the test. Enabled by default, use --no-auto-google-login to disable."] + names = ["--environment-variables"], + split = ",", + description = ["A comma-separated, key=value map of environment variables " + + "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + + "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + + "passed to your test runner (typically AndroidJUnitRunner)"] ) - var autoGoogleLogin: Boolean? = null + var environmentVariables: Map? = null @Option( - names = ["--no-auto-google-login"], - description = ["Google account not logged in. See --auto-google-login."] + names = ["--directories-to-pull"], + split = ",", + description = ["A list of paths that will be copied from the device's " + + "storage to the designated results bucket after the test is complete. These must be absolute paths under " + + "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + + "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + + "storage, the system will replace it with the external storage path prefix for that device."] ) - var noAutoGoogleLogin: Boolean? = null + var directoriesToPull: List? = null @Option( names = ["--performance-metrics"], description = ["Monitor and record performance metrics: CPU, memory, " + - "network usage, and FPS (game-loop only). Enabled by default, use --no-performance-metrics to disable."] + "network usage, and FPS (game-loop only). Enabled by default, use --no-performance-metrics to disable."] ) var performanceMetrics: Boolean? = null @@ -117,26 +132,15 @@ class AndroidRunCommand : Runnable { var noPerformanceMetrics: Boolean? = null @Option( - names = ["--environment-variables"], - split = ",", - description = ["A comma-separated, key=value map of environment variables " + - "and their desired values. --environment-variables=coverage=true,coverageFile=/sdcard/coverage.ec " + - "The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and " + - "passed to your test runner (typically AndroidJUnitRunner)"] - ) - var environmentVariables: Map? = null - - @Option( - names = ["--directories-to-pull"], + names = ["--test-targets"], split = ",", - description = ["A list of paths that will be copied from the device's " + - "storage to the designated results bucket after the test is complete. These must be absolute paths under " + - "/sdcard or /data/local/tmp (for example, --directories-to-pull /sdcard/tempDir1,/data/local/tmp/tempDir2). " + - "Path names are restricted to the characters a-zA-Z0-9_-./+. The paths /sdcard and /data will be made available " + - "and treated as implicit path substitutions. E.g. if /sdcard on a particular device does not map to external " + - "storage, the system will replace it with the external storage path prefix for that device."] + description = ["A list of one or more test target filters to apply " + + "(default: run all test targets). Each target filter must be fully qualified with the package name, class name, " + + "or test annotation desired. Any test filter supported by am instrument -e … is supported. " + + "See https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner for more " + + "information."] ) - var directoriesToPull: List? = null + var testTargets: List? = null @Option( names = ["--device"], @@ -162,6 +166,8 @@ class AndroidRunCommand : Runnable { var device: MutableList? = null + // GcloudYml.kt + @Option( names = ["--results-bucket"], description = ["The name of a Google Cloud Storage bucket where raw test " + @@ -171,6 +177,16 @@ class AndroidRunCommand : Runnable { ) var resultsBucket: String? = null + @Option( + names = ["--results-dir"], + description = [ + "The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + + "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + + "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + + "intermingled."] + ) + var resultsDir: String? = null + @Option( names = ["--record-video"], description = ["Enable video recording during the test. " + @@ -187,9 +203,9 @@ class AndroidRunCommand : Runnable { @Option( names = ["--timeout"], description = ["The max time this test execution can run before it is cancelled " + - "(default: 15m). It does not include any time necessary to prepare and clean up the target device. The maximum " + - "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " + - "or s. If no unit is given, seconds are assumed. "] + "(default: 15m). It does not include any time necessary to prepare and clean up the target device. The maximum " + + "possible testing time is 30m on physical devices and 60m on virtual devices. The TIMEOUT units can be h, m, " + + "or s. If no unit is given, seconds are assumed. "] ) var timeout: String? = null @@ -199,24 +215,26 @@ class AndroidRunCommand : Runnable { ) var async: Boolean? = null - @Option( - names = ["--project"], - description = ["The Google Cloud Platform project name to use for this invocation. " + - "If omitted, then the project from the service account credential is used"] - ) - var project: String? = null - @Option( names = ["--results-history-name"], description = ["The history name for your test results " + - "(an arbitrary string label; default: the application's label from the APK manifest). All tests which use the " + - "same history name will have their results grouped together in the Firebase console in a time-ordered test " + - "history list."] + "(an arbitrary string label; default: the application's label from the APK manifest). All tests which use the " + + "same history name will have their results grouped together in the Firebase console in a time-ordered test " + + "history list."] ) var resultsHistoryName: String? = null @Option( - names = ["--test-shards"], + names = ["--flaky-test-attempts"], + description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + + "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] + ) + var flakyTestAttempts: Int? = null + + // FlankYml.kt + + @Option( + names = ["--max-test-shards"], description = ["The amount of matrices to split the tests across."] ) var maxTestShards: Int? = null @@ -233,6 +251,25 @@ class AndroidRunCommand : Runnable { ) var repeatTests: Int? = null + @Option( + names = ["--smart-flank-gcs-path"], + split = ",", + description = ["Google cloud storage path to save test timing data used by smart flank."] + ) + var smartFlankGcsPath: String? = null + + @Option( + names = ["--smart-flank-disable-upload"], + description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] + ) + var smartFlankDisableUpload: Boolean? = null + + @Option( + names = ["--disable-sharding"], + description = ["Disable sharding."] + ) + var disableSharding: Boolean? = null + @Option( names = ["--test-targets-always-run"], split = ",", @@ -252,37 +289,15 @@ class AndroidRunCommand : Runnable { var filesToDownload: List? = null @Option( - names = ["--disable-sharding"], - description = ["Disable sharding."] - ) - var disableSharding: Boolean? = null - - @Option( - names = ["--results-dir"], - description = [ - "The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + - "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + - "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + - "intermingled."] - ) - var resultsDir: String? = null - - @Option( - names = ["--flaky-test-attempts"], - description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + - "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] + names = ["--project"], + description = ["The Google Cloud Platform project name to use for this invocation. " + + "If omitted, then the project from the service account credential is used"] ) - var flakyTestAttempts: Int? = null + var project: String? = null @Option( names = ["--local-result-dir"], description = ["Saves test result to this local folder. Deleted before each run."] ) var localResultDir: String? = null - - @Option( - names = ["--smart-flank-disable-upload"], - description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] - ) - var smartFlankDisableUpload: Boolean? = null } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 1217a009f9..5d1a74fdf1 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -33,6 +33,8 @@ class IosRunCommand : Runnable { } } + // Flank specific + @Option( names = ["-c", "--config"], description = ["YAML config file path"] @@ -46,6 +48,60 @@ class IosRunCommand : Runnable { ) var usageHelpRequested: Boolean = false + // IosGcloudYml.kt + + @Option( + names = ["--test"], + description = ["The path to the test package (a zip file containing the iOS app " + + "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + + "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + + "is specified."] + ) + var test: String? = null + + @Option( + names = ["--xctestrun-file"], + description = ["The path to an .xctestrun file that will override any " + + ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + + "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + + "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] + ) + var xctestrunFile: String? = null + + @Option( + names = ["--xcode-version"], + description = ["The version of Xcode that should be used to run an XCTest. " + + "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + + "all iOS versions selected in the test matrix."] + ) + var xcodeVersion: String? = null + + @Option( + names = ["--device"], + split = ",", + description = ["A list of DIMENSION=VALUE pairs which specify a target " + + "device to test against. This flag may be repeated to specify multiple devices. The four device dimensions are: " + + "model, version, locale, and orientation. If any dimensions are omitted, they will use a default value. Omitting " + + "all of the preceding dimension-related flags will run tests against a single device using defaults for all four " + + "device dimensions."] + ) + fun deviceMap(map: Map?) { + if (map.isNullOrEmpty()) return + val androidDevice = Device( + model = map.getOrDefault("model", defaultIosModel), + version = map.getOrDefault("version", defaultIosVersion), + locale = map.getOrDefault("locale", FtlConstants.defaultLocale), + orientation = map.getOrDefault("orientation", FtlConstants.defaultOrientation) + ) + + if (device == null) device = mutableListOf() + device?.add(androidDevice) + } + + var device: MutableList? = null + + // GcloudYml + @Option( names = ["--results-bucket"], description = ["The name of a Google Cloud Storage bucket where raw test " + @@ -55,6 +111,16 @@ class IosRunCommand : Runnable { ) var resultsBucket: String? = null + @Option( + names = ["--results-dir"], + description = [ + "The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + + "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + + "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + + "intermingled."] + ) + var resultsDir: String? = null + @Option( names = ["--record-video"], description = ["Enable video recording during the test. " + @@ -83,13 +149,6 @@ class IosRunCommand : Runnable { ) var async: Boolean? = null - @Option( - names = ["--project"], - description = ["The Google Cloud Platform project name to use for this invocation. " + - "If omitted, then the project from the service account credential is used"] - ) - var project: String? = null - @Option( names = ["--results-history-name"], description = ["The history name for your test results " + @@ -100,7 +159,15 @@ class IosRunCommand : Runnable { var resultsHistoryName: String? = null @Option( - names = ["--test-shards"], + names = ["--flaky-test-attempts"], + description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + + "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] + ) + var flakyTestAttempts: Int? = null + + // FlankYml.kt + @Option( + names = ["--max-test-shards"], description = ["The amount of matrices to split the tests across."] ) var maxTestShards: Int? = null @@ -118,29 +185,17 @@ class IosRunCommand : Runnable { var repeatTests: Int? = null @Option( - names = ["--test-targets-always-run"], - split = ",", - description = ["A list of one or more test methods to always run first in every shard."] - ) - var testTargetsAlwaysRun: List? = null - - @Option( - names = ["--test-targets"], + names = ["--smart-flank-gcs-path"], split = ",", - description = ["A list of one or more test method " + - "names to run (default: run all test targets)."] + description = ["Google cloud storage path to save test timing data used by smart flank."] ) - var testTargets: List? = null + var smartFlankGcsPath: String? = null @Option( - names = ["--files-to-download"], - split = ",", - description = ["A list of paths that will be downloaded from the resulting bucket " + - "to the local results folder after the test is complete. These must be absolute paths " + - "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " + - "Path names are restricted to the characters a-zA-Z0-9_-./+."] + names = ["--smart-flank-disable-upload"], + description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] ) - var filesToDownload: List? = null + var smartFlankDisableUpload: Boolean? = null @Option( names = ["--disable-sharding"], @@ -149,71 +204,28 @@ class IosRunCommand : Runnable { var disableSharding: Boolean? = null @Option( - names = ["--test"], - description = ["The path to the test package (a zip file containing the iOS app " + - "and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL " + - "beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file " + - "is specified."] - ) - var test: String? = null - - @Option( - names = ["--xctestrun-file"], - description = ["The path to an .xctestrun file that will override any " + - ".xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables " + - "along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The " + - "given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://."] - ) - var xctestrunFile: String? = null - - @Option( - names = ["--xcode-version"], - description = ["The version of Xcode that should be used to run an XCTest. " + - "Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by " + - "all iOS versions selected in the test matrix."] - ) - var xcodeVersion: String? = null - - @Option( - names = ["--device"], + names = ["--test-targets-always-run"], split = ",", - description = ["A list of DIMENSION=VALUE pairs which specify a target " + - "device to test against. This flag may be repeated to specify multiple devices. The four device dimensions are: " + - "model, version, locale, and orientation. If any dimensions are omitted, they will use a default value. Omitting " + - "all of the preceding dimension-related flags will run tests against a single device using defaults for all four " + - "device dimensions."] + description = ["A list of one or more test methods to always run first in every shard."] ) - fun deviceMap(map: Map?) { - if (map.isNullOrEmpty()) return - val androidDevice = Device( - model = map.getOrDefault("model", defaultIosModel), - version = map.getOrDefault("version", defaultIosVersion), - locale = map.getOrDefault("locale", FtlConstants.defaultLocale), - orientation = map.getOrDefault("orientation", FtlConstants.defaultOrientation) - ) - - if (device == null) device = mutableListOf() - device?.add(androidDevice) - } - - var device: MutableList? = null + var testTargetsAlwaysRun: List? = null @Option( - names = ["--results-dir"], - description = [ - "The name of a unique Google Cloud Storage object within the results bucket where raw test results will be " + - "stored (default: a timestamp with a random suffix). Caution: if specified, this argument must be unique for " + - "each test matrix you create, otherwise results from multiple test matrices will be overwritten or " + - "intermingled."] + names = ["--files-to-download"], + split = ",", + description = ["A list of paths that will be downloaded from the resulting bucket " + + "to the local results folder after the test is complete. These must be absolute paths " + + "(for example, --files-to-download /images/tempDir1,/data/local/tmp/tempDir2). " + + "Path names are restricted to the characters a-zA-Z0-9_-./+."] ) - var resultsDir: String? = null + var filesToDownload: List? = null @Option( - names = ["--flaky-test-attempts"], - description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " + - "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."] + names = ["--project"], + description = ["The Google Cloud Platform project name to use for this invocation. " + + "If omitted, then the project from the service account credential is used"] ) - var flakyTestAttempts: Int? = null + var project: String? = null @Option( names = ["--local-result-dir"], @@ -221,9 +233,13 @@ class IosRunCommand : Runnable { ) var localResultsDir: String? = null + // IosFlankYml.kt + @Option( - names = ["--smart-flank-disable-upload"], - description = ["Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated."] + names = ["--test-targets"], + split = ",", + description = ["A list of one or more test method " + + "names to run (default: run all test targets)."] ) - var smartFlankDisableUpload: Boolean? = null + var testTargets: List? = null } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index cc49fcfc26..b93086fcb6 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -687,7 +687,7 @@ AndroidArgs @Test fun cli_maxTestShards() { val cli = AndroidRunCommand() - CommandLine(cli).parse("--test-shards=3") + CommandLine(cli).parse("--max-test-shards=3") val yaml = """ gcloud: @@ -797,6 +797,22 @@ AndroidArgs assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) } + @Test + fun `cli smart-flank-gcs-path`() { + val cli = AndroidRunCommand() + CommandLine(cli).parse("--smart-flank-gcs-path=foo") + + val yaml = """ + gcloud: + app: $appApk + test: $testApk + """ + assertThat(AndroidArgs.load(yaml).smartFlankGcsPath).isEqualTo("") + + val args = AndroidArgs.load(yaml, cli) + assertThat(args.smartFlankGcsPath).isEqualTo("foo") + } + @Test fun `cli local-result-dir`() { val cli = AndroidRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index a5fd397143..f07f5da41a 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -76,7 +76,7 @@ class IosArgsTest { val systemErrRule = SystemErrRule().muteForSuccessfulTests()!! @Test - fun iosArgs_invalidDeviceExits() { + fun args_invalidDeviceExits() { exceptionRule.expectMessage("iOS 99.9 on iphoneZ is not a supported device") val invalidDevice = listOf(Device("iphoneZ", "99.9")) IosArgs( @@ -89,7 +89,7 @@ class IosArgsTest { } @Test - fun iosArgs_invalidXcodeExits() { + fun args_invalidXcodeExits() { exceptionRule.expectMessage("Xcode 99.9 is not a supported Xcode version") IosArgs( GcloudYml(), @@ -101,10 +101,10 @@ class IosArgsTest { } @Test - fun iosArgs() { - val iosArgs = IosArgs.load(iosNonDefault) + fun args() { + val args = IosArgs.load(iosNonDefault) - with(iosArgs) { + with(args) { // GcloudYml assert(resultsBucket, "mockBucket") assert(recordVideo, false) @@ -135,10 +135,10 @@ class IosArgsTest { } @Test - fun iosArgs_toString() { - val iosArgs = IosArgs.load(iosNonDefault) + fun args_toString() { + val args = IosArgs.load(iosNonDefault) assert( - iosArgs.toString(), """ + args.toString(), """ IosArgs gcloud: results-bucket: mockBucket @@ -185,8 +185,8 @@ IosArgs } @Test - fun iosArgsDefault() { - val iosArgs = IosArgs.load( + fun argsDefault() { + val args = IosArgs.load( """ gcloud: test: $testPath @@ -194,7 +194,7 @@ IosArgs """ ) - with(iosArgs) { + with(args) { // GcloudYml assert(resultsBucket, "mockBucket") assert(recordVideo, true) @@ -223,7 +223,7 @@ IosArgs @Test fun negativeOneTestShards() { - val iosArgs = IosArgs.load( + val args = IosArgs.load( """ gcloud: test: $testPath @@ -234,7 +234,7 @@ IosArgs """ ) - with(iosArgs) { + with(args) { assert(maxTestShards, -1) assert(testShardChunks.size, 17) testShardChunks.forEach { chunk -> assert(chunk.size, 1) } @@ -242,8 +242,8 @@ IosArgs } @Test - fun iosArgs_emptyFlank() { - val iosArgs = IosArgs.load( + fun args_emptyFlank() { + val args = IosArgs.load( """ gcloud: test: $testPath @@ -253,7 +253,7 @@ IosArgs """ ) - with(iosArgs) { + with(args) { assertThat(xctestrunZip).isEqualTo(testAbsolutePath) assertThat(xctestrunFile).isEqualTo(xctestrunFileAbsolutePath) } @@ -393,7 +393,7 @@ IosArgs @Test fun cli_maxTestShards() { val cli = IosRunCommand() - CommandLine(cli).parse("--test-shards=3") + CommandLine(cli).parse("--max-test-shards=3") val yaml = """ gcloud: @@ -545,9 +545,9 @@ IosArgs assertThat(defaultDevices.first()).isEqualTo(expectedDefaultDevice) assertThat(defaultDevices.size).isEqualTo(1) - val iosArgs = IosArgs.load(yaml, cli) + val args = IosArgs.load(yaml, cli) val expectedDevice = Device("iphone8", "12.0", "zh_CN", "default") - val actualDevices = iosArgs.devices + val actualDevices = args.devices assertThat(actualDevices.first()).isEqualTo(expectedDevice) assertThat(actualDevices.size).isEqualTo(1) } @@ -563,9 +563,9 @@ IosArgs test: $testPath xctestrun-file: $testPath """ - val iosArgs = IosArgs.load(yaml, cli) + val args = IosArgs.load(yaml, cli) val expectedDevice = Device("iphone8", "12.0", "zh_CN", "default") - val actualDevices = iosArgs.devices + val actualDevices = args.devices assertThat(actualDevices.size).isEqualTo(2) assertThat(actualDevices[0]).isEqualTo(expectedDevice) assertThat(actualDevices[1]).isEqualTo(expectedDevice) @@ -599,8 +599,8 @@ IosArgs """ assertThat(IosArgs.load(yaml).filesToDownload).isEmpty() - val androidArgs = IosArgs.load(yaml, cli) - assertThat(androidArgs.filesToDownload).isEqualTo(listOf("a", "b")) + val args = IosArgs.load(yaml, cli) + assertThat(args.filesToDownload).isEqualTo(listOf("a", "b")) } @Test @@ -615,8 +615,8 @@ IosArgs """ assertThat(IosArgs.load(yaml).flakyTestAttempts).isEqualTo(0) - val androidArgs = IosArgs.load(yaml, cli) - assertThat(androidArgs.flakyTestAttempts).isEqualTo(3) + val args = IosArgs.load(yaml, cli) + assertThat(args.flakyTestAttempts).isEqualTo(3) } @Test @@ -631,8 +631,24 @@ IosArgs """ assertThat(IosArgs.load(yaml).localResultDir).isEqualTo("results") - val androidArgs = IosArgs.load(yaml, cli) - assertThat(androidArgs.localResultDir).isEqualTo("a") + val args = IosArgs.load(yaml, cli) + assertThat(args.localResultDir).isEqualTo("a") + } + + @Test + fun `cli smart-flank-gcs-path`() { + val cli = IosRunCommand() + CommandLine(cli).parse("--smart-flank-gcs-path=foo") + + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + """ + assertThat(IosArgs.load(yaml).smartFlankGcsPath).isEqualTo("") + + val args = IosArgs.load(yaml, cli) + assertThat(args.smartFlankGcsPath).isEqualTo("foo") } @Test @@ -647,8 +663,8 @@ IosArgs """ assertThat(IosArgs.load(yaml).smartFlankDisableUpload).isEqualTo(false) - val androidArgs = IosArgs.load(yaml, cli) - assertThat(androidArgs.smartFlankDisableUpload).isEqualTo(true) + val args = IosArgs.load(yaml, cli) + assertThat(args.smartFlankDisableUpload).isEqualTo(true) } private fun getValidTestsSample() = listOf( diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index 91f5849cce..ec5b0ad95d 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -89,6 +89,7 @@ class AndroidRunCommandTest { assertThat(cmd.disableSharding).isNull() assertThat(cmd.localResultDir).isNull() assertThat(cmd.smartFlankDisableUpload).isNull() + assertThat(cmd.smartFlankGcsPath).isNull() } @Test @@ -253,7 +254,7 @@ class AndroidRunCommandTest { @Test fun maxTestShards_parse() { val cmd = AndroidRunCommand() - CommandLine(cmd).parse("--test-shards=3") + CommandLine(cmd).parse("--max-test-shards=3") assertThat(cmd.maxTestShards).isEqualTo(3) } @@ -329,4 +330,12 @@ class AndroidRunCommandTest { assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) } + + @Test + fun `smartFlankGcsPath parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parse("--smart-flank-gcs-path=foo") + + assertThat(cmd.smartFlankGcsPath).isEqualTo("foo") + } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 10ccb05f51..a002475647 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -83,6 +83,7 @@ class IosRunCommandTest { assertThat(cmd.flakyTestAttempts).isNull() assertThat(cmd.localResultsDir).isNull() assertThat(cmd.smartFlankDisableUpload).isNull() + assertThat(cmd.smartFlankGcsPath).isNull() } @Test @@ -146,7 +147,7 @@ class IosRunCommandTest { @Test fun maxTestShards_parse() { val cmd = IosRunCommand() - CommandLine(cmd).parse("--test-shards=3") + CommandLine(cmd).parse("--max-test-shards=3") assertThat(cmd.maxTestShards).isEqualTo(3) } @@ -264,4 +265,12 @@ class IosRunCommandTest { assertThat(cmd.smartFlankDisableUpload).isEqualTo(true) } + + @Test + fun `smart-flank-gcs-path parse`() { + val cmd = IosRunCommand() + CommandLine(cmd).parse("--smart-flank-gcs-path=foo") + + assertThat(cmd.smartFlankGcsPath).isEqualTo("foo") + } }