diff --git a/docs/index.md b/docs/index.md index c9f7a721c2..db23b40e7c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -72,26 +72,32 @@ Run `test_runner/flank.ios.yml` with flank to verify iOS execution is working. # https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run gcloud: # -- GcloudYml -- - + + ### Results Bucket ## The name of a Google Cloud Storage bucket where raw test results will be stored # results-bucket: tmp_flank - + + ### Results Directory ## The name of a unique Google Cloud Storage object within the results bucket where raw test results will be stored ## (default: a timestamp with a random suffix). # results-dir: tmp + ### Record Video flag ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true + ### Timeout ## The max time this test execution can run before it is cancelled (default: 15m). ## It does not include any time necessary to prepare and clean up the target device. ## The maximum possible testing time is 45m on physical devices and 60m on virtual devices. ## The TIMEOUT units can be h, m, or s. If no unit is given, seconds are assumed. # timeout: 30m + ### Asynchronous flag ## Invoke a test asynchronously without waiting for test results. # async: false + ### Client Details ## A key-value map of additional details to attach to the test matrix. ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run. ## When consuming the test results, such as in Cloud Functions or a CI system, @@ -100,27 +106,32 @@ gcloud: # key1: value1 # key2: value2 + ### Network Profile ## The name of the network traffic profile, for example LTE, HSPA, etc, ## which consists of a set of parameters to emulate network conditions when running the test ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ## This feature only works on physical devices. # network-profile: LTE + ### Result History Name ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history + ### Number of Flaky Test Attempts ## The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. ## The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. # num-flaky-test-attempts: 0 # -- IosGcloudYml -- + ### IOS Test Package Path ## The path to the test package (a zip file containing the iOS app and XCTest files). ## The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. ## Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. test: ./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + ### IOS XCTestrun File Path ## The path to an .xctestrun file that will override any .xctestrun file contained in the --test package. ## Because the .xctestrun file contains environment variables along with test methods to run and/or ignore, ## this can be useful for customizing or sharding test suites. The given path should be in the local filesystem. @@ -128,11 +139,13 @@ gcloud: ## For example ./derivedDataPath/Build/Products/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + ### Xcode Version ## The version of Xcode that should be used to run an XCTest. ## Defaults to the latest Xcode version supported in Firebase Test Lab. ## This Xcode version must be supported by all iOS versions selected in the test matrix. # xcode-version: 10.1 + ### IOS Device Parameters ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. @@ -146,12 +159,14 @@ gcloud: # locale: es_ES # orientation: landscape + ### Directories to Pull ## A list of paths that will be copied from the device's storage to the designated results bucket after the test ## is complete. These must be absolute paths under /private/var/mobile/Media or /Documents ## of the app under test. If the path is under an app's /Documents, it must be prefixed with the app's bundle id and a colon # directories-to-pull: # - /private/var/mobile/Media + ### Other File paths ## A list of device-path=file-path pairs that specify the paths of the test device and the files you want pushed to the device prior to testing. ## Device paths should either be under the Media shared folder (e.g. prefixed with /private/var/mobile/Media) or ## within the documents directory of the filesystem of an app under test (e.g. /Documents). Device paths to app @@ -160,12 +175,14 @@ gcloud: # com.my.app:/Documents/file.txt: local/file.txt # /private/var/mobile/Media/file.jpg: gs://bucket/file.jpg + ### Additional IPA's ## List of up to 100 additional IPAs to install, in addition to the one being directly tested. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. # additional-ipas: # - gs://bucket/additional.ipa # - path/to/local/ipa/file.ipa + ### Scenario Numbers ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. # scenario-numbers: @@ -173,16 +190,18 @@ gcloud: # - 2 # - 3 + ### Test type ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest # type: xctest + ### Application Path ## The path to the application archive (.ipa file) for game-loop testing. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## This flag is only valid when --type=game-loop is also set # app: # - gs://bucket/additional.ipa OR path/to/local/ipa/file.ipa - + ### Testing with Special Entitlements ## Enables testing special app entitlements. Re-signs an app having special entitlements with a new application-identifier. ## This currently supports testing Push Notifications (aps-environment) entitlement for up to one app in a project. ## Note: Because this changes the app's identifier, make sure none of the resources in your zip file contain direct references to the test app's bundle id. @@ -191,85 +210,105 @@ gcloud: flank: # -- FlankYml -- + ### Max Test Shards ## test shards - the amount of groups to split the test suite into ## set to -1 to use one shard per test. default: 1 # max-test-shards: 1 + ## Shard Time ## shard time - the amount of time tests within a shard should take ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards ## 2 minutes (120) is recommended. ## default: -1 (unlimited) # shard-time: -1 + ### Number of Test Runs ## test runs - the amount of times to run the tests. ## 1 runs the tests once. 10 runs all the tests 10x # num-test-runs: 1 + ### Smart Flank GCS Paths ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml + ### Smart Flank Disable Upload flag ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. ## Default: false # smart-flank-disable-upload: false + ### Use Average Test Time For New Tests flag ## Enable using average time from previous tests duration when using SmartShard and tests did not run before. ## Default: false # use-average-test-time-for-new-tests: true + ### Default Test Time ## Set default test time used for calculating shards. ## Default: 120.0 # default-test-time: 15 + ### Default Class Test Time ## Set default test time (in seconds) used for calculating shards of parametrized classes when previous tests results are not available. ## Default test time for classes should be different from the default time for test ## Default: 240.0 # default-class-test-time: 30 + ### Disable Sharding flag ## Disables sharding. Useful for parameterized tests. # disable-sharding: false + ### Test targets always Run ## always run - these tests are inserted at the beginning of every shard ## Execution order is not guaranteed by Flank. Users are responsible for configuring their own device test runner logic. # test-targets-always-run: # - className/testName + ### Files to Download ## regex is matched against bucket paths, for example: 2019-01-09_00:18:07.314000_hCMY/shard_0/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun # files-to-download: # - .*\.mp4$ # -- IosFlankYml -- + ### Test Targets ## test targets - a list of tests to run. omit to run all tests. # test-targets: # - className/testName + ### Billing Project Name ## The billing enabled Google Cloud Platform project name to use # project: flank-open-source + ### Local Result Directory Storage ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). # run-timeout: 60m + ### Keep File Path flag ## Keeps the full path of downloaded files. Required when file names are not unique. ## Default: false # keep-file-path: false + ### Ignore Failed Tests flag ## Terminate with exit code 0 when there are failed tests. ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. ## The JUnit XML is used to determine failure. (default: false) # ignore-failed-tests: true + ### Output Style ## Output style of execution status. May be one of [verbose, multi, single]. ## For runs with only one test execution the default value is 'verbose', in other cases ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles ## which don't support ansi codes, to avoid corrupted output use single or verbose. # output-style: single + ### Full Junit Result flag ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. # full-junit-result: false + ### Disable Result Upload flag ## Disables flank results upload on gcloud storage. ## Default: false # disable-results-upload: false @@ -288,25 +327,31 @@ Run `test_runner/flank.yml` with flank to verify Android execution is working. gcloud: # -- GcloudYml -- + ### Result Bucket ## The name of a Google Cloud Storage bucket where raw test results will be stored # results-bucket: tmp_flank + ### Result Directory ## The name of a unique Google Cloud Storage object within the results bucket where raw test results will be stored ## (default: a timestamp with a random suffix). # results-dir: tmp + ### Record Video flag ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true + ### Timeout ## The max time this test execution can run before it is cancelled (default: 15m). ## It does not include any time necessary to prepare and clean up the target device. ## The maximum possible testing time is 45m on physical devices and 60m on virtual devices. ## The TIMEOUT units can be h, m, or s. If no unit is given, seconds are assumed. # timeout: 30m + ### Asynchronous flag ## Invoke a test asynchronously without waiting for test results. # async: false + ### Client Details ## A key-value map of additional details to attach to the test matrix. ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run. ## When consuming the test results, such as in Cloud Functions or a CI system, @@ -315,44 +360,53 @@ gcloud: # key1: value1 # key2: value2 + ### Network Profile ## The name of the network traffic profile, for example LTE, HSPA, etc, ## which consists of a set of parameters to emulate network conditions when running the test ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ## This feature only works on physical devices. # network-profile: LTE + ### Result History Name ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history + ### Number of Flaky Test Attempts ## The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. ## The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. # num-flaky-test-attempts: 0 # -- AndroidGcloudYml -- + ## Android Application Path ## The path to the application binary file. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## Android App Bundles are specified as .aab, all other files are assumed to be APKs. app: ../test_projects/android/apks/app-debug.apk + ### Android Binary File Path ## The path to the binary file containing instrumentation tests. ## The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. test: ../test_projects/android/apks/app-debug-androidTest.apk + ### Additional APK's ## A list of up to 100 additional APKs to install, in addition to those being directly tested. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. # additional-apks: additional-apk1.apk,additional-apk2.apk,additional-apk3.apk + ### Auto Google Login flag ## Automatically log into the test device using a preconfigured Google account before beginning the test. ## Disabled by default. Use --auto-google-login to enable. # auto-google-login: true + ### Use Orchestrator Flag ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator ## (default: Orchestrator is used). Disable with --no-use-orchestrator. ## See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator # use-orchestrator: true + ### Environment Variables ## A comma-separated, key=value map of environment variables and their desired values. This flag is repeatable. ## The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and ## passed to your test runner (typically AndroidJUnitRunner) @@ -361,18 +415,22 @@ gcloud: # coverageFilePath: /sdcard/ # clearPackageData: true + ### Directories to Pull ## A list of paths that will be copied from the device's storage to the designated results bucket after the test ## is complete. These must be absolute paths under /sdcard or /data/local/tmp # directories-to-pull: # - /sdcard/ + ### Grant Permissions flag ## Whether to grant runtime permissions on the device before the test begins. ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all + ### Test Type ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. # type: instrumentation + ### Other Files ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). @@ -380,12 +438,14 @@ gcloud: # - /sdcard/dir1/file1.txt: local/file.txt # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + ### OBB Files ## A list of one or two Android OBB file names which will be copied to each test device before the tests will run (default: None). ## Each OBB file name must conform to the format as specified by Android (e.g. [main|patch].0300110.com.example.android.obb) and will be installed into /Android/obb// on the test device. # obb-files: # - local/file/path/test1.obb # - local/file/path/test2.obb + ### Scenario Numbers ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. # scenario-numbers: @@ -393,16 +453,26 @@ gcloud: # - 2 # - 3 + ### Scenario Labels + ## A list of game-loop scenario labels (default: None). Each game-loop scenario may be labeled in the APK manifest file with one or more arbitrary strings, creating logical groupings (e.g. GPU_COMPATIBILITY_TESTS). + ## If --scenario-numbers and --scenario-labels are specified together, Firebase Test Lab will first execute each scenario from --scenario-numbers. + ## It will then expand each given scenario label into a list of scenario numbers marked with that label, and execute those scenarios. + # scenario-labels: + # - label1 + # - label2 + + ### OBB filenames ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. ## [main|patch].0300110.com.example.android.obb which will be installed into /Android/obb// on the device. # obb-names: # - [main|patch]..com.example.android.obb - + ### Performance Metric flag ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Disabled by default. Use --performance-metrics to enable. # performance-metrics: true + ### Number of Uniform Shards ## Specifies the number of shards into which you want to evenly distribute test cases. ## The shards are run in parallel on separate devices. For example, ## if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases. @@ -413,10 +483,12 @@ gcloud: ## default: null # num-uniform-shards: 50 + ### Instrumentation Test Runner Class ## The fully-qualified Java class name of the instrumentation test runner ## (default: the last name extracted from the APK manifest). # test-runner-class: com.foo.TestRunner + ### Test Targets ## A list of one or more test target filters to apply (default: run all test targets). ## Each target filter must be fully qualified with the package name, class name, or test annotation desired. ## Supported test filters by am instrument -e … include: @@ -425,6 +497,7 @@ gcloud: # test-targets: # - class com.example.app.ExampleUiTest#testPasses + ### Robo Directives ## A map of robo_directives that you can use to customize the behavior of Robo test. ## The type specifies the action type of the directive, which may take on values click, text or ignore. ## If no type is provided, text will be used by default. @@ -434,12 +507,14 @@ gcloud: # "text:input_resource_name": message # "click:button_resource_name": "" + ### Robo Scripts ## The path to a Robo Script JSON file. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument. ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. # robo-script: path_to_robo_script + ### Android Device Parameters ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. @@ -454,62 +529,77 @@ gcloud: flank: # -- FlankYml -- + ### Max Test Shards ## test shards - the amount of groups to split the test suite into ## set to -1 to use one shard per test. default: 1 # max-test-shards: 1 + ### Shard Time ## shard time - the amount of time tests within a shard should take ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards ## 2 minutes (120) is recommended. ## default: -1 (unlimited) # shard-time: -1 + ### Number of Test Runs ## test runs - the amount of times to run the tests. ## 1 runs the tests once. 10 runs all the tests 10x # num-test-runs: 1 + ### Smart Flank GCS Path ## Google cloud storage path where the JUnit XML results from the last run is stored. # smart-flank-gcs-path: gs://tmp_flank/tmp/JUnitReport.xml + ### Smart Flank Upload Disable flag ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. ## Default: false # smart-flank-disable-upload: false + ### Use Average Test Time for New Tests flag ## Enable using average time from previous tests duration when using SmartShard and tests did not run before. ## Default: false # use-average-test-time-for-new-tests: true + ### Default Test Time ## Set default test time used for calculating shards. ## Default: 120.0 # default-test-time: 15 + ### Default Class Test Time ## Set default test time (in seconds) used for calculating shards of parametrized classes when previous tests results are not available. ## Default test time for classes should be different from the default time for test ## Default: 240.0 # default-class-test-time: 30 + ### Disable Sharding flag ## Disables sharding. Useful for parameterized tests. # disable-sharding: false + ### Test Targets Always Run ## always run - these tests are inserted at the beginning of every shard ## Execution order is not guaranteed by Flank. Users are responsible for configuring their own device test runner logic. # test-targets-always-run: # - class com.example.app.ExampleUiTest#testPasses + ### Files to Download ## regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt # files-to-download: # - .*\.mp4$ + ### Billing Project Name ## The billing enabled Google Cloud Platform project name to use # project: flank-open-source + ### Local Results Directory ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + ### Keep File Path flag ## Keeps the full path of downloaded files. Required when file names are not unique. ## Default: false # keep-file-path: false + ### Additional App/Test APKS ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path! ## If app is omitted, then the top level app is used for that pair. # additional-app-test-apks: @@ -517,29 +607,35 @@ flank: # test: ../test_projects/android/apks/app1-debug-androidTest.apk # - test: ../test_projects/android/apks/app2-debug-androidTest.apk + ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). # run-timeout: 60m + ### Ignore Failed Test flag ## Terminate with exit code 0 when there are failed tests. ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. ## The JUnit XML is used to determine failure. (default: false) # ignore-failed-tests: true + ### Legacy Junit Results flag ## Flank provides two ways for parsing junit xml results. ## New way uses google api instead of merging xml files, but can generate slightly different output format. ## This flag allows fallback for legacy xml junit results parsing ## Currently available for android, iOS still uses only legacy way. # legacy-junit-result: false + ### Output Style flag ## Output style of execution status. May be one of [verbose, multi, single]. ## For runs with only one test execution the default value is 'verbose', in other cases ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles ## which don't support ansi codes, to avoid corrupted output use single or verbose. # output-style: single + ### Full Junit Result flag ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. # full-junit-result: false + ### Disable Results Upload flag ## Disables flank results upload on gcloud storage. ## Default: false # disable-results-upload: false diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 1da5dd0cab..d333824d2c 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -3,25 +3,31 @@ gcloud: # -- GcloudYml -- + ### Results Bucket ## The name of a Google Cloud Storage bucket where raw test results will be stored # results-bucket: tmp_flank + ### Results Directory ## The name of a unique Google Cloud Storage object within the results bucket where raw test results will be stored ## (default: a timestamp with a random suffix). # results-dir: tmp + ### Record Video flag ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true + ### Timeout ## The max time this test execution can run before it is cancelled (default: 15m). ## It does not include any time necessary to prepare and clean up the target device. ## The maximum possible testing time is 45m on physical devices and 60m on virtual devices. ## The TIMEOUT units can be h, m, or s. If no unit is given, seconds are assumed. # timeout: 30m + ### Asynchronous flag ## Invoke a test asynchronously without waiting for test results. # async: false + ### Client Details ## A key-value map of additional details to attach to the test matrix. ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run. ## When consuming the test results, such as in Cloud Functions or a CI system, @@ -30,28 +36,32 @@ gcloud: # key1: value1 # key2: value2 + ### Network Profile ## The name of the network traffic profile, for example LTE, HSPA, etc, ## which consists of a set of parameters to emulate network conditions when running the test ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ## This feature only works on physical devices. # network-profile: LTE + ### Result History Name ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history - ## Experimental! + ### Number of Flaky Test Attempts ## The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. ## The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. # num-flaky-test-attempts: 0 # -- IosGcloudYml -- + ### IOS Test Package Path ## The path to the test package (a zip file containing the iOS app and XCTest files). ## The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. ## Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. test: ./src/test/kotlin/ftl/fixtures/tmp/earlgrey_example.zip + ### IOS XCTestrun File Path ## The path to an .xctestrun file that will override any .xctestrun file contained in the --test package. ## Because the .xctestrun file contains environment variables along with test methods to run and/or ignore, ## this can be useful for customizing or sharding test suites. The given path should be in the local filesystem. @@ -59,11 +69,13 @@ gcloud: ## For example ./derivedDataPath/Build/Products/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun + ### Xcode Version ## The version of Xcode that should be used to run an XCTest. ## Defaults to the latest Xcode version supported in Firebase Test Lab. ## This Xcode version must be supported by all iOS versions selected in the test matrix. # xcode-version: 10.1 + ### IOS Device Parameters ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. @@ -77,12 +89,14 @@ gcloud: # locale: es_ES # orientation: landscape + ### Directories to Pull ## A list of paths that will be copied from the device's storage to the designated results bucket after the test ## is complete. These must be absolute paths under /private/var/mobile/Media or /Documents ## of the app under test. If the path is under an app's /Documents, it must be prefixed with the app's bundle id and a colon # directories-to-pull: # - /private/var/mobile/Media + ### Other File paths ## A list of device-path=file-path pairs that specify the paths of the test device and the files you want pushed to the device prior to testing. ## Device paths should either be under the Media shared folder (e.g. prefixed with /private/var/mobile/Media) or ## within the documents directory of the filesystem of an app under test (e.g. /Documents). Device paths to app @@ -91,12 +105,14 @@ gcloud: # - com.my.app:/Documents/file.txt: local/file.txt # - /private/var/mobile/Media/file.jpg: gs://bucket/file.jpg + ### Additional IPA's ## List of up to 100 additional IPAs to install, in addition to the one being directly tested. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. # additional-ipas: # - gs://bucket/additional.ipa # - path/to/local/ipa/file.ipa + ### Scenario Numbers ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. # scenario-numbers: @@ -104,15 +120,18 @@ gcloud: # - 2 # - 3 + ### Test type + ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest + # type: xctest + + ### Application Path ## The path to the application archive (.ipa file) for game-loop testing. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## This flag is only valid when --type=game-loop is also set # app: # - gs://bucket/additional.ipa OR path/to/local/ipa/file.ipa - ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest - # type: xctest - + ### Testing with Special Entitlements ## Enables testing special app entitlements. Re-signs an app having special entitlements with a new application-identifier. ## This currently supports testing Push Notifications (aps-environment) entitlement for up to one app in a project. ## Note: Because this changes the app's identifier, make sure none of the resources in your zip file contain direct references to the test app's bundle id. @@ -121,79 +140,105 @@ gcloud: flank: # -- FlankYml -- + ### Max Test Shards ## test shards - the amount of groups to split the test suite into ## set to -1 to use one shard per test. default: 1 # max-test-shards: 1 + ## Shard Time ## shard time - the amount of time tests within a shard should take ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards ## 2 minutes (120) is recommended. ## default: -1 (unlimited) # shard-time: -1 + ### Number of Test Runs ## test runs - the amount of times to run the tests. ## 1 runs the tests once. 10 runs all the tests 10x # num-test-runs: 1 + ### Smart Flank GCS Paths ## Google cloud storage path to store the JUnit XML results from the last run. # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_ios.xml + ### Smart Flank Disable Upload flag ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. ## Default: false # smart-flank-disable-upload: false + ### Use Average Test Time For New Tests flag ## Enable using average time from previous tests duration when using SmartShard and tests did not run before. ## Default: false # use-average-test-time-for-new-tests: true + ### Default Test Time ## Set default test time used for calculating shards. ## Default: 120.0 # default-test-time: 15 + ### Default Class Test Time ## Set default test time (in seconds) used for calculating shards of parametrized classes when previous tests results are not available. ## Default test time for classes should be different from the default time for test ## Default: 240.0 # default-class-test-time: 30 + ### Disable Sharding flag ## Disables sharding. Useful for parameterized tests. # disable-sharding: false + ### Test targets always Run ## always run - these tests are inserted at the beginning of every shard - ## useful if you need to grant permissions or login before other tests run + ## Execution order is not guaranteed by Flank. Users are responsible for configuring their own device test runner logic. # test-targets-always-run: # - className/testName + ### Files to Download ## regex is matched against bucket paths, for example: 2019-01-09_00:18:07.314000_hCMY/shard_0/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun # files-to-download: # - .*\.mp4$ # -- IosFlankYml -- + ### Test Targets ## test targets - a list of tests to run. omit to run all tests. # test-targets: # - className/testName + ### Billing Project Name ## The billing enabled Google Cloud Platform project name to use # project: flank-open-source + ### Local Result Directory Storage ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). # run-timeout: 60m + ### Keep File Path flag ## Keeps the full path of downloaded files. Required when file names are not unique. ## Default: false # keep-file-path: false + ### Ignore Failed Tests flag + ## Terminate with exit code 0 when there are failed tests. + ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. + ## The JUnit XML is used to determine failure. (default: false) + # ignore-failed-tests: true + + ### Output Style ## Output style of execution status. May be one of [verbose, multi, single]. ## For runs with only one test execution the default value is 'verbose', in other cases ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles ## which don't support ansi codes, to avoid corrupted output use single or verbose. # output-style: single + ### Full Junit Result flag ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. # full-junit-result: false + ### Disable Result Upload flag ## Disables flank results upload on gcloud storage. + ## Default: false # disable-results-upload: false diff --git a/test_runner/flank.yml b/test_runner/flank.yml index da0ba71589..eaaf18a065 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -3,25 +3,31 @@ gcloud: # -- GcloudYml -- + ### Result Bucket ## The name of a Google Cloud Storage bucket where raw test results will be stored # results-bucket: tmp_flank + ### Result Directory ## The name of a unique Google Cloud Storage object within the results bucket where raw test results will be stored ## (default: a timestamp with a random suffix). # results-dir: tmp + ### Record Video flag ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true + ### Timeout ## The max time this test execution can run before it is cancelled (default: 15m). ## It does not include any time necessary to prepare and clean up the target device. ## The maximum possible testing time is 45m on physical devices and 60m on virtual devices. ## The TIMEOUT units can be h, m, or s. If no unit is given, seconds are assumed. # timeout: 30m + ### Asynchronous flag ## Invoke a test asynchronously without waiting for test results. # async: false + ### Client Details ## A key-value map of additional details to attach to the test matrix. ## Arbitrary key-value pairs may be attached to a test matrix to provide additional context about the tests being run. ## When consuming the test results, such as in Cloud Functions or a CI system, @@ -30,45 +36,53 @@ gcloud: # key1: value1 # key2: value2 + ### Network Profile ## The name of the network traffic profile, for example LTE, HSPA, etc, ## which consists of a set of parameters to emulate network conditions when running the test ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ## This feature only works on physical devices. # network-profile: LTE + ### Result History Name ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history - ## Experimental! + ### Number of Flaky Test Attempts ## The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. ## The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. # num-flaky-test-attempts: 0 # -- AndroidGcloudYml -- + ## Android Application Path ## The path to the application binary file. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## Android App Bundles are specified as .aab, all other files are assumed to be APKs. app: ../test_projects/android/apks/app-debug.apk + ### Android Binary File Path ## The path to the binary file containing instrumentation tests. ## The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. test: ../test_projects/android/apks/app-debug-androidTest.apk + ### Additional APK's ## A list of up to 100 additional APKs to install, in addition to those being directly tested. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. # additional-apks: additional-apk1.apk,additional-apk2.apk,additional-apk3.apk + ### Auto Google Login flag ## Automatically log into the test device using a preconfigured Google account before beginning the test. ## Disabled by default. Use --auto-google-login to enable. # auto-google-login: true + ### Use Orchestrator Flag ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator ## (default: Orchestrator is used). Disable with --no-use-orchestrator. ## See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator # use-orchestrator: true + ### Environment Variables ## A comma-separated, key=value map of environment variables and their desired values. This flag is repeatable. ## The environment variables are mirrored as extra options to the am instrument -e KEY1 VALUE1 … command and ## passed to your test runner (typically AndroidJUnitRunner) @@ -77,18 +91,22 @@ gcloud: # coverageFilePath: /sdcard/ # clearPackageData: true + ### Directories to Pull ## A list of paths that will be copied from the device's storage to the designated results bucket after the test ## is complete. These must be absolute paths under /sdcard or /data/local/tmp # directories-to-pull: # - /sdcard/ - ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. - # type: instrumentation - + ### Grant Permissions flag ## Whether to grant runtime permissions on the device before the test begins. ## By default, all permissions are granted. PERMISSIONS must be one of: all, none # grant-permissions: all + ### Test Type + ## The type of test to run. TYPE must be one of: instrumentation, robo, game-loop. + # type: instrumentation + + ### Other Files ## A list of device-path: file-path pairs that indicate the device paths to push files to the device before starting tests, and the paths of files to push. ## Device paths must be under absolute, whitelisted paths (${EXTERNAL_STORAGE}, or ${ANDROID_DATA}/local/tmp). ## Source file paths may be in the local filesystem or in Google Cloud Storage (gs://…). @@ -96,6 +114,14 @@ gcloud: # - /sdcard/dir1/file1.txt: local/file.txt # - /sdcard/dir2/file2.jpg: gs://bucket/file.jpg + ### OBB Files + ## A list of one or two Android OBB file names which will be copied to each test device before the tests will run (default: None). + ## Each OBB file name must conform to the format as specified by Android (e.g. [main|patch].0300110.com.example.android.obb) and will be installed into /Android/obb// on the test device. + # obb-files: + # - local/file/path/test1.obb + # - local/file/path/test2.obb + + ### Scenario Numbers ## A list of game-loop scenario numbers which will be run as part of the test (default: all scenarios). ## A maximum of 1024 scenarios may be specified in one test matrix, but the maximum number may also be limited by the overall test --timeout setting. # scenario-numbers: @@ -103,6 +129,7 @@ gcloud: # - 2 # - 3 + ### Scenario Labels ## A list of game-loop scenario labels (default: None). Each game-loop scenario may be labeled in the APK manifest file with one or more arbitrary strings, creating logical groupings (e.g. GPU_COMPATIBILITY_TESTS). ## If --scenario-numbers and --scenario-labels are specified together, Firebase Test Lab will first execute each scenario from --scenario-numbers. ## It will then expand each given scenario label into a list of scenario numbers marked with that label, and execute those scenarios. @@ -110,21 +137,18 @@ gcloud: # - label1 # - label2 - ## A list of one or two Android OBB file names which will be copied to each test device before the tests will run (default: None). - ## Each OBB file name must conform to the format as specified by Android (e.g. [main|patch].0300110.com.example.android.obb) and will be installed into /Android/obb// on the test device. - # obb-files: - # - local/file/path/test1.obb - # - local/file/path/test2.obb - + ### OBB filenames ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. ## [main|patch].0300110.com.example.android.obb which will be installed into /Android/obb// on the device. # obb-names: # - [main|patch]..com.example.android.obb + ### Performance Metric flag ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only). ## Disabled by default. Use --performance-metrics to enable. # performance-metrics: true + ### Number of Uniform Shards ## Specifies the number of shards into which you want to evenly distribute test cases. ## The shards are run in parallel on separate devices. For example, ## if your test execution contains 20 test cases and you specify four shards, each shard executes five test cases. @@ -135,10 +159,12 @@ gcloud: ## default: null # num-uniform-shards: 50 + ### Instrumentation Test Runner Class ## The fully-qualified Java class name of the instrumentation test runner ## (default: the last name extracted from the APK manifest). # test-runner-class: com.foo.TestRunner + ### Test Targets ## A list of one or more test target filters to apply (default: run all test targets). ## Each target filter must be fully qualified with the package name, class name, or test annotation desired. ## Supported test filters by am instrument -e … include: @@ -147,6 +173,7 @@ gcloud: # test-targets: # - class com.example.app.ExampleUiTest#testPasses + ### Robo Directives ## A map of robo_directives that you can use to customize the behavior of Robo test. ## The type specifies the action type of the directive, which may take on values click, text or ignore. ## If no type is provided, text will be used by default. @@ -156,12 +183,14 @@ gcloud: # "text:input_resource_name": message # "click:button_resource_name": "" + ### Robo Scripts ## The path to a Robo Script JSON file. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## You can guide the Robo test to perform specific actions by recording a Robo Script in Android Studio and then specifying this argument. ## Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. # robo-script: path_to_robo_script + ### Android Device Parameters ## A list of DIMENSION=VALUE pairs which specify a target device to test against. ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. @@ -176,62 +205,77 @@ gcloud: flank: # -- FlankYml -- + ### Max Test Shards ## test shards - the amount of groups to split the test suite into ## set to -1 to use one shard per test. default: 1 # max-test-shards: 1 + ### Shard Time ## shard time - the amount of time tests within a shard should take ## when set to > 0, the shard count is dynamically set based on time up to the maximum limit defined by max-test-shards ## 2 minutes (120) is recommended. ## default: -1 (unlimited) # shard-time: -1 + ### Number of Test Runs ## test runs - the amount of times to run the tests. ## 1 runs the tests once. 10 runs all the tests 10x # num-test-runs: 1 - ## Google cloud storage path to store the JUnit XML results from the last run. - # smart-flank-gcs-path: gs://tmp_flank/flank/test_app_android.xml + ### Smart Flank GCS Path + ## Google cloud storage path where the JUnit XML results from the last run is stored. + # smart-flank-gcs-path: gs://tmp_flank/tmp/JUnitReport.xml + ### Smart Flank Upload Disable flag ## Disables smart flank JUnit XML uploading. Useful for preventing timing data from being updated. ## Default: false # smart-flank-disable-upload: false + ### Use Average Test Time for New Tests flag ## Enable using average time from previous tests duration when using SmartShard and tests did not run before. ## Default: false # use-average-test-time-for-new-tests: true + ### Default Test Time ## Set default test time used for calculating shards. ## Default: 120.0 # default-test-time: 15 + ### Default Class Test Time ## Set default test time (in seconds) used for calculating shards of parametrized classes when previous tests results are not available. ## Default test time for classes should be different from the default time for test ## Default: 240.0 # default-class-test-time: 30 + ### Disable Sharding flag ## Disables sharding. Useful for parameterized tests. # disable-sharding: false + ### Test Targets Always Run ## always run - these tests are inserted at the beginning of every shard - ## useful if you need to grant permissions or login before other tests run + ## Execution order is not guaranteed by Flank. Users are responsible for configuring their own device test runner logic. # test-targets-always-run: # - class com.example.app.ExampleUiTest#testPasses + ### Files to Download ## regex is matched against bucket paths, for example: 2019-01-09_00:13:06.106000_YCKl/shard_0/NexusLowRes-28-en-portrait/bugreport.txt # files-to-download: # - .*\.mp4$ + ### Billing Project Name ## The billing enabled Google Cloud Platform project name to use # project: flank-open-source + ### Local Results Directory ## Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. # local-result-dir: flank + ### Keep File Path flag ## Keeps the full path of downloaded files. Required when file names are not unique. ## Default: false # keep-file-path: false + ### Additional App/Test APKS ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path! ## If app is omitted, then the top level app is used for that pair. # additional-app-test-apks: @@ -239,28 +283,35 @@ flank: # test: ../test_projects/android/apks/app1-debug-androidTest.apk # - test: ../test_projects/android/apks/app2-debug-androidTest.apk + ### Run Timeout ## The max time this test run can execute before it is cancelled (default: unlimited). # run-timeout: 60m + ### Ignore Failed Test flag ## Terminate with exit code 0 when there are failed tests. ## Useful for Fladle and other gradle plugins that don't expect the process to have a non-zero exit code. ## The JUnit XML is used to determine failure. (default: false) # ignore-failed-tests: true + ### Legacy Junit Results flag ## Flank provides two ways for parsing junit xml results. ## New way uses google api instead of merging xml files, but can generate slightly different output format. ## This flag allows fallback for legacy xml junit results parsing ## Currently available for android, iOS still uses only legacy way. # legacy-junit-result: false + ### Output Style flag ## Output style of execution status. May be one of [verbose, multi, single]. ## For runs with only one test execution the default value is 'verbose', in other cases ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles ## which don't support ansi codes, to avoid corrupted output use single or verbose. # output-style: single + ### Full Junit Result flag ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. # full-junit-result: false + ### Disable Results Upload flag ## Disables flank results upload on gcloud storage. + ## Default: false # disable-results-upload: false