From 38e09ede9061f5daa815f7407c1d69f9ad784df8 Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Wed, 3 Feb 2021 17:35:55 +0100 Subject: [PATCH] fix: Move shards json to result directory (#1557) Fixes #1475 ## Test Plan > How do we know the code works? 1. Run flank with android or ios test 1. Wait until ```results-directory``` is created 1. Open the results directory and check ```*_shards.json``` exists 1. If ```disable-results-upload``` not set, check shards file exists in gcloud bucket 1. Flank shouldn't upload shards file to gcs if ```disable-results-upload``` set ## Checklist - [X] Documented [(SDD in ticket comment)](https://github.com/Flank/flank/issues/1475#issuecomment-769227112) - [x] Unit tests updated - [X] Integration tests updated --- .../compare/AllTestFilteredIT-android-compare | 2 +- .../test/resources/compare/IgnoreFailedIT-compare | 2 +- .../test/resources/compare/MultipleApksIT-compare | 2 +- .../compare/MultipleDevicesIT-android-compare | 2 +- .../src/test/resources/compare/RunTimeoutIT-compare | 2 +- .../test/resources/compare/TestFilteringIT-compare | 2 +- test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt | 5 +++++ test_runner/src/main/kotlin/ftl/args/IArgs.kt | 4 ++++ test_runner/src/main/kotlin/ftl/args/IosArgs.kt | 5 +++++ test_runner/src/main/kotlin/ftl/run/DumpShards.kt | 4 ++++ .../main/kotlin/ftl/run/platform/RunAndroidTests.kt | 12 ++++++------ .../src/main/kotlin/ftl/run/platform/RunIosTests.kt | 8 +++++--- .../src/test/kotlin/ftl/run/DumpShardsKtTest.kt | 2 +- 13 files changed, 36 insertions(+), 16 deletions(-) diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index ce5d216d95..fd02f9cdf2 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -66,7 +66,7 @@ AndroidArgs RunTests No tests for app-single-success-debug-androidTest.apk - Saved 0 shards to android_shards.json + Saved 0 shards to .*\/android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* There are no tests to run. diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index 63c5e572a3..9caf681be2 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 disable-usage-statistics: false RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-single-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 disable-usage-statistics: false RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to .*\/android_shards.json Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-single-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 745ebb7626..1e11f6b424 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to .*\/android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index a5f7fd7024..8145053fb8 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to .*\/android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index c159d3d21d..0d2e73dbad 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -63,7 +63,7 @@ AndroidArgs disable-usage-statistics: false RunTests - Saved 1 shards to android_shards.json + Saved 1 shards to .*\/android_shards.json 1 test \/ 1 shard diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index 436656ea7a..16bce33c20 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -68,7 +68,7 @@ AndroidArgs RunTests No tests for app-single-success-debug-androidTest.apk \s* - Saved 1 shards to android_shards.json + Saved 1 shards to .*\/android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-multiple-success-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 7ed5dc1127..8165c58ceb 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -3,6 +3,8 @@ package ftl.args import ftl.analytics.AnonymizeInStatistics import ftl.args.yml.AppTestPair import ftl.args.yml.Type +import ftl.run.ANDROID_SHARD_FILE +import java.nio.file.Paths data class AndroidArgs( val commonArgs: CommonArgs, @@ -151,5 +153,8 @@ val AndroidArgs.isSanityRobo val AndroidArgs.isGameLoop get() = if (type == null) false else (type == Type.GAMELOOP) +val AndroidArgs.shardsFilePath + get() = Paths.get(localStorageDirectory, ANDROID_SHARD_FILE).toAbsolutePath().toString() + private fun String?.isNull() = this == null private fun String?.isNotNull() = isNull().not() diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 36757967fa..f6d4e59218 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -10,6 +10,7 @@ import ftl.config.Device import ftl.config.common.CommonFlankConfig.Companion.defaultLocalResultsDir import ftl.run.status.OutputStyle import ftl.util.timeoutToMils +import java.nio.file.Paths // Properties common to both Android and iOS interface IArgs { @@ -117,3 +118,6 @@ fun IArgs.setupLogLevel() { val IArgs.blockSendingUsageStatistics get() = disableUsageStatistics || isTest() + +val IArgs.localStorageDirectory + get() = if (useLocalResultDir()) localResultDir else Paths.get(localResultDir, resultsDir).toString() diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 517780135b..c88bddf3be 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -7,7 +7,9 @@ import ftl.args.yml.Type import ftl.ios.xctest.XcTestRunData import ftl.ios.xctest.calculateXcTestRunData import ftl.ios.xctest.common.XctestrunMethods +import ftl.run.IOS_SHARD_FILE import ftl.run.exception.FlankConfigurationError +import java.nio.file.Paths data class IosArgs( val commonArgs: CommonArgs, @@ -103,6 +105,9 @@ IosArgs val IosArgs.isXcTest: Boolean get() = type == Type.XCTEST +val IosArgs.shardsFilePath + get() = Paths.get(localStorageDirectory, IOS_SHARD_FILE).toAbsolutePath().toString() + @VisibleForTesting internal fun filterTests( validTestMethods: XctestrunMethods, diff --git a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt index dfcceb8cae..b1813140cf 100644 --- a/test_runner/src/main/kotlin/ftl/run/DumpShards.kt +++ b/test_runner/src/main/kotlin/ftl/run/DumpShards.kt @@ -2,6 +2,7 @@ package ftl.run import flank.common.OutputLogLevel import flank.common.logLn +import flank.common.toFile import ftl.args.AndroidArgs import ftl.args.IosArgs import ftl.args.isInstrumentationTest @@ -13,6 +14,7 @@ import ftl.run.exception.FlankConfigurationError import ftl.run.model.AndroidMatrixTestShards import ftl.run.platform.android.getAndroidMatrixShards import ftl.util.obfuscatePrettyPrinter +import java.io.File import java.nio.file.Files import java.nio.file.Paths @@ -62,12 +64,14 @@ fun saveShardChunks( size: Int, obfuscatedOutput: Boolean ) { + shardFilePath.createDirectories() Files.write( Paths.get(shardFilePath), getGson(obfuscatedOutput).toJson(shards).toByteArray() ) logLn("${FtlConstants.indent}Saved $size shards to $shardFilePath", OutputLogLevel.DETAILED) } +private fun String.createDirectories() = File(this).parent.toFile().mkdirs() private fun getGson(obfuscatedOutput: Boolean) = if (obfuscatedOutput) obfuscatePrettyPrinter else prettyPrint diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index ec8c517a49..dd01d4e5b3 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -5,12 +5,12 @@ import com.google.testing.model.TestMatrix import flank.common.logLn import ftl.args.AndroidArgs import ftl.args.isInstrumentationTest +import ftl.args.shardsFilePath import ftl.gc.GcAndroidDevice import ftl.gc.GcAndroidTestMatrix import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry -import ftl.run.ANDROID_SHARD_FILE import ftl.run.exception.FlankGeneralError import ftl.run.model.AndroidMatrixTestShards import ftl.run.model.AndroidTestContext @@ -88,15 +88,15 @@ private fun List.dumpShards(config: AndroidArgs) = takeIf { if (config.testTargetsForShard.isEmpty()) filterIsInstance() .asMatrixTestShards() - .saveShards(config.obfuscateDumpShards) - if (config.disableResultsUpload.not()) GcStorage.upload(ANDROID_SHARD_FILE, config.resultsBucket, config.resultsDir) + .saveShards(config) + if (config.disableResultsUpload.not()) GcStorage.upload(config.shardsFilePath, config.resultsBucket, config.resultsDir) } ?: this -private fun AndroidMatrixTestShards.saveShards(obfuscateOutput: Boolean) = saveShardChunks( - shardFilePath = ANDROID_SHARD_FILE, +private fun AndroidMatrixTestShards.saveShards(config: AndroidArgs) = saveShardChunks( + shardFilePath = config.shardsFilePath, shards = this, size = size, - obfuscatedOutput = obfuscateOutput + obfuscatedOutput = config.obfuscateDumpShards ) private suspend fun executeAndroidTestMatrix( diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index 95b12c456b..ec2404cbb5 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -3,13 +3,13 @@ package ftl.run.platform import flank.common.logLn import ftl.args.IosArgs import ftl.args.isXcTest +import ftl.args.shardsFilePath import ftl.gc.GcIosMatrix import ftl.gc.GcIosTestMatrix import ftl.gc.GcStorage import ftl.gc.GcToolResults import ftl.http.executeWithRetry import ftl.ios.xctest.flattenShardChunks -import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.model.TestResult import ftl.run.platform.android.uploadAdditionalIpas @@ -70,7 +70,9 @@ internal suspend fun IosArgs.runIosTests(): TestResult = } private fun IosArgs.dumpShardsIfXcTest() = takeIf { isXcTest }?.let { - dumpShards() + dumpShards( + shardFilePath = shardsFilePath + ) if (disableResultsUpload.not()) - GcStorage.upload(IOS_SHARD_FILE, resultsBucket, resultsDir) + GcStorage.upload(shardsFilePath, resultsBucket, resultsDir) } diff --git a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt index 41e62dbb8d..ac35168a77 100644 --- a/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/DumpShardsKtTest.kt @@ -219,4 +219,4 @@ class DumpShardsKtTest { } } -private const val TEST_SHARD_FILE = "test_dump_shard_file.json" +private const val TEST_SHARD_FILE = "./test_dump_shard_file.json"