From 9bf7b237d2787c82df8883f86106e67e03c1b8e4 Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Mon, 11 Jan 2021 15:34:58 +0100 Subject: [PATCH] feat: Migrate from bugsnag to sentry (#1471) Fixes #1318 ## Test Plan > How do we know the code works? 1. Flank should display session id below ```revision``` 2. Should save session id to ```session_id.txt``` in matrix path 3. Should upload ```session_id.txt``` when ```disable-results-upload``` not set 4. Unit & Integration tests should pass ## Checklist - [X] Save and upload session id - [X] Unit tested - [X] Integration tests updated --- buildSrc/src/main/kotlin/Dependencies.kt | 3 +- buildSrc/src/main/kotlin/Versions.kt | 4 +- .../compare/GameloopIT-android-compare | 1 + .../resources/compare/GameloopIT-ios-compare | 1 + .../resources/compare/MultipleApksIT-compare | 2 +- .../compare/MultipleDevicesIT-android-compare | 2 +- .../resources/compare/SanityRoboIT-compare | 1 + .../resources/compare/TestFilteringIT-compare | 1 + test_runner/build.gradle.kts | 2 +- test_runner/src/main/kotlin/ftl/Main.kt | 2 + .../test/android/AndroidRunCommand.kt | 7 +++ .../cli/firebase/test/ios/IosRunCommand.kt | 7 +++ .../main/kotlin/ftl/config/FtlConstants.kt | 7 --- .../src/main/kotlin/ftl/gc/GcStorage.kt | 9 +++ .../main/kotlin/ftl/http/ExecuteWithRetry.kt | 4 +- .../src/main/kotlin/ftl/log/LogbackLogger.kt | 4 +- .../src/main/kotlin/ftl/log/Loggers.kt | 2 +- .../src/main/kotlin/ftl/run/NewTestRun.kt | 7 ++- .../kotlin/ftl/run/common/SaveSessionId.kt | 9 +++ .../ftl/run/exception/ExceptionHandler.kt | 6 +- .../ftl/run/platform/RunAndroidTests.kt | 1 - .../ftl/run/platform/common/AfterRunTests.kt | 5 +- .../main/kotlin/ftl/util/BugsnagInitHelper.kt | 24 -------- .../src/main/kotlin/ftl/util/CrashReporter.kt | 60 +++++++++++++++++++ test_runner/src/main/kotlin/ftl/util/Utils.kt | 4 +- test_runner/src/test/kotlin/Debug.kt | 4 +- ...erTest.kt => FlankSentryInitHelperTest.kt} | 12 ++-- .../src/test/kotlin/ftl/util/UtilsTest.kt | 4 +- 28 files changed, 134 insertions(+), 61 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt delete mode 100644 test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt create mode 100644 test_runner/src/main/kotlin/ftl/util/CrashReporter.kt rename test_runner/src/test/kotlin/ftl/util/{FlankBugsnagInitHelperTest.kt => FlankSentryInitHelperTest.kt} (78%) diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index cffd57a9ab..dda33ce66e 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -1,6 +1,5 @@ object Dependencies { - const val BUGSNAG = "com.bugsnag:bugsnag:${Versions.BUGSNAG}" - + const val SENTRY = "io.sentry:sentry:${Versions.SENTRY}" const val DD_PLIST = "com.googlecode.plist:dd-plist:${Versions.DD_PLIST}" const val DEX_TEST_PARSER = "com.linkedin.dextestparser:parser:${Versions.DEX_TEST_PARSER}" diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 32ffac002b..bbf19acfd5 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,6 +1,6 @@ object Versions { - // https://github.com/bugsnag/bugsnag-java/releases - const val BUGSNAG = "3.6.2" + // https://github.com/getsentry/sentry-java/releases + const val SENTRY = "3.1.0" // https://github.com/3breadt/dd-plist/releases const val DD_PLIST = "1.23" diff --git a/integration_tests/src/test/resources/compare/GameloopIT-android-compare b/integration_tests/src/test/resources/compare/GameloopIT-android-compare index 825aa5d923..4e7e2356b1 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-android-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-android-compare @@ -69,6 +69,7 @@ RunTests \s* 0 test \/ 0 shard \s* + Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare index 8d20ecdab1..57f8a56821 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare @@ -56,6 +56,7 @@ RunTests 0 test \/ 0 shard Uploading \[FlankGameLoopExample.ipa\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* + Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 1ec6125f6f..260c4e5ae4 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (?=.*Uploading \[MainActivity_robo_script.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-success-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (?=.*Uploading \[MainActivity_robo_script.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-success-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index 734ef8f0e1..af53f43d76 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 60adf5f096..9186f6f6d3 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -66,6 +66,7 @@ RunTests 0 test \/ 0 shard + Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index e70bd06ab9..3aafeafa3c 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -74,6 +74,7 @@ RunTests 1 test \/ 1 shard + Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index bca2726194..23b15cb663 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -185,7 +185,7 @@ tasks.withType { dependencies { implementation(project(":common")) - implementation(Dependencies.BUGSNAG) + implementation(Dependencies.SENTRY) implementation(Dependencies.DD_PLIST) implementation(Dependencies.DEX_TEST_PARSER) diff --git a/test_runner/src/main/kotlin/ftl/Main.kt b/test_runner/src/main/kotlin/ftl/Main.kt index 6d46b34951..e8ef89d31f 100644 --- a/test_runner/src/main/kotlin/ftl/Main.kt +++ b/test_runner/src/main/kotlin/ftl/Main.kt @@ -14,6 +14,7 @@ import ftl.log.setDebugLogging import ftl.run.exception.withGlobalExceptionHandling import ftl.util.readRevision import ftl.util.readVersion +import ftl.util.sessionId import picocli.CommandLine @CommandLine.Command( @@ -56,6 +57,7 @@ class Main : Runnable { withGlobalExceptionHandling { logLn("version: " + readVersion()) logLn("revision: " + readRevision()) + logLn("session id: $sessionId") logLn() CommandLine(Main()).execute(*args) } diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index 323111fdda..f5d893ba56 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -13,6 +13,9 @@ import ftl.mock.MockServer import ftl.run.ANDROID_SHARD_FILE import ftl.run.dumpShards import ftl.run.newTestRun +import ftl.util.DEVICE_SYSTEM +import ftl.util.TEST_TYPE +import ftl.util.setCrashReportTag import kotlinx.coroutines.runBlocking import picocli.CommandLine import picocli.CommandLine.Command @@ -58,6 +61,10 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { AndroidArgs.load(Paths.get(configPath), cli = this).apply { setupLogLevel() logLn(this) + setCrashReportTag( + DEVICE_SYSTEM to "android", + TEST_TYPE to type?.name.orEmpty() + ) }.validate().run { runBlocking { if (dumpShards) dumpShards() diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index f1f31292a7..0237de6084 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -13,6 +13,9 @@ import ftl.mock.MockServer import ftl.run.IOS_SHARD_FILE import ftl.run.dumpShards import ftl.run.newTestRun +import ftl.util.DEVICE_SYSTEM +import ftl.util.TEST_TYPE +import ftl.util.setCrashReportTag import kotlinx.coroutines.runBlocking import picocli.CommandLine import picocli.CommandLine.Command @@ -58,6 +61,10 @@ class IosRunCommand : CommonRunCommand(), Runnable { IosArgs.load(Paths.get(configPath), cli = this).apply { setupLogLevel() logLn(this) + setCrashReportTag( + DEVICE_SYSTEM to "ios", + TEST_TYPE to type?.name.orEmpty() + ) }.validate().run { if (dumpShards) dumpShards() else runBlocking { newTestRun() } diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index c9263de0f4..4fc00421bf 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -1,6 +1,5 @@ package ftl.config -import com.bugsnag.Bugsnag import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport import com.google.api.client.googleapis.util.Utils import com.google.api.client.http.javanet.NetHttpTransport @@ -10,8 +9,6 @@ import ftl.args.IArgs import ftl.args.IosArgs import ftl.run.exception.FlankConfigurationError import ftl.run.exception.FlankGeneralError -import ftl.util.BugsnagInitHelper.initBugsnag -import ftl.util.readRevision object FtlConstants { var useMock = false @@ -34,10 +31,6 @@ object FtlConstants { val JSON_FACTORY: JsonFactory by lazy { Utils.getDefaultJsonFactory() } - val bugsnag: Bugsnag? by lazy { - initBugsnag(useMock)?.apply { setAppVersion(readRevision()) } - } - val httpTransport: NetHttpTransport by lazy { try { return@lazy GoogleNetHttpTransport.newTrustedTransport() diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index dd83441be3..72b6ff7acf 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -20,6 +20,7 @@ import ftl.json.MatrixMap import ftl.reports.xml.model.JUnitTestResult import ftl.reports.xml.parseAllSuitesXml import ftl.reports.xml.xmlToString +import ftl.run.common.SESSION_ID_FILE import ftl.run.common.getMatrixFilePath import ftl.run.exception.FlankGeneralError import ftl.util.runWithProgress @@ -101,6 +102,14 @@ object GcStorage { ) } + fun IArgs.uploadSessionId() = takeUnless { disableResultsUpload }?.let { + upload( + file = Paths.get(localResultDir, SESSION_ID_FILE).toString(), + rootGcsBucket = resultsBucket, + runGcsPath = resultsDir + ) + } + fun uploadReportResult(testResult: String, args: IArgs, fileName: String) { if (args.resultsBucket.isBlank() || args.resultsDir.isBlank() || args.disableResultsUpload) return upload( diff --git a/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt b/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt index df3a9f816a..198a11824a 100644 --- a/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt +++ b/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt @@ -2,10 +2,10 @@ package ftl.http import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest import com.google.api.client.http.HttpResponseException -import ftl.config.FtlConstants import ftl.run.exception.FailureToken import ftl.run.exception.PermissionDenied import ftl.run.exception.ProjectNotFound +import ftl.util.captureError import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import java.io.IOException @@ -24,7 +24,7 @@ private inline fun withRetry(crossinline block: () -> T): T = runBlocking { } catch (err: IOException) { // We want to send every occurrence of Google API error for statistic purposes // https://github.com/Flank/flank/issues/701 - FtlConstants.bugsnag?.notify(FlankGoogleApiError(err)) + captureError(FlankGoogleApiError(err)) lastError = err if (err is HttpResponseException) { diff --git a/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt b/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt index 57dd00f91a..bd6654b779 100644 --- a/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt +++ b/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt @@ -2,7 +2,7 @@ package ftl.log import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger -import com.bugsnag.Bugsnag +import io.sentry.Sentry import org.slf4j.LoggerFactory.getLogger import kotlin.properties.Delegates @@ -17,5 +17,5 @@ sealed class LogbackLogger(private val logger: Logger) : FlankLogger { } object Root : LogbackLogger(getLogger(Logger.ROOT_LOGGER_NAME)) - object FlankBugsnag : LogbackLogger(getLogger(Bugsnag::class.java)) + object FlankSentry : LogbackLogger(getLogger(Sentry::class.java)) } diff --git a/test_runner/src/main/kotlin/ftl/log/Loggers.kt b/test_runner/src/main/kotlin/ftl/log/Loggers.kt index 7a976a40e0..a5de98bc33 100644 --- a/test_runner/src/main/kotlin/ftl/log/Loggers.kt +++ b/test_runner/src/main/kotlin/ftl/log/Loggers.kt @@ -4,7 +4,7 @@ import com.google.api.client.http.GoogleApiLogger private val LOGGERS = listOf( LogbackLogger.Root, - LogbackLogger.FlankBugsnag, + LogbackLogger.FlankSentry, GoogleApiLogger ) diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt index a66fb1f6fc..9b21c50aee 100644 --- a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt @@ -20,11 +20,16 @@ import kotlinx.coroutines.withTimeoutOrNull suspend fun IArgs.newTestRun() = withTimeoutOrNull(parsedTimeout) { val args: IArgs = this@newTestRun + val (matrixMap, testShardChunks, ignoredTests) = cancelTestsOnTimeout(project) { runTests() } if (!args.async) { - cancelTestsOnTimeout(args.project, matrixMap.map) { pollMatrices(matrixMap.map.keys, args).updateMatrixMap(matrixMap) } + cancelTestsOnTimeout(args.project, matrixMap.map) { + pollMatrices(matrixMap.map.keys, args).updateMatrixMap( + matrixMap + ) + } ReportManager.generate(matrixMap, args, testShardChunks, ignoredTests) cancelTestsOnTimeout(args.project, matrixMap.map) { fetchArtifacts(matrixMap, args) } diff --git a/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt b/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt new file mode 100644 index 0000000000..6de279c709 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt @@ -0,0 +1,9 @@ +package ftl.run.common + +import ftl.args.IArgs +import ftl.util.sessionId +import java.nio.file.Path + +const val SESSION_ID_FILE = "session_id.txt" + +fun IArgs.saveSessionId() = Path.of(localResultDir, SESSION_ID_FILE).toFile().writeText(sessionId) diff --git a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt index 1416a9ceca..aa25df1a95 100644 --- a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt +++ b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt @@ -1,9 +1,9 @@ package ftl.run.exception import flank.common.logLn -import ftl.config.FtlConstants import ftl.json.SavedMatrix import ftl.run.cancelMatrices +import ftl.util.captureError import kotlinx.coroutines.runBlocking import kotlin.system.exitProcess @@ -39,6 +39,7 @@ fun withGlobalExceptionHandling(block: () -> Int) { } is InfrastructureError -> { + captureError(t) printError("An infrastructure error occurred.") printError("Details: ${t.messageOrUnavailable}") exitProcess(INFRASTRUCTURE_ERROR) @@ -50,6 +51,7 @@ fun withGlobalExceptionHandling(block: () -> Int) { } is FTLError -> { + captureError(t) t.matrix.logError() exitProcess(UNEXPECTED_ERROR) } @@ -64,7 +66,7 @@ fun withGlobalExceptionHandling(block: () -> Int) { // We need to cover the case where some component in the call stack starts a non-daemon // thread, and then throws an Error that kills the main thread. This is extra safe implementation else -> { - FtlConstants.bugsnag?.notify(t) + captureError(t) t.printStackTrace() exitProcess(UNEXPECTED_ERROR) } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 154339c9b9..ec8c517a49 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -46,7 +46,6 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope val otherGcsFiles = uploadOtherFiles() val additionalApks = uploadAdditionalApks() val obbFiles = uploadObbFiles() - createAndroidTestContexts() .dumpShards(args) .upload(resultsBucket, resultsDir) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index 55e162f116..bd3ad88fd2 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -6,9 +6,11 @@ import flank.common.startWithNewLine import ftl.args.IArgs import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_STORAGE_LINK +import ftl.gc.GcStorage.uploadSessionId import ftl.gc.GcTestMatrix import ftl.json.MatrixMap import ftl.json.createSavedMatrix +import ftl.run.common.saveSessionId import ftl.run.common.updateMatrixFile import ftl.util.StopWatch import ftl.util.isInvalid @@ -29,7 +31,8 @@ internal suspend fun IArgs.afterRunTests( ).also { matrixMap -> updateMatrixFile(matrixMap) saveConfigFile(matrixMap) - + saveSessionId() + uploadSessionId() logLn(FtlConstants.indent + "${matrixMap.map.size} matrix ids created in ${stopwatch.check()}") val gcsBucket = GCS_STORAGE_LINK + resultsBucket + "/" + matrixMap.runPath logLn("${FtlConstants.indent}Raw results will be stored in your GCS bucket at [$gcsBucket]") diff --git a/test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt b/test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt deleted file mode 100644 index b564574781..0000000000 --- a/test_runner/src/main/kotlin/ftl/util/BugsnagInitHelper.kt +++ /dev/null @@ -1,24 +0,0 @@ -package ftl.util - -import com.bugsnag.Bugsnag -import java.io.File - -internal object BugsnagInitHelper { - - private const val GSUTIL_FOLDER = ".gsutil" - private const val ANALYTICS_FILE = "analytics-uuid" - private const val DISABLED = "DISABLED" - private const val FLANK_API_KEY = "3d5f8ba4ee847d6bb51cb9c347eda74f" - - internal fun initBugsnag( - useMock: Boolean, - rootPath: String = System.getProperty("user.home") - ) = when { - useMock -> null - analyticsFileExistAndIsDisabled(rootPath) -> null - else -> Bugsnag(FLANK_API_KEY) - } - - private fun analyticsFileExistAndIsDisabled(rootPath: String) = - File(rootPath, "$GSUTIL_FOLDER/$ANALYTICS_FILE").run { exists() && readText().trim() == DISABLED } -} diff --git a/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt b/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt new file mode 100644 index 0000000000..66324485ec --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt @@ -0,0 +1,60 @@ +package ftl.util + +import ftl.config.FtlConstants +import io.sentry.Sentry +import java.io.File +import java.util.UUID + +private const val FLANK_API_KEY = "https://f626934e7811480db91c40f62f5035f0@o475862.ingest.sentry.io/5514444" +private const val GSUTIL_FOLDER = ".gsutil" +private const val ANALYTICS_FILE = "analytics-uuid" +private const val DISABLED = "DISABLED" + +const val SESSION_ID = "session.id" +const val OS_NAME = "os.name" +const val FLANK_VERSION = " flank.version" +const val FLANK_REVISION = "flank.revision" +const val DEVICE_SYSTEM = "device.system" +const val TEST_TYPE = "test.type" + +val captureError by lazy { + initCrashReporter(FtlConstants.useMock) + ::notify +} + +internal fun initCrashReporter( + useMock: Boolean, + rootPath: String = System.getProperty("user.home") +) = when { + useMock -> null + isGoogleAnalyticsDisabled(rootPath) -> null + else -> initializeCrashReportWrapper() +} + +private fun isGoogleAnalyticsDisabled(rootPath: String) = + File(rootPath, "$GSUTIL_FOLDER/$ANALYTICS_FILE").run { exists() && readText().trim() == DISABLED } + +private fun initializeCrashReportWrapper() { + Sentry.init { + it.dsn = FLANK_API_KEY + it.release = readRevision() + } + setCrashReportTag( + SESSION_ID to sessionId, + OS_NAME to System.getProperty("os.name"), + FLANK_VERSION to readVersion(), + FLANK_REVISION to readRevision() + ) +} + +val sessionId by lazy { + UUID.randomUUID().toString() +} + +fun setCrashReportTag(vararg tags: Pair) = tags.forEach { + Sentry.setTag(it.first, it.second) +} + +private fun notify(error: Throwable) { + Sentry.captureException(error) +} diff --git a/test_runner/src/main/kotlin/ftl/util/Utils.kt b/test_runner/src/main/kotlin/ftl/util/Utils.kt index 9806b3e7ba..b804ed130a 100644 --- a/test_runner/src/main/kotlin/ftl/util/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/util/Utils.kt @@ -100,7 +100,9 @@ fun , T> mutableMapProperty( */ fun KMutableProperty.require() = getter.call() ?: throw FlankGeneralError( - "Invalid value for [${setter.annotations.filterIsInstance().first().value}]: no argument value found" + "Invalid value for [${ + setter.annotations.filterIsInstance().first().value + }]: no argument value found" ) fun getGACPathOrEmpty(): String = System.getenv("GOOGLE_APPLICATION_CREDENTIALS").orEmpty() diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index a621c88395..7fd66f0b42 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -22,14 +22,14 @@ fun main() { // "--debug", "firebase", "test", - "ios", + "android", "run", // "--dry", // "--dump-shards", "--output-style=single", // "--full-junit-result", // "--legacy-junit-result", - "-c=./test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-single-gameloop-ios.yml", + "-c=./test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-single-success.yml", "--project=$projectId" // "--client-details=key1=value1,key2=value2" ) diff --git a/test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt b/test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt similarity index 78% rename from test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt rename to test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt index bdfeba2467..64fd4b87a2 100644 --- a/test_runner/src/test/kotlin/ftl/util/FlankBugsnagInitHelperTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt @@ -16,16 +16,14 @@ private const val GSUTIL_FOLDER = ".gsutil" private const val ANALYTICS_FILE = "analytics-uuid" private const val DISABLED = "DISABLED\n" -class FlankBugsnagInitHelperTest { - - private val helper = BugsnagInitHelper +class FlankSentryInitHelperTest { @get:Rule val folder = TemporaryFolder() @Before fun setUp() { - LogbackLogger.FlankBugsnag.isEnabled = false + LogbackLogger.FlankSentry.isEnabled = false } @After @@ -38,14 +36,14 @@ class FlankBugsnagInitHelperTest { val subfolder = folder.newFolder(GSUTIL_FOLDER) File(subfolder, ANALYTICS_FILE).also { it.writeText(DISABLED) } - assertNull(helper.initBugsnag(useMock = false, folder.root.absolutePath)) + assertNull(initCrashReporter(useMock = false, folder.root.absolutePath)) } @Test fun `should not create Bugsnag object if user provided analytics-uuid`() { val subfolder = folder.newFolder(GSUTIL_FOLDER) File(subfolder, ANALYTICS_FILE).also { it.writeText(UUID.randomUUID().toString()) } - assertNotNull(helper.initBugsnag(useMock = false, folder.root.absolutePath)) + assertNotNull(initCrashReporter(useMock = false, folder.root.absolutePath)) } @Test @@ -53,6 +51,6 @@ class FlankBugsnagInitHelperTest { val subfolder = folder.newFolder(GSUTIL_FOLDER) File(subfolder, ANALYTICS_FILE).also { it.writeText(UUID.randomUUID().toString()) } - assertNull(helper.initBugsnag(useMock = true, folder.root.absolutePath)) + assertNull(initCrashReporter(useMock = true, folder.root.absolutePath)) } } diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index 650810c259..1a2bf69c71 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -263,9 +263,7 @@ class UtilsTest { val message = "not flank related error thrown" mockkObject(FtlConstants) every { FtlConstants.useMock } returns false - every { FtlConstants.bugsnag } returns mockk { - every { notify(any()) } returns true - } + val block = { throw FlankGeneralError(message) } // will