diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index d9361cc982..aba714de43 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -1,5 +1,4 @@ object Dependencies { - const val SENTRY = "io.sentry:sentry:${Versions.SENTRY}" const val MIXPANEL = "com.mixpanel:mixpanel-java:${Versions.MIXPANEL}" const val DD_PLIST = "com.googlecode.plist:dd-plist:${Versions.DD_PLIST}" const val DEX_TEST_PARSER = "com.linkedin.dextestparser:parser:${Versions.DEX_TEST_PARSER}" diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 75e0436535..8cd25c5149 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,7 +1,4 @@ object Versions { - // https://github.com/getsentry/sentry-java/releases - const val SENTRY = "6.1.4" - // https://github.com/mixpanel/mixpanel-java/releases const val MIXPANEL = "1.5.0" diff --git a/docs/error_monitoring.md b/docs/error_monitoring.md deleted file mode 100644 index 7510166b95..0000000000 --- a/docs/error_monitoring.md +++ /dev/null @@ -1,37 +0,0 @@ -# Flank Error Monitoring - -Flank uses Sentry to monitor test runner stability. Sentry enables data driven decisions when prioritizing bug fixes. - -- https://sentry.io/ -- https://docs.sentry.io/ - -## Data Captured - -Sentry captures the following error data: - -Flank - - Stacktrace - - releaseStage of Flank (production or snapshot) - - version of Flank (git commit Flank was built from) - -Device - - hostname - - locale - - osArch - - osName - - osVersion - - runtimeVersions of Java - -## Disable Sentry - -Flank respects the same analytics opt out as gcloud CLI. - -`echo "DISABLED" > ~/.gsutil/analytics-uuid` - -## More information - -To see how Sentry is integrated within the Flank project please see the -[Flank Sentry testcase](../test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt) and -[the actual Sentry implementation](../test_runner/src/main/kotlin/ftl/util/CrashReporter.kt) - -- [Sentry Data Security, Privacy, and Compliance Overview](https://sentry.io/security/) diff --git a/docs/index.md b/docs/index.md index ec333654b5..333ced5a23 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ # Flank [![codecov](https://codecov.io/gh/Flank/flank/branch/master/graph/badge.svg)](https://codecov.io/gh/Flank/flank) -Flank is a [massively parallel Android and iOS test runner](https://docs.google.com/presentation/d/1goan9cXpimSJsS3L60WjljnFA_seUyaWb2e-bezm084/edit#slide=id.p1) for [Firebase Test Lab](https://firebase.google.com/docs/test-lab/). +Flank is a [massively parallel Android and iOS test runner](https://docs.google.com/presentation/d/1goan9cXpimSJsS3L60WjljnFA_seUyaWb2e-bezm084/edit#slide=id.p1) for [Firebase Test Lab](https://firebase.google.com/docs/test-lab/). Flank is YAML compatible with [the gcloud CLI](https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test). Flank provides extra features to accelerate velocity and increase quality. @@ -8,10 +8,6 @@ Flank is YAML compatible with [the gcloud CLI](https://cloud.google.com/sdk/gclo https://github.com/Flank/flank/releases/latest/download/flank.jar -## Sponsors - -See [error monitoring docs](./error_monitoring.md) to disable Sentry error monitoring. - ## Contributing - Install JDK 15 (it works also correctly on the previous version, a newer version is not guaranteed to work properly): @@ -37,14 +33,14 @@ See [error monitoring docs](./error_monitoring.md) to disable Sentry error monit | Exit code | Description | | ---: | :--- | -| 0 | All tests passed -| 1 | A general failure occurred. Possible causes include: a filename that does not exist or an HTTP/network error. -| 2 | Usually indicates missing or wrong usage of flags, incorrect parameters, errors in config files. -| 10 | At least one matrix not finished (usually a FTL internal error) or unexpected error occurred. -| 15 | Firebase Test Lab could not determine if the test matrix passed or failed, because of an unexpected error. -| 18 | The test environment for this test execution is not supported because of incompatible test dimensions. This error might occur if the selected Android API level is not supported by the selected device type. -| 19 | The test matrix was canceled by the user. -| 20 | A test infrastructure error occurred. +| 0 | All tests passed +| 1 | A general failure occurred. Possible causes include: a filename that does not exist or an HTTP/network error. +| 2 | Usually indicates missing or wrong usage of flags, incorrect parameters, errors in config files. +| 10 | At least one matrix not finished (usually a FTL internal error) or unexpected error occurred. +| 15 | Firebase Test Lab could not determine if the test matrix passed or failed, because of an unexpected error. +| 18 | The test environment for this test execution is not supported because of incompatible test dimensions. This error might occur if the selected Android API level is not supported by the selected device type. +| 19 | The test matrix was canceled by the user. +| 20 | A test infrastructure error occurred. ## CLI @@ -70,11 +66,11 @@ Run `test_runner/flank.ios.yml` with flank to verify iOS execution is working. # https://cloud.google.com/sdk/gcloud/reference/alpha/firebase/test/ios/run gcloud: # -- GcloudYml -- - - ### Results Bucket + + ### Results Bucket ## The name of a Google Cloud Storage bucket where raw test results will be stored # results-bucket: tmp_flank - + ### Results Directory ## The name of a unique Google Cloud Storage object within the results bucket where raw test results will be stored ## (default: a timestamp with a random suffix). @@ -164,7 +160,7 @@ gcloud: # version: 12.0 # locale: es_ES # orientation: landscape - + ### Directories to Pull ## A list of paths that will be copied from the device's storage to the designated results bucket after the test ## is complete. These must be absolute paths under /private/var/mobile/Media or /Documents @@ -180,9 +176,9 @@ gcloud: # other-files # com.my.app:/Documents/file.txt: local/file.txt # /private/var/mobile/Media/file.jpg: gs://bucket/file.jpg - + ### Additional IPA's - ## List of up to 100 additional IPAs to install, in addition to the one being directly tested. + ## List of up to 100 additional IPAs to install, in addition to the one being directly tested. ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. # additional-ipas: # - gs://bucket/additional.ipa @@ -199,10 +195,10 @@ gcloud: ### Test type ## The type of iOS test to run. TYPE must be one of: xctest, game-loop. Default: xctest # type: xctest - + ### Application Path - ## The path to the application archive (.ipa file) for game-loop testing. - ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. + ## The path to the application archive (.ipa file) for game-loop testing. + ## The path may be in the local filesystem or in Google Cloud Storage using gs:// notation. ## This flag is only valid when --type=game-loop is also set # app: # - gs://bucket/additional.ipa OR path/to/local/ipa/file.ipa @@ -328,14 +324,14 @@ flank: ### Only Test Configuration ## Constrains a test action to only test a specified test configuration within a test plan and exclude all other test configurations. - ## Flank can combine multiple constraint options, but -only-test-configuration has precedence over -skip-test-configuration. + ## Flank can combine multiple constraint options, but -only-test-configuration has precedence over -skip-test-configuration. ## Each test configuration name must match the name of a configuration specified in a test plan and is case-sensitive. ## Default: null (run all test configurations) # only-test-configuration: en ### Skip Test Configuration ## Constrains a test action to skip a specified test configuration and include all other test configurations. - ## Flank can combine multiple constraint options, but -only-test-configuration has precedence over -skip-test-configuration. + ## Flank can combine multiple constraint options, but -only-test-configuration has precedence over -skip-test-configuration. ## Each test configuration name must match the name of a configuration specified in a test plan and is case-sensitive. ## Default: null (run all test configurations) # skip-test-configuration: en @@ -384,7 +380,7 @@ gcloud: ## Enable video recording during the test. Disabled by default. Use --record-video to enable. # record-video: true - ### Timeout + ### Timeout ## The max time this test execution can run before it is cancelled (default: 15m). ## It does not include any time necessary to prepare and clean up the target device. ## The maximum possible testing time is 45m on physical devices and 60m on virtual devices. @@ -514,7 +510,7 @@ gcloud: # - label2 ### OBB filenames - ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. + ## A list of OBB required filenames. OBB file name must conform to the format as specified by Android e.g. ## [main|patch].0300110.com.example.android.obb which will be installed into /Android/obb// on the device. # obb-names: # - [main|patch]..com.example.android.obb @@ -579,8 +575,8 @@ gcloud: # version: 27 ### test-targets-for-shard - ## Specifies a group of packages, classes, and/or test cases to run in each shard (a group of test cases). - ## The shards are run in parallel on separate devices. You can repeat this flag up to 50 times to specify multiple shards when one or more physical devices are selected, + ## Specifies a group of packages, classes, and/or test cases to run in each shard (a group of test cases). + ## The shards are run in parallel on separate devices. You can repeat this flag up to 50 times to specify multiple shards when one or more physical devices are selected, ## or up to 500 times when no physical devices are selected. ## Note: If you include the flags environment-variable or test-targets when running test-targets-for-shard, the flags are applied to all the shards you create. # test-target-for-shard: @@ -635,7 +631,7 @@ flank: ## Default: false # use-average-test-time-for-new-tests: true - ### Default Test Time + ### Default Test Time ## Set default test time used for calculating shards. ## Default: 120.0 # default-test-time: 15 @@ -981,7 +977,7 @@ v3.0-SNAPSHOT ## Maven -You can consume Flank via maven. +You can consume Flank via maven. See the [maven repo](https://bintray.com/flank/maven/flank) for all supported versions. ``` @@ -1041,30 +1037,30 @@ and flank's example [gradle-export-api](https://github.com/Flank/flank/tree/mast 3) > Test run failed to complete. Expected 786 tests, received 660 Try setting `use-orchestrator: false`. Parameterized tests [are not compatible with orchestrator](https://stackoverflow.com/questions/48735268/unable-to-run-parameterized-tests-with-android-test-orchestrator). Flank uses [orchestrator by default on Android.](https://developer.android.com/training/testing/junit-runner) - + 4) > I have an issue when attempting to sync the Flank Gradle project - > Task 'prepareKotlinBuildScriptModel' not found in project ':test_runner'. + > Task 'prepareKotlinBuildScriptModel' not found in project ':test_runner'. > or similar - + - Make sure you do not change any module specific settings for Gradle - Clear IDE cache using `File > Invalidate Caches / Restart` - Re-import project using root `build.gradle.kts` - Sync project again - + 5) > Does Flank support Cucumber? - + Please check [document](cucumber_support.md) for more info 6) > How can I find project id? - + Please check the [firebase documentation](https://firebase.google.com/docs/projects/learn-more?hl=en#find_the_project_id) about finding the project id 7) > How do I run Flank with a proxy? - + `java -Dhttp.proxyHost=localhost -Dhttp.proxyPort=8080 -Dhttp.proxyUser=user -Dhttp.proxyPassword=pass -jar ./test_runner/build/libs/flank.jar firebase test android run` - + See [google-auth-library-java](https://github.com/googleapis/google-auth-library-java#configuring-a-proxy) for details. - + # Resources diff --git a/docs/refactor/flank_run/flank_run_refactor_sdd.md b/docs/refactor/flank_run/flank_run_refactor_sdd.md index 49a2f118b2..2245de5317 100644 --- a/docs/refactor/flank_run/flank_run_refactor_sdd.md +++ b/docs/refactor/flank_run/flank_run_refactor_sdd.md @@ -2,19 +2,19 @@ # References -* `https://github.com/Flank/flank/issues/1317` +* `https://github.com/Flank/flank/issues/1317` # Motivation -Flank run is getting bigger over the time. -Currently, the amount of code related to test run is large, +Flank run is getting bigger over the time. +Currently, the amount of code related to test run is large, so it could be hard to understand and keep in mind the whole process. To make the work more convenient, faster and scalable, we should re-think and reorganize the code between packages and files. # Goals -* `All logic related to flank run grouped in one root package` +* `All logic related to flank run grouped in one root package` * `Package structure flat as possible.` * `Flank run process simplified to sequence of synchronous atomic steps.` * `Packages and code are easy to identify as steps.` @@ -32,7 +32,7 @@ and reorganize the code between packages and files. # API > What will the proposed API look like? - + # Results > What was the outcome of the project? @@ -60,7 +60,6 @@ ftl/run/ │ ├── GetLastMatrices.kt │ ├── PollMatrices.kt │ ├── PrettyPrint.kt -│ ├── SaveSessionId.kt │ └── UpdateMatrixFile.kt ├── exception │ ├── ExceptionHandler.kt @@ -142,7 +141,6 @@ ftl/run/ * `ftl/shard/Shard.kt` * `ftl/shard/TestCasesCreator.kt` * `ftl/util/Artifacts.kt` -* `ftl/util/CrashReporter.kt` * `ftl/util/FileReference.kt` * `ftl/util/FlankTestMethod.kt` * `ftl/util/FlowExt.kt` diff --git a/docs/refactor/investigation.md b/docs/refactor/investigation.md index f74e3de252..76582cf95c 100644 --- a/docs/refactor/investigation.md +++ b/docs/refactor/investigation.md @@ -118,7 +118,6 @@ command that: * `com.google.cloud:google-cloud-nio` * `com.google.cloud:google-cloud-storage` * `com.google.apis:google-api-services-toolresults` -* `io.sentry:sentry` * `com.mixpanel:mixpanel-java` #### List of external API usages in files @@ -446,7 +445,7 @@ where * `NetworkProfilesListCommand` * `IosOrientationsListCommand` * `IPBlocksListCommand` - + * `AndroidRunCommand` * `AndroidArgs/validate` * `AndroidArgs/assertDevicesSupported` @@ -471,7 +470,6 @@ where * `GcAndroidTestMatrix/build` * `AbstractGoogleJsonClientRequest/executeWithRetry` * `IArgs/afterRunTests` - * `IArgs/uploadSessionId` -> `GcStorage/upload` * `MatrixMap/printMatricesWebLinks` -> `getOrUpdateWebLink` -> `GcTestMatrix/refresh` * `pollMatrices` -> `matrixChangesFlow` -> `GcTestMatrix/refresh` * `Iterable/updateMatrixMap` -> `SavedMatrix/updateWithMatrix` -> `TestMatrix/fetchTestOutcomeContext` @@ -494,7 +492,7 @@ where * `GcToolResults/listTestCases` * `GcToolResults/getStepResult` * `FullJUnitReport.run` -> `GcStorage.uploadReportResult` - * `ReportManager/createAndUploadPerformanceMetricsForAndroid` -> `List>.getAndUploadPerformanceMetrics` -> + * `ReportManager/createAndUploadPerformanceMetricsForAndroid` -> `List>.getAndUploadPerformanceMetrics` -> * `TestExecution.getPerformanceMetric` -> `GcToolResults.getPerformanceMetric` * `PerfMetricsSummary.upload` -> `GcStorage.uploadPerformanceMetrics` * `GcStorage/uploadMatricesId` @@ -504,7 +502,7 @@ where * `GcStorage.storage.list` * `Blob.downloadTo` * `MatrixMap/printMatricesWebLinks` -> `getOrUpdateWebLink` -> `GcTestMatrix/refresh` - + * `IosRunCommand` * `IosArgs/validate` * `IosArgs/assertDevicesSupported` @@ -517,12 +515,12 @@ where * `IArgs/uploadOtherFiles` -> `GcStorage/upload` * `IosArgs.uploadAdditionalIpas` * `IosArgs.dumpShardsIfXcTest` -> `GcStorage/upload` - * `IosArgs/createIosTestContexts` + * `IosArgs/createIosTestContexts` * `IosArgs.createXcTestContexts` * `IArgs.uploadIfNeeded` -> `FileReference.uploadIfNeeded` -> `GcStorage.upload` * `GcStorage.uploadXCTestFile` * `IosArgs.createGameloopTestContexts` - * `IArgs.uploadIfNeeded` -> `FileReference.uploadIfNeeded` -> `GcStorage.upload` + * `IArgs.uploadIfNeeded` -> `FileReference.uploadIfNeeded` -> `GcStorage.upload` * `GcIosTestMatrix/build` * `AbstractGoogleJsonClientRequest/executeWithRetry` * `IArgs/afterRunTests` - the rest of steps are same as for android diff --git a/docs/refactor/uml/flank_run_relations.puml b/docs/refactor/uml/flank_run_relations.puml index bcf04dfd52..c8549ed1ae 100644 --- a/docs/refactor/uml/flank_run_relations.puml +++ b/docs/refactor/uml/flank_run_relations.puml @@ -83,10 +83,6 @@ run_common_pollMatrices --> gc_GcTestMatrix run_common_pollMatrices --> run_status_TestMatrixStatusPrinter run_common_pollMatrices --> util_MatrixState -run_common_saveSessionId --> args_IArgs -run_common_saveSessionId --> util_sessionId -run_common_saveSessionId --> util_sessionId - run_common_updateMatrixFile --> args_IArgs run_common_updateMatrixFile --> json_MatrixMap @@ -96,7 +92,6 @@ run_common_getMatrixFilePath --> json_MatrixMap run_exception_withGlobalExceptionHandling --> json_SavedMatrix run_exception_withGlobalExceptionHandling --> run_cancelMatrices -run_exception_withGlobalExceptionHandling --> util_closeCrashReporter run_exception_withGlobalExceptionHandling --> util_report run_platform_android_createAndroidLoopConfig --> args_AndroidArgs @@ -169,11 +164,9 @@ run_platform_android_uploadObbFiles --> gc_GcStorage run_platform_common_afterRunTests --> args_IArgs run_platform_common_afterRunTests --> config_FtlConstants run_platform_common_afterRunTests --> config_FtlConstants_GCS_STORAGE_LINK -run_platform_common_afterRunTests --> gc_GcStorage_uploadSessionId run_platform_common_afterRunTests --> gc_GcTestMatrix run_platform_common_afterRunTests --> json_MatrixMap run_platform_common_afterRunTests --> json_createSavedMatrix -run_platform_common_afterRunTests --> run_common_saveSessionId run_platform_common_afterRunTests --> run_common_updateMatrixFile run_platform_common_afterRunTests --> util_StopWatch run_platform_common_afterRunTests --> util_isInvalid diff --git a/flank_wrapper/build.gradle.kts b/flank_wrapper/build.gradle.kts index 3683c2ba9d..1f10e8597c 100644 --- a/flank_wrapper/build.gradle.kts +++ b/flank_wrapper/build.gradle.kts @@ -78,7 +78,6 @@ publishing { dependencies { implementation(project(Modules.MIXPANEL_ANALYTICS)) implementation(project(Modules.COMMON)) - implementation(Dependencies.SENTRY) implementation(Dependencies.Fuel.CORE) testImplementation(Dependencies.JUNIT) testImplementation(Dependencies.TRUTH) diff --git a/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/CrashReporter.kt b/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/CrashReporter.kt deleted file mode 100644 index e975d9a06d..0000000000 --- a/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/CrashReporter.kt +++ /dev/null @@ -1,40 +0,0 @@ -package com.github.flank.wrapper.internal - -import flank.common.config.isTest -import flank.tool.analytics.mixpanel.Mixpanel -import io.sentry.Sentry - -private const val SESSION_ID = "session.id" -private const val OS_NAME = "os.name" -private const val FLANK_WRAPPER_VERSION = "flank_wrapper.version" -private const val DSN = "https://2df245c4e26d44e1bff106a12637d5e4@o475862.ingest.sentry.io/5855220" - -fun setupCrashReporter() { - if (isTest().not()) { - Sentry.init { options -> - options.dsn = DSN - options.tracesSampleRate = 1.0 - } - - logTags( - SESSION_ID to Mixpanel.sessionId, - OS_NAME to osName, - FLANK_WRAPPER_VERSION to flankWrapperVersion, - ) - } -} - -private fun logTags(vararg tags: Pair) { - tags.forEach { (key, value) -> Sentry.setTag(key, value) } -} - -private val osName: String - get() = System.getProperty("os.name") - -private val flankWrapperVersion by lazy { - (object {}.javaClass.getResource("/version.txt")?.readText() ?: "UNKNOWN") -} - -internal fun Throwable.report() { - Sentry.captureException(this) -} diff --git a/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/RunFlankOnLatestVersion.kt b/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/RunFlankOnLatestVersion.kt index 33fbc3201f..de8d70ba31 100644 --- a/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/RunFlankOnLatestVersion.kt +++ b/flank_wrapper/src/main/kotlin/com/github/flank/wrapper/internal/RunFlankOnLatestVersion.kt @@ -1,13 +1,10 @@ package com.github.flank.wrapper.internal internal fun runFlankOnLatestVersion(args: Array) { - runCatching { - setupCrashReporter() - if (!compareLatestVersionCheckSumWithCurrent()) { - println("The new Flank version is available. Downloading new version...") - downloadLatestFlankVersion() - println("The new Flank version is ready to use") - } - executeFlank(args) - }.onFailure(Throwable::report) + if (!compareLatestVersionCheckSumWithCurrent()) { + println("The new Flank version is available. Downloading new version...") + downloadLatestFlankVersion() + println("The new Flank version is ready to use") + } + executeFlank(args) } diff --git a/integration_tests/src/test/resources/compare/GameloopIT-android-compare b/integration_tests/src/test/resources/compare/GameloopIT-android-compare index 3fd08e3a56..5d0e305ae2 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-android-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-android-compare @@ -72,7 +72,6 @@ RunTests \s* 0 test \/ 0 shard \s* - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare index 0a6b86deaf..d71af8f517 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare @@ -61,7 +61,6 @@ RunTests 0 test \/ 0 shard Uploading \[FlankGameLoopExample.ipa\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index a1416f5263..470091a131 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -87,7 +87,6 @@ RunTests [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index c310fb6d93..af256f6f09 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -97,7 +97,6 @@ RunTests [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index ed46c74597..2f0c58a733 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -69,7 +69,6 @@ RunTests 0 test \/ 0 shard - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index a8e277fd82..2b68b0e29e 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -77,7 +77,6 @@ RunTests 1 test \/ 1 shard - Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index 72b835a780..dcbd8806a7 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -173,7 +173,6 @@ tasks.withType { dependencies { implementation(project(Modules.COMMON)) implementation(project(Modules.MIXPANEL_ANALYTICS)) - implementation(Dependencies.SENTRY) implementation(Dependencies.DD_PLIST) implementation(Dependencies.DEX_TEST_PARSER) diff --git a/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt b/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt index 52bd6d9aff..8a65fd92c1 100644 --- a/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt +++ b/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt @@ -19,13 +19,9 @@ import ftl.reports.output.outputReport import ftl.reports.output.toOutputReportConfiguration import ftl.run.dumpShards import ftl.run.newTestRun -import ftl.util.DEVICE_SYSTEM -import ftl.util.PROJECT_ID import ftl.util.StopWatch -import ftl.util.TEST_TYPE import ftl.util.loadFile import ftl.util.printVersionInfo -import ftl.util.setCrashReportTag import java.nio.file.Paths interface RunTestAndroid : Output { @@ -53,11 +49,6 @@ operator fun RunTestAndroid.invoke() { outputReport.configure(toOutputReportConfiguration()) outputReport.log(this) - setCrashReportTag( - PROJECT_ID to project, - DEVICE_SYSTEM to "android", - TEST_TYPE to type?.name.orEmpty() - ) reportConfiguration() }.validate().also { args -> runBlockingWithObservingRunState { diff --git a/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt b/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt index 0cb2125cc9..f1a593ba1e 100644 --- a/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt +++ b/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt @@ -19,13 +19,9 @@ import ftl.reports.output.outputReport import ftl.reports.output.toOutputReportConfiguration import ftl.run.dumpShards import ftl.run.newTestRun -import ftl.util.DEVICE_SYSTEM -import ftl.util.PROJECT_ID import ftl.util.StopWatch -import ftl.util.TEST_TYPE import ftl.util.loadFile import ftl.util.printVersionInfo -import ftl.util.setCrashReportTag import java.nio.file.Paths interface RunIosTest : Output { @@ -53,11 +49,6 @@ operator fun RunIosTest.invoke() { setupLogLevel() outputReport.configure(toOutputReportConfiguration()) outputReport.log(this) - setCrashReportTag( - PROJECT_ID to project, - DEVICE_SYSTEM to "ios", - TEST_TYPE to type?.name.orEmpty() - ) reportConfiguration() diff --git a/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt b/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt index 6bbd3a1898..a426a8086c 100644 --- a/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt +++ b/test_runner/src/main/kotlin/ftl/http/ExecuteWithRetry.kt @@ -5,8 +5,6 @@ import com.google.api.client.http.HttpResponseException import ftl.run.exception.FailureToken import ftl.run.exception.PermissionDenied import ftl.run.exception.ProjectNotFound -import ftl.util.report -import io.sentry.SentryLevel import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import java.io.IOException @@ -23,10 +21,6 @@ private inline fun withRetry(crossinline block: () -> T): T = runBlocking { try { return@runBlocking block() } catch (err: IOException) { - // We want to send every occurrence of Google API error for statistic purposes - // https://github.com/Flank/flank/issues/701 - FlankGoogleApiError(err).report(SentryLevel.DEBUG) - lastError = err if (err is HttpResponseException) { // we want to handle some FTL errors with special care @@ -43,8 +37,6 @@ private inline fun withRetry(crossinline block: () -> T): T = runBlocking { throw IOException("Request failed", lastError) } -private class FlankGoogleApiError(exception: Throwable) : Error(exception) - private fun HttpResponseException.containsBadTokenMessage() = message?.let { it.contains("\"error\": \"invalid_grant\"") && it.contains("https://oauth2.googleapis.com/token") } ?: false diff --git a/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt b/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt index bd6654b779..4d27762c72 100644 --- a/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt +++ b/test_runner/src/main/kotlin/ftl/log/LogbackLogger.kt @@ -2,7 +2,6 @@ package ftl.log import ch.qos.logback.classic.Level import ch.qos.logback.classic.Logger -import io.sentry.Sentry import org.slf4j.LoggerFactory.getLogger import kotlin.properties.Delegates @@ -17,5 +16,4 @@ sealed class LogbackLogger(private val logger: Logger) : FlankLogger { } object Root : LogbackLogger(getLogger(Logger.ROOT_LOGGER_NAME)) - object FlankSentry : LogbackLogger(getLogger(Sentry::class.java)) } diff --git a/test_runner/src/main/kotlin/ftl/log/Loggers.kt b/test_runner/src/main/kotlin/ftl/log/Loggers.kt index a5de98bc33..8de2089a7c 100644 --- a/test_runner/src/main/kotlin/ftl/log/Loggers.kt +++ b/test_runner/src/main/kotlin/ftl/log/Loggers.kt @@ -4,7 +4,6 @@ import com.google.api.client.http.GoogleApiLogger private val LOGGERS = listOf( LogbackLogger.Root, - LogbackLogger.FlankSentry, GoogleApiLogger ) diff --git a/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt b/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt deleted file mode 100644 index e045b8cace..0000000000 --- a/test_runner/src/main/kotlin/ftl/run/common/SaveSessionId.kt +++ /dev/null @@ -1,9 +0,0 @@ -package ftl.run.common - -import flank.tool.analytics.mixpanel.Mixpanel -import ftl.args.IArgs -import java.nio.file.Paths - -const val SESSION_ID_FILE = "session_id.txt" - -fun IArgs.saveSessionId() = Paths.get(localResultDir, SESSION_ID_FILE).toFile().writeText(Mixpanel.sessionId) diff --git a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt index 859ca75046..c7405c42fb 100644 --- a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt +++ b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt @@ -7,15 +7,11 @@ import ftl.reports.output.generate import ftl.reports.output.outputReport import ftl.reports.printTotalDuration import ftl.run.cancelMatrices -import ftl.util.closeCrashReporter -import ftl.util.report -import io.sentry.SentryLevel import kotlinx.coroutines.runBlocking import kotlin.system.exitProcess fun withGlobalExceptionHandling(block: () -> Int) { withGlobalExceptionHandling(block) { - closeCrashReporter() outputReport.generate() printTotalDuration() exitProcess(it) @@ -55,7 +51,6 @@ internal fun withGlobalExceptionHandling(block: () -> Int, exitProcessFunction: } is InfrastructureError -> { - t.report(SentryLevel.DEBUG) printError("An infrastructure error occurred.") printError("Details: ${t.messageOrUnavailable}") exitProcessFunction(INFRASTRUCTURE_ERROR) @@ -67,7 +62,6 @@ internal fun withGlobalExceptionHandling(block: () -> Int, exitProcessFunction: } is FTLError -> { - t.report() t.matrix.logError() exitProcessFunction(UNEXPECTED_ERROR) } @@ -88,7 +82,6 @@ internal fun withGlobalExceptionHandling(block: () -> Int, exitProcessFunction: // thread, and then throws an Error that kills the main thread. This is extra safe implementation else -> { t.printStackTrace() - t.report() exitProcessFunction(UNEXPECTED_ERROR) } } diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index aaba21da46..83c17f95d2 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -5,14 +5,11 @@ import flank.common.startWithNewLine import ftl.api.RemoteStorage import ftl.api.TestMatrix import ftl.api.refreshTestMatrix -import ftl.api.uploadToRemoteStorage import ftl.args.IArgs import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_STORAGE_LINK import ftl.json.MatrixMap import ftl.reports.addStepTime -import ftl.run.common.SESSION_ID_FILE -import ftl.run.common.saveSessionId import ftl.run.common.updateMatrixFile import ftl.util.StopWatch import ftl.util.formatted @@ -33,8 +30,6 @@ internal suspend fun IArgs.afterRunTests( ).also { matrixMap -> updateMatrixFile(matrixMap) saveConfigFile(matrixMap) - saveSessionId() - uploadSessionId() logLn(FtlConstants.indent + "${matrixMap.map.size} matrix ids created in ${stopwatch.check().formatted()}") val gcsBucket = GCS_STORAGE_LINK + resultsBucket + "/" + matrixMap.runPath logLn("${FtlConstants.indent}Raw results will be stored in your GCS bucket at [$gcsBucket]") @@ -76,12 +71,3 @@ private tailrec suspend fun getOrUpdateWebLink(link: String, project: String, ma project = project, matrixId = matrixId ) - -fun IArgs.uploadSessionId() = takeUnless { disableResultsUpload }?.let { - val file = Paths.get(localResultDir, SESSION_ID_FILE).toString() - if (file.startsWith(FtlConstants.GCS_PREFIX)) return file - uploadToRemoteStorage( - RemoteStorage.Dir(resultsBucket, resultsDir), - RemoteStorage.Data(file, Files.readAllBytes(Paths.get(file))) - ) -} diff --git a/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt b/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt deleted file mode 100644 index d4ee87bc20..0000000000 --- a/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt +++ /dev/null @@ -1,78 +0,0 @@ -package ftl.util - -import flank.common.config.isTest -import flank.tool.analytics.mixpanel.Mixpanel -import flank.tool.resource.readRevision -import flank.tool.resource.readVersion -import io.sentry.Sentry -import io.sentry.SentryLevel -import java.io.File - -private const val FLANK_API_KEY = "https://f626934e7811480db91c40f62f5035f0@o475862.ingest.sentry.io/5514444" -private const val GSUTIL_FOLDER = ".gsutil" -private const val ANALYTICS_FILE = "analytics-uuid" -private const val DISABLED = "DISABLED" - -const val OS_NAME = "os.name" -const val FLANK_VERSION = "flank.version" -const val FLANK_REVISION = "flank.revision" -const val DEVICE_SYSTEM = "device.system" -const val TEST_TYPE = "test.type" -const val PROJECT_ID = "project.id" - -private val configureCrashReporter by lazy { initCrashReporter() } - -private var crashReportingEnabled = true - -fun disableCrashReporting() { - crashReportingEnabled = false - configureCrashReporter -} - -fun Throwable.report(logLevel: SentryLevel = SentryLevel.ERROR) { - if (isTest().not()) { - configureCrashReporter - notify(this, logLevel) - } -} - -internal fun initCrashReporter( - disabledCrashReporter: Boolean = isTest() || crashReportingEnabled.not(), - rootPath: String = System.getProperty("user.home") -) = when { - disabledCrashReporter -> null - isGoogleAnalyticsDisabled(rootPath) -> null - else -> initializeCrashReportWrapper() -} - -internal fun isGoogleAnalyticsDisabled(rootPath: String) = - File(rootPath, "$GSUTIL_FOLDER/$ANALYTICS_FILE").run { exists() && readText().trim() == DISABLED } - -private fun initializeCrashReportWrapper() { - Sentry.init { - it.dsn = FLANK_API_KEY - it.release = readRevision() - } - setCrashReportTag( - Mixpanel.SESSION_ID to Mixpanel.sessionId, - OS_NAME to System.getProperty("os.name"), - FLANK_VERSION to readVersion(), - FLANK_REVISION to readRevision() - ) -} - -fun setCrashReportTag( - vararg tags: Pair -) = tags.forEach { (property, value) -> Sentry.setTag(property, value) } - -private fun notify(error: Throwable, logLevel: SentryLevel) { - Sentry.withScope { - it.level = logLevel - Sentry.captureException(error) - } -} - -fun closeCrashReporter() { - Sentry.endSession() - Sentry.close() -} diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index ab8b1ec15e..ab37725b03 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -3,15 +3,12 @@ import ftl.presentation.cli.MainCommand import ftl.reports.startDurationMeasurement import ftl.run.exception.withGlobalExceptionHandling -import ftl.util.disableCrashReporting import picocli.CommandLine import java.io.File fun main() { println(File("").absolutePath) - disableCrashReporting() // GoogleApiLogger.logAllToStdout() - // for debugging. Run test from IntelliJ IDEA // run "gradle check" to generate required fixtures diff --git a/test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt b/test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt deleted file mode 100644 index 23334cbf64..0000000000 --- a/test_runner/src/test/kotlin/ftl/util/FlankSentryInitHelperTest.kt +++ /dev/null @@ -1,61 +0,0 @@ -package ftl.util - -import ftl.log.LogbackLogger -import io.mockk.unmockkAll -import org.junit.After -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertNull -import org.junit.Before -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import java.io.File -import java.util.UUID - -private const val GSUTIL_FOLDER = ".gsutil" -private const val ANALYTICS_FILE = "analytics-uuid" -private const val DISABLED = "DISABLED\n" - -class FlankSentryInitHelperTest { - - @get:Rule - val folder = TemporaryFolder() - - @Before - fun setUp() { - LogbackLogger.FlankSentry.isEnabled = false - } - - @After - fun tearDown() { - unmockkAll() - } - - @Test - fun `should not create Sentry object if user has analytics disabled`() { - val subfolder = folder.newFolder(GSUTIL_FOLDER) - File(subfolder, ANALYTICS_FILE).also { it.writeText(DISABLED) } - - assertNull(initCrashReporter(disabledCrashReporter = false, folder.root.absolutePath)) - } - @Test - fun `should not create Sentry object if user provided analytics-uuid`() { - val subfolder = folder.newFolder(GSUTIL_FOLDER) - File(subfolder, ANALYTICS_FILE).also { it.writeText(UUID.randomUUID().toString()) } - - assertNotNull(initCrashReporter(disabledCrashReporter = false, folder.root.absolutePath)) - } - - @Test - fun `should create Sentry object if mock server used`() { - val subfolder = folder.newFolder(GSUTIL_FOLDER) - File(subfolder, ANALYTICS_FILE).also { it.writeText(UUID.randomUUID().toString()) } - - assertNull(initCrashReporter(disabledCrashReporter = true, folder.root.absolutePath)) - } - - @Test - fun `should not create Sentry object when running tests`() { - assertNull(initCrashReporter()) - } -} diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index 0b85cc6e6d..aec9e1cf17 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -263,23 +263,6 @@ class UtilsTest { ) } - @Test - fun `should notify sentry if non related flank error occurred`() { - // given - val message = "not flank related error thrown" - val exception = Exception(message) - mockkStatic(Throwable::report) - - // when - then - withGlobalExceptionHandling( - block = { throw exception }, - exitProcessFunction = { exitCode -> - assertThat(exitCode).isEqualTo(UNEXPECTED_ERROR) - verify { exception.report() } - } - ) - } - @Test fun `should terminate process with exit code 0 if at least one matrix failed and ignore-failed-tests flag is true`() { // given