From f497ca35e8582834d3f93d056c9db8c4826abb3e Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Wed, 11 Dec 2024 13:55:47 +0100 Subject: [PATCH 1/6] simple dart app with sentry metrics --- .github/workflows/metrics.yml | 21 ++++++++++++++ metrics/.gitignore | 1 + metrics/build-dart.sh | 12 ++++++++ metrics/compare_sizes.sh | 38 +++++++++++++++++++++++++ metrics/prepare-dart.sh | 52 +++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+) create mode 100755 metrics/build-dart.sh create mode 100755 metrics/compare_sizes.sh create mode 100755 metrics/prepare-dart.sh diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 5d9256ade6..131f9d6f66 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -91,3 +91,24 @@ jobs: config: ./metrics/metrics-${{ matrix.platform }}.yml sauce-user: ${{ secrets.SAUCE_USERNAME }} sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} + + metrics-dart: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@f2c4f6686ca8e8d6e6d0f28410eeef506ed66aff # pin@v2.18.0 + + - name: create dart sample apps + working-directory: ./metrics + run: ./prepare-dart.sh + + - name: build dart sample apps + working-directory: ./metrics + run: ./build-dart.sh + + - name: Set file diff max threshold + run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB + + - name: Compare APK sizes + working-directory: ./metrics + run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD diff --git a/metrics/.gitignore b/metrics/.gitignore index f69bea8ddc..4ef0cea4f0 100644 --- a/metrics/.gitignore +++ b/metrics/.gitignore @@ -1 +1,2 @@ perf-test-app* +perf_test_console* \ No newline at end of file diff --git a/metrics/build-dart.sh b/metrics/build-dart.sh new file mode 100755 index 0000000000..73a5e1b87f --- /dev/null +++ b/metrics/build-dart.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd perf_test_console_plain +dart pub get +cd .. +dart compile exe perf_test_console_plain/bin/perf_test_console_plain.dart -o ./perf_test_console_plain.bin + +cd perf_test_console_sentry +dart pub get +cd .. +dart compile exe perf_test_console_sentry/bin/perf_test_console_sentry.dart -o ./perf_test_console_sentry.bin diff --git a/metrics/compare_sizes.sh b/metrics/compare_sizes.sh new file mode 100755 index 0000000000..cf9e424fb2 --- /dev/null +++ b/metrics/compare_sizes.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# Inputs: file1, file2, threshold +FILE1=$1 +FILE2=$2 +THRESHOLD=$3 + +if [ ! -f "$FILE1" ]; then + echo "File not found: $FILE1" + exit 1 +fi + +if [ ! -f "$FILE2" ]; then + echo "File not found: $FILE2" + exit 1 +fi + +# Get sizes of files (macOS-specific) +SIZE1=$(stat -f%z "$FILE1") +SIZE2=$(stat -f%z "$FILE2") + +# Calculate absolute size difference +SIZE_DIFF=$(( SIZE1 - SIZE2 )) +SIZE_DIFF=${SIZE_DIFF#-} # Convert to absolute value + +# Print results +echo "File 1: $FILE1 (Size: $SIZE1 bytes)" +echo "Binary 2: $FILE2 (Size: $SIZE2 bytes)" +echo "Difference: $SIZE_DIFF bytes" + +# Check if the size difference exceeds the threshold +if [ "$SIZE_DIFF" -gt "$THRESHOLD" ]; then + echo "ERROR: Size difference between $FILE1 and $FILE2 exceeds $THRESHOLD bytes!" + exit 1 +else + echo "SUCCESS: Size difference between $FILE1 and $FILE2 is within $THRESHOLD bytes." +fi diff --git a/metrics/prepare-dart.sh b/metrics/prepare-dart.sh new file mode 100755 index 0000000000..f0bac4730e --- /dev/null +++ b/metrics/prepare-dart.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +targetDir=$( + cd $(dirname $0) + pwd +) +[[ "$targetDir" != "" ]] || exit 1 + +dartCreate() { + name=${1//-/_} + dir=$targetDir/$1 + rm -rf $dir + echo "::group::dart create $1" + dart create -t console $name + echo '::endgroup::' +} + +dartCreate 'perf_test_console_plain' +dartCreate 'perf_test_console_sentry' + +echo '::group::Patch perf_test_console_sentry' +pubspec="$targetDir/perf_test_console_sentry/pubspec_overrides.yaml" +echo "Adding dependencies to $pubspec" +cat <>"$pubspec" + +dependency_overrides: + sentry: + path: ../../dart + +EOF + +fileToReplace="$targetDir/perf_test_console_sentry/bin/perf_test_console_sentry.dart" +if [[ -f "$fileToReplace" ]]; then + echo "Replacing $fileToReplace with new content" + cat <<'NEW_FILE_CONTENT' >"$fileToReplace" +import 'package:perf_test_console_sentry/perf_test_console_sentry.dart' as perf_test_console_sentry; +import 'package:sentry/sentry.dart'; + +Future main(List arguments) async { + await Sentry.init((options) { + options.dsn = 'https://e85b375ffb9f43cf8bdf9787768149e0@o447951.ingest.sentry.io/5428562'; + }); + + print('Hello world: ${perf_test_console_sentry.calculate()}!'); +} +NEW_FILE_CONTENT +else + echo "Error: File $fileToReplace not found!" + exit 1 +fi +echo '::endgroup::' From cc84f26daf5001782c1b35a7f867a8b237495f88 Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Tue, 17 Dec 2024 14:32:33 +0100 Subject: [PATCH 2/6] change naming --- .github/workflows/metrics.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 131f9d6f66..e90fb56a95 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -23,7 +23,7 @@ jobs: access_token: ${{ github.token }} metrics: - name: ${{ matrix.name }} + name: Sentry Flutter ${{ matrix.name }} runs-on: ${{ matrix.host }} timeout-minutes: 30 strategy: @@ -93,6 +93,7 @@ jobs: sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} metrics-dart: + name: Sentry Dart runs-on: macos-latest steps: - uses: actions/checkout@v4 @@ -109,6 +110,6 @@ jobs: - name: Set file diff max threshold run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB - - name: Compare APK sizes + - name: Compare executable size working-directory: ./metrics run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD From c6001136d437b3e11039bb7678a4186f95d7bcfe Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Tue, 17 Dec 2024 14:43:19 +0100 Subject: [PATCH 3/6] use action-app-sdk-overhead-metrics for console app --- .github/workflows/metrics.yml | 22 +++++++++++++++------- metrics/metrics-console.yml | 9 +++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 metrics/metrics-console.yml diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index e90fb56a95..3fd7927dbc 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -23,7 +23,7 @@ jobs: access_token: ${{ github.token }} metrics: - name: Sentry Flutter ${{ matrix.name }} + name: ${{ matrix.name }} runs-on: ${{ matrix.host }} timeout-minutes: 30 strategy: @@ -93,7 +93,7 @@ jobs: sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} metrics-dart: - name: Sentry Dart + name: Console runs-on: macos-latest steps: - uses: actions/checkout@v4 @@ -107,9 +107,17 @@ jobs: working-directory: ./metrics run: ./build-dart.sh - - name: Set file diff max threshold - run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB +# - name: Set file diff max threshold +# run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB - - name: Compare executable size - working-directory: ./metrics - run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD +# - name: Compare executable size +# working-directory: ./metrics +# run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD + + - name: Collect executable metrics + uses: getsentry/action-app-sdk-overhead-metrics@v1 + with: + name: Console + config: ./metrics/metrics-console.yml + sauce-user: ${{ secrets.SAUCE_USERNAME }} + sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} diff --git a/metrics/metrics-console.yml b/metrics/metrics-console.yml new file mode 100644 index 0000000000..f3d2a0670d --- /dev/null +++ b/metrics/metrics-console.yml @@ -0,0 +1,9 @@ +apps: + - name: io.sentry.dart.perfTestConsolePlain + path: perf_test_console_plain.bin + - name: io.sentry.dart.perfTestConsoleWithSentry + path: perf_test_console_sentry.bin + +binarySizeTest: + diffMin: 1200 KiB + diffMax: 1500 KiB From 7f8b13fbfdfdb331e8bd442cb139eca958b89604 Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Tue, 17 Dec 2024 14:50:34 +0100 Subject: [PATCH 4/6] revert for now --- .github/workflows/metrics.yml | 18 +++++------------- metrics/metrics-console.yml | 9 --------- 2 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 metrics/metrics-console.yml diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 3fd7927dbc..883d35a5a1 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -107,17 +107,9 @@ jobs: working-directory: ./metrics run: ./build-dart.sh -# - name: Set file diff max threshold -# run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB + - name: Set file diff max threshold + run: echo "THRESHOLD=13100000" >> $GITHUB_ENV # 1,31 MB -# - name: Compare executable size -# working-directory: ./metrics -# run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD - - - name: Collect executable metrics - uses: getsentry/action-app-sdk-overhead-metrics@v1 - with: - name: Console - config: ./metrics/metrics-console.yml - sauce-user: ${{ secrets.SAUCE_USERNAME }} - sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} + - name: Compare executable size + working-directory: ./metrics + run: ./compare_sizes.sh perf_test_console_plain.bin perf_test_console_sentry.bin $THRESHOLD diff --git a/metrics/metrics-console.yml b/metrics/metrics-console.yml deleted file mode 100644 index f3d2a0670d..0000000000 --- a/metrics/metrics-console.yml +++ /dev/null @@ -1,9 +0,0 @@ -apps: - - name: io.sentry.dart.perfTestConsolePlain - path: perf_test_console_plain.bin - - name: io.sentry.dart.perfTestConsoleWithSentry - path: perf_test_console_sentry.bin - -binarySizeTest: - diffMin: 1200 KiB - diffMax: 1500 KiB From 4726039caefb45d13c39feebb37fbdaa4b07adcf Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Mon, 13 Jan 2025 14:35:45 +0100 Subject: [PATCH 5/6] build on ubuntu --- .github/workflows/metrics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 883d35a5a1..f5e884d85a 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -94,7 +94,7 @@ jobs: metrics-dart: name: Console - runs-on: macos-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: subosito/flutter-action@f2c4f6686ca8e8d6e6d0f28410eeef506ed66aff # pin@v2.18.0 From b3a89d1b29c4682b6cde15abd82ac20b4ff7e7ab Mon Sep 17 00:00:00 2001 From: Denis Andrasec Date: Wed, 15 Jan 2025 10:56:57 +0100 Subject: [PATCH 6/6] fix code for ubuntu runner --- metrics/compare_sizes.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metrics/compare_sizes.sh b/metrics/compare_sizes.sh index cf9e424fb2..5d2d57cb1a 100755 --- a/metrics/compare_sizes.sh +++ b/metrics/compare_sizes.sh @@ -16,9 +16,9 @@ if [ ! -f "$FILE2" ]; then exit 1 fi -# Get sizes of files (macOS-specific) -SIZE1=$(stat -f%z "$FILE1") -SIZE2=$(stat -f%z "$FILE2") +# Get sizes of files (Linux-compatible) +SIZE1=$(stat --format=%s "$FILE1") +SIZE2=$(stat --format=%s "$FILE2") # Calculate absolute size difference SIZE_DIFF=$(( SIZE1 - SIZE2 )) @@ -26,7 +26,7 @@ SIZE_DIFF=${SIZE_DIFF#-} # Convert to absolute value # Print results echo "File 1: $FILE1 (Size: $SIZE1 bytes)" -echo "Binary 2: $FILE2 (Size: $SIZE2 bytes)" +echo "File 2: $FILE2 (Size: $SIZE2 bytes)" echo "Difference: $SIZE_DIFF bytes" # Check if the size difference exceeds the threshold @@ -35,4 +35,4 @@ if [ "$SIZE_DIFF" -gt "$THRESHOLD" ]; then exit 1 else echo "SUCCESS: Size difference between $FILE1 and $FILE2 is within $THRESHOLD bytes." -fi +fi \ No newline at end of file