ci: collect SDK app metrics (#1034)

.github/workflows/metrics.yml

@@ -0,0 +1,81 @@
+name: SDK metrics
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - .github/workflows/metrics.yml
+      - dart/**
+      - flutter/**
+      - metrics/**
+
+jobs:
+  metrics:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.host }}
+    strategy:
+      # We want that the matrix keeps running, default is to cancel all jobs if a single fails.
+      fail-fast: false
+      matrix:
+        include:
+          - platform: ios
+            name: iOS
+            appPlain: test-app-plain.ipa
+            host: macos-latest
+          - platform: android
+            name: Android
+            appPlain: metrics/perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk
+            host: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Let's stick to an explicit version and update manually because a version change may affect results.
+      # If it would update implicitly it could confuse people to think the change is actually caused by the PR.
+      # Instead, we use Updater (update-deps.yml) to create PRs for explicit Flutter SDK update.
+      - name: Read configured Flutter SDK version
+        id: conf
+        run: |
+          version=$(grep "version" metrics/flutter.properties | cut -d'=' -f2 | xargs)
+          echo "::set-output name=flutter::$version"
+
+      - name: Install Flutter v${{ steps.conf.outputs.flutter }}
+        uses: subosito/flutter-action@1e6ee87cb840500837bcd50a667fb28815d8e310 # pin@v2
+        with:
+          flutter-version: ${{ steps.conf.outputs.flutter }}
+
+      - uses: actions/setup-java@v3
+        if: ${{ matrix.platform == 'android' }}
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - run: ./metrics/prepare.sh
+
+      - uses: actions/cache@v3
+        id: app-plain-cache
+        with:
+          path: ${{ matrix.appPlain }}
+          key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ matrix.platform }}-${{ hashFiles('metrics/perf-test-app-plain/pubspec.yaml') }}
+
+      - name: Build
+        run: ./metrics/build.sh ${{ matrix.platform }}
+        env:
+          # Necessary to build an iOS .ipa (using fastlane)
+          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
+          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
+          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
+          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
+          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
+          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
+          APP_PLAIN: ${{ matrix.appPlain }}
+
+      - name: Collect apps metrics
+        uses: getsentry/action-app-sdk-overhead-metrics@v1
+        with:
+          name: ${{ matrix.name }}
+          config: ./metrics/metrics-${{ matrix.platform }}.yml
+          sauce-user: ${{ secrets.SAUCE_USERNAME }}
+          sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}

.github/workflows/update-deps.yml

@@ -25,3 +25,11 @@ jobs:
       name: Cocoa SDK
     secrets:
       api-token: ${{ secrets.CI_DEPLOY_KEY }}
+
+  metrics-flutter:
+    uses: getsentry/github-workflows/.github/workflows/updater.yml@v2
+    with:
+      path: metrics/flutter.properties
+      name: Flutter SDK (metrics)
+    secrets:
+      api-token: ${{ secrets.CI_DEPLOY_KEY }}

metrics/.gitignore

@@ -0,0 +1 @@
+perf-test-app*

metrics/build.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+targetDir=$(
+  cd $(dirname $0)
+  pwd
+)
+[[ "$targetDir" != "" ]] || exit 1
+
+if [ "$#" -gt 2 ]; then
+  shift
+  args="$@"
+  target=$1
+else
+  # Use default args if no were given
+  case "$1" in
+  ios)
+    target="ipa"
+    args="--release --no-codesign"
+    ;;
+
+  android)
+    target="apk"
+    args="--release --target-platform android-arm64 --split-per-abi"
+    ;;
+
+  *)
+    echo "Unknown platform: '$1'"
+    exit 1
+    ;;
+  esac
+fi
+
+for dir in "$targetDir/perf-test-app-"*; do
+  app="$(basename "$dir")"
+  echo "Building $app"
+
+  # Support caching if this is running in CI.
+  if [[ -n ${CI+x} && $app == *-plain && -f $APP_PLAIN ]]; then
+    echo "Cached app exists: $APP_PLAIN - skipping build"
+    continue
+  fi
+
+  (
+    echo "::group::Flutter build $app"
+    cd $dir
+    flutter pub get
+    if [[ "$1" == "ios" && -f "ios/Podfile" ]]; then
+      cd ios
+      pod install --repo-update
+      cd -
+    fi
+    flutter build $target $args
+    echo '::endgroup::'
+  )
+
+  if [[ "$1" == "ios" ]]; then
+    flJob=${app//-/_}
+    (
+      echo "::group::Fastlane build $app"
+      cd $targetDir
+      fastlane "build_$flJob" --verbose
+      echo '::endgroup::'
+    )
+  fi
+done

metrics/fastlane/.gitignore

@@ -0,0 +1,5 @@
+test_output/
+README.md
+report.xml
+.env
+cobertura.xml

metrics/fastlane/Appfile

@@ -0,0 +1,6 @@
+itc_team_id("96157806") # App Store Connect Team ID
+team_id("97JCY7859U") # Developer Portal Team ID
+
+# For more information about the Appfile, see:
+#     https://docs.fastlane.tools/advanced/#appfile
+

metrics/fastlane/Fastfile

@@ -0,0 +1,54 @@
+default_platform(:ios)
+
+platform :ios do
+  desc "Build perf-test app without Sentry"
+  lane :build_perf_test_app_plain do
+    setup_ci
+
+    sync_code_signing(
+      type: "development",
+      readonly: true,
+      app_identifier: ["io.sentry.dart.perfTestAppPlain"]
+    )
+
+    build_app(
+      workspace: "perf-test-app-plain/ios/Runner.xcworkspace",
+      scheme: "Runner",
+      include_bitcode: false,
+      include_symbols: false,
+      export_method: "development",
+      export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id),
+      output_name: "test-app-plain.ipa",
+      skip_build_archive: true,
+      archive_path: "perf-test-app-plain/build/ios/archive/Runner.xcarchive",
+    )
+
+    delete_keychain(name: "fastlane_tmp_keychain") unless is_ci
+  end
+
+  desc "Build perf-test app with Sentry"
+  lane :build_perf_test_app_with_sentry do
+    setup_ci
+
+    sync_code_signing(
+      type: "development",
+      readonly: true,
+      app_identifier: ["io.sentry.dart.perfTestAppWithSentry"]
+    )
+
+    build_app(
+      workspace: "perf-test-app-with-sentry/ios/Runner.xcworkspace",
+      scheme: "Runner",
+      include_bitcode: false,
+      include_symbols: false,
+      export_method: "development",
+      export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id),
+      output_name: "test-app-sentry.ipa",
+      skip_build_archive: true,
+      archive_path: "perf-test-app-with-sentry/build/ios/archive/Runner.xcarchive",
+    )
+
+    delete_keychain(name: "fastlane_tmp_keychain") unless is_ci
+  end
+
+end

metrics/fastlane/Matchfile

@@ -0,0 +1,5 @@
+git_url("[email protected]:getsentry/codesigning.git")
+storage_mode("git")
+username("[email protected]") # Your Apple Developer Portal username
+
+# The docs are available on https://docs.fastlane.tools/actions/match

metrics/fastlane/Pluginfile

@@ -0,0 +1,5 @@
+# Autogenerated by fastlane
+#
+# Ensure this file is checked in to source control!
+
+gem 'fastlane-plugin-sentry'

metrics/flutter.properties

@@ -0,0 +1,2 @@
+version = 3.3.3
+repo = https://github.com/flutter/flutter

metrics/metrics-android.yml

@@ -0,0 +1,16 @@
+apps:
+  - name: io.sentry.dart.perf_test_app_plain
+    activity: MainActivity
+    path: perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk
+  - name: io.sentry.dart.perf_test_app_with_sentry
+    activity: MainActivity
+    path: perf-test-app-with-sentry/build/app/outputs/apk/release/app-arm64-v8a-release.apk
+
+startupTimeTest:
+  runs: 50
+  diffMin: 0
+  diffMax: 150
+
+binarySizeTest:
+  diffMin: 900 KiB
+  diffMax: 1100 KiB

metrics/metrics-ios.yml

@@ -0,0 +1,14 @@
+apps:
+  - name: io.sentry.dart.perfTestAppPlain
+    path: test-app-plain.ipa
+  - name: io.sentry.dart.perfTestAppWithSentry
+    path: test-app-sentry.ipa
+
+startupTimeTest:
+  runs: 50
+  diffMin: 0
+  diffMax: 150
+
+binarySizeTest:
+  diffMin: 900 KiB
+  diffMax: 1100 KiB

metrics/prepare.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+targetDir=$(
+  cd $(dirname $0)
+  pwd
+)
+[[ "$targetDir" != "" ]] || exit 1
+
+flutterCreate() {
+  name=${1//-/_}
+  dir=$targetDir/$1
+  rm -rf $dir
+  echo "::group::Flutter create $1"
+  flutter create --template=app --no-pub --org 'io.sentry.dart' --project-name $name "$dir"
+  echo '::endgroup::'
+}
+
+flutterCreate 'perf-test-app-plain'
+flutterCreate 'perf-test-app-with-sentry'
+
+echo '::group::Patch perf-test-app-with-sentry'
+pubspec="$targetDir/perf-test-app-with-sentry/pubspec.yaml"
+echo "Adding dependencies to $pubspec"
+cat <<EOF >>"$pubspec"
+
+dependency_overrides:
+  sentry:
+    path: ../../dart
+  sentry_flutter:
+    path: ../../flutter
+
+EOF
+
+patch -p0 "$targetDir/perf-test-app-with-sentry/lib/main.dart" <<'EOF'
+@@ -1,7 +1,12 @@
+ import 'package:flutter/material.dart';
++import 'package:sentry_flutter/sentry_flutter.dart';
+
+-void main() {
+-  runApp(const MyApp());
++Future<void> main() async {
++  await SentryFlutter.init(
++    (options) => options.dsn =
++        'https://[email protected]/5428562',
++    appRunner: () => runApp(const MyApp()),
++  );
+ }
+
+ class MyApp extends StatelessWidget {
+EOF
+echo '::endgroup::'