ref: split benchmarking into separate target/action/sauce-config

.github/workflows/benchmarking.yml

@@ -0,0 +1,68 @@
+name: Benchmarking
+on:
+  schedule:
+    - cron: '0 0 * * *' # every night at midnight UTC
+  push:
+    branches:
+      - master
+
+  pull_request:
+    paths:
+      - 'Samples/iOS-Swift/iOS-Swift/**'
+      - 'Samples/iOS-Swift/PerformanceBenchmarks/**'
+      - '.github/workflows/benchmarking.yml'
+      - '.sauce/benchmarking-config.yml'
+      - 'fastlane/**'
+
+jobs:
+  build-benchmark-test-target:
+    name: Build UITests with Xcode ${{matrix.xcode}}
+    runs-on: macos-11
+    steps:
+      - uses: actions/checkout@v3
+      - run: ./scripts/ci-select-xcode.sh
+      - run: git apply ./scripts/set-device-tests-environment.patch
+      - run: fastlane build_ios_benchmark_test
+        env:
+          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
+          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
+          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
+          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
+          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
+          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
+      - name: Archiving DerivedData
+        uses: actions/upload-artifact@v3
+        with:
+          name: DerivedData-Xcode-${{matrix.xcode}}
+          path: |
+            **/Debug-iphoneos/iOS-Swift.app
+            **/Debug-iphoneos/PerformanceBenchmarking-Runner.app
+
+  run-ui-tests-with-sauce:
+    name: Run Benchmarks on ${{ matrix.suite }} in Sauce Labs
+    runs-on: ubuntu-latest
+    needs: build-benchmark-test-target
+    strategy:      
+      fail-fast: false
+      matrix:
+        include:
+          - xcode: "13.2"
+            suite: "High-end device"
+          - xcode: "13.2"
+            suite: "Mid-tier device"
+          - xcode: "13.2"
+            suite: "Low-end device"
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: DerivedData-Xcode-${{matrix.xcode}}
+      - run: npm install -g [email protected]
+      - name: Run Benchmarks in SauceLab
+        env:
+          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        # As Sauce Labs is a bit flaky we retry 5 times
+        run: for i in {1..5}; do saucectl run --select-suite ${{ matrix.suite }} && break ; done

.sauce/benchmarking-config.yml

@@ -0,0 +1,28 @@
+apiVersion: v1alpha
+kind: xcuitest
+sauce:
+  region: us-west-1
+  concurrency: 2
+
+defaults:
+  timeout: 20m
+
+xcuitest:
+  app: ./DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
+  testApp: ./DerivedData/Build/Products/Debug-iphoneos/PerformanceBenchmarking-Runner.app
+
+suites:
+  - name: "High-end device"
+    devices:
+      - name: "iPad Pro 12.9 2021"
+        platformVersion: "14.7"
+
+  - name: "Mid-tier device"
+    devices:
+      - name: "iPhone 7 Plus"
+        platformVersion: "14.8"
+
+  - name: "Low-end device"
+    devices:
+      - name: "iPhone 6S"
+        platformVersion: "13.5"

Samples/iOS-Swift/PerformanceBenchmarks/PerformanceBenchmarks-Info.plist

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>$(DEVELOPMENT_LANGUAGE)</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+</dict>
+</plist>

Samples/iOS-Swift/iOS-SwiftUITests/SDKPerformanceBenchmarkTests.swift → Samples/iOS-Swift/PerformanceBenchmarks/SDKPerformanceBenchmarkTests.swift

@@ -2,15 +2,14 @@ import XCTest
 
 class SDKPerformanceBenchmarkTests: XCTestCase {
     func testCPUBenchmark() throws {
-        let allowedIOSVersion = "15"
-        try XCTSkipUnless(UIDevice.current.systemVersion.components(separatedBy: ".").first ?? "" == allowedIOSVersion, "Only run benchmarks on iOS \(allowedIOSVersion).")
         var isSimulator: Bool
         #if targetEnvironment(simulator)
         isSimulator = true
         #else
         isSimulator = false
         #endif
         try XCTSkipIf(isSimulator, "Only run benchmarks on real devices, not in simulators.")
+
         var results = [Double]()
         for _ in 0..<5 {
             let app = XCUIApplication()