test(e2e): Switch from Appium to Maestro

.github/workflows/e2e.yml

@@ -14,6 +14,8 @@ concurrency:
 
 env:
   SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+  MAESTRO_VERSION: '1.39.0'
+  IOS_DEVICE: 'iPhone 14'
 
 jobs:
   diff_check:
@@ -168,12 +170,10 @@ jobs:
             rn-version: '0.76.0'
             runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: ios
             rn-version: '0.65.3'
             runs-on: macos-13
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: android
             runs-on: ubuntu-latest
         exclude:
@@ -308,12 +308,10 @@ jobs:
             rn-version: '0.76.0'
             runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: ios
             rn-version: '0.65.3'
             runs-on: macos-latest
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: android
             runs-on: ubuntu-latest
         exclude:
@@ -329,12 +327,18 @@ jobs:
           - rn-version: '0.76.0'
             platform: 'ios'
             rn-architecture: 'new'
-    env:
-      PLATFORM: ${{ matrix.platform }}
-      DEVICE: ${{ matrix.device }}
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install Maestro
+        uses: dniHze/maestro-test-action@bda8a93211c86d0a05b7a4597c5ad134566fbde4 # [email protected]
+        with:
+          version: ${{env.MAESTRO_VERSION}}
+
+      - name: Install iDB Companion
+        if: ${{ matrix.platform == 'ios' }}
+        run: brew tap facebook/fb && brew install facebook/fb/idb-companion
+
       - uses: ./.github/actions/disk-cleanup
         if: ${{ matrix.platform == 'android' }}
 
@@ -400,11 +404,10 @@ jobs:
             -timezone US/Pacific
           script: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
 
-      - uses: actions/cache@v4
+      - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3
         if: ${{ matrix.platform == 'ios' }}
         with:
-          path: test/e2e/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app
-          key: appium-webdriveragent-${{ hashFiles('test/e2e/yarn.lock') }}
+          model: ${{ env.IOS_DEVICE }}
 
       - name: Run tests on iOS
         if: ${{ matrix.platform == 'ios' }}
@@ -415,6 +418,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-logs
-          path: |
-            test/e2e/*.log
-            test/e2e/*.png
+          path: ./dev-packages/e2e-tests/maestro-logs

dev-packages/e2e-tests/.gitignore

@@ -2,3 +2,6 @@
 *.log
 *.app
 *.apk
+
+/react-native-versions
+/maestro-logs

dev-packages/e2e-tests/cli.mjs

@@ -64,8 +64,9 @@ const appRepoDir = `${e2eDir}/react-native-versions/${RNVersion}`;
 const appName = 'RnDiffApp';
 const appDir = `${appRepoDir}/${appName}`;
 const testAppName = `${appName}.${platform == 'ios' ? 'app' : 'apk'}`;
-const runtime = env.IOS_RUNTIME ? env.IOS_RUNTIME : 'latest';
-const device = env.IOS_DEVICE ? env.IOS_DEVICE : 'iPhone 15';
+const testApp = `${e2eDir}/${testAppName}`;
+const appId = platform === 'ios' ? 'org.reactjs.native.example.RnDiffApp' : 'com.rndiffapp';
+const sentryAuthToken = env.SENTRY_AUTH_TOKEN;
 
 // Build and publish the SDK - we only need to do this once in CI.
 // Locally, we may want to get updates from the latest build so do it on every app build.
@@ -112,9 +113,12 @@ if (actions.includes('create')) {
     env: Object.assign(env, { YARN_ENABLE_IMMUTABLE_INSTALLS: false }),
   });
 
-  console.log(`done`);
-
-  console.log(`done2`);
+  execSync(`yarn add [email protected]`, {
+    stdio: 'inherit',
+    cwd: appDir,
+    // yarn v3 run immutable install by default in CI
+    env: Object.assign(env, { YARN_ENABLE_IMMUTABLE_INSTALLS: false }),
+  });
 
   // Patch the app
   execSync(`patch --verbose --strip=0 --force --ignore-whitespace --fuzz 4 < ${patchScriptsDir}/rn.patch`, {
@@ -138,9 +142,11 @@ if (actions.includes('create')) {
       cwd: `${appDir}/ios`,
       env: env,
     });
-    console.log(`done3`);
 
     if (fs.existsSync(`${appDir}/Gemfile`)) {
+      // TMP Fix for https://github.com/CocoaPods/Xcodeproj/issues/989
+      fs.appendFileSync(`${appDir}/Gemfile`, "gem 'xcodeproj', '< 1.26.0'\n");
+
       execSync(`bundle install`, { stdio: 'inherit', cwd: appDir, env: env });
       execSync('bundle exec pod install --repo-update', { stdio: 'inherit', cwd: `${appDir}/ios`, env: env });
     } else {
@@ -185,7 +191,8 @@ if (actions.includes('build')) {
                   -workspace ${appName}.xcworkspace \
                   -configuration ${buildType} \
                   -scheme ${appName} \
-                  -destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
+                  -sdk 'iphonesimulator' \
+                  -destination 'generic/platform=iOS Simulator' \
                   ONLY_ACTIVE_ARCH=yes \
                   -derivedDataPath DerivedData \
                   build | tee xcodebuild.log | xcbeautify`,
@@ -202,123 +209,40 @@ if (actions.includes('build')) {
     appProduct = `${appDir}/android/app/build/outputs/apk/release/app-release.apk`;
   }
 
-  var testApp = `${e2eDir}/${testAppName}`;
   console.log(`Moving ${appProduct} to ${testApp}`);
   if (fs.existsSync(testApp)) fs.rmSync(testApp, { recursive: true });
   fs.renameSync(appProduct, testApp);
 }
 
 if (actions.includes('test')) {
-  if (
-    platform == 'ios' &&
-    !fs.existsSync(`${e2eDir}/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app`)
-  ) {
-    // Build iOS WebDriverAgent
-    execSync(
-      `set -o pipefail && xcodebuild \
-                  -project node_modules/appium-webdriveragent/WebDriverAgent.xcodeproj \
-                  -scheme WebDriverAgentRunner \
-                  -destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
-                  GCC_TREAT_WARNINGS_AS_ERRORS=0 \
-                  COMPILER_INDEX_STORE_ENABLE=NO \
-                  ONLY_ACTIVE_ARCH=yes \
-                  -derivedDataPath DerivedData \
-                  build | tee xcodebuild-agent.log | xcbeautify`,
-      { stdio: 'inherit', cwd: e2eDir, env: env },
-    );
-  }
-
-  // Start the appium server.
-  var processesToKill = {};
-  async function newProcess(name, process) {
-    await new Promise((resolve, reject) => {
-      process.on('error', e => {
-        console.error(`Failed to start process '${name}': ${e}`);
-        reject(e);
-      });
-      process.on('spawn', () => {
-        console.log(`Process '${name}' (${process.pid}) started`);
-        resolve();
-      });
-    });
-
-    processesToKill[name] = {
-      process: process,
-      complete: new Promise((resolve, _reject) => {
-        process.on('close', resolve);
-      }),
-    };
-  }
-  await newProcess(
-    'appium',
-    spawn('node_modules/.bin/appium', ['--log-timestamp', '--log-no-colors', '--log', `appium${platform}.log`], {
-      stdio: 'inherit',
-      cwd: e2eDir,
-      env: env,
-      shell: false,
-    }),
-  );
-
-  try {
-    await waitForAppium();
-
-    // Run e2e tests
-    const testEnv = env;
-    testEnv.PLATFORM = platform;
-    testEnv.APPIUM_APP = `./${testAppName}`;
-
-    if (platform == 'ios') {
-      testEnv.APPIUM_DERIVED_DATA = 'DerivedData';
-    } else if (platform == 'android') {
-      execSync(`adb devices -l`, { stdio: 'inherit', cwd: e2eDir, env: env });
-
-      execSync(`adb logcat -c`, { stdio: 'inherit', cwd: e2eDir, env: env });
-
-      var adbLogStream = fs.createWriteStream(`${e2eDir}/adb.log`);
-      const adbLogProcess = spawn('adb', ['logcat'], { cwd: e2eDir, env: env, shell: false });
-      adbLogProcess.stdout.pipe(adbLogStream);
-      adbLogProcess.stderr.pipe(adbLogStream);
-      adbLogProcess.on('close', () => adbLogStream.close());
-      await newProcess('adb logcat', adbLogProcess);
-    }
-
-    execSync(`yarn test:e2e:runner --verbose`, { stdio: 'inherit', cwd: e2eDir, env: testEnv });
-  } finally {
-    for (const [name, info] of Object.entries(processesToKill)) {
-      console.log(`Sending termination signal to process '${name}' (${info.process.pid})`);
-
-      // Send SIGTERM first to allow graceful shutdown.
-      info.process.kill(15);
-
-      // Also send SIGKILL after 10 seconds.
-      const killTimeout = setTimeout(() => process.kill(9), '10000');
-
-      // Wait for the process to exit (either via SIGTERM or SIGKILL).
-      const code = await info.complete;
-
-      // Successfully exited now, no need to kill (if it hasn't run yet).
-      clearTimeout(killTimeout);
-
-      console.log(`Process '${name}' (${info.process.pid}) exited with code ${code}`);
+  // Run e2e tests
+  if (platform == 'ios') {
+    try {
+      execSync('xcrun simctl list devices | grep -q "(Booted)"');
+    } catch (error) {
+      throw new Error('No simulator is currently booted. Please boot a simulator before running this script.');
     }
-  }
-}
 
-async function waitForAppium() {
-  console.log('Waiting for Appium server to start...');
-  for (let i = 0; i < 60; i++) {
+    execSync(`xcrun simctl install booted ${testApp}`);
+  } else if (platform == 'android') {
     try {
-      await fetch('http://127.0.0.1:4723/sessions', { method: 'HEAD' });
-      console.log('Appium server started');
-      return;
+      execSync('adb devices | grep -q "emulator"');
     } catch (error) {
-      console.log(`Appium server hasn't started yet (${error})...`);
-      await sleep(1000);
+      throw new Error('No Android emulator is currently running. Please start an emulator before running this script.');
     }
+
+    execSync(`adb install -r -d ${testApp}`);
   }
-  throw new Error('Appium server failed to start');
-}
 
-async function sleep(millis) {
-  return new Promise(resolve => setTimeout(resolve, millis));
+  execSync(
+    `maestro test maestro \
+      --env=APP_ID="${appId}" \
+      --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
+      --debug-output maestro-logs \
+      --flatten-debug-output`,
+    {
+      stdio: 'inherit',
+      cwd: e2eDir,
+    },
+  );
 }

dev-packages/e2e-tests/maestro/captureException.yml

@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Capture Exception"
+- runFlow: utils/assertEventIdVisible.yml

dev-packages/e2e-tests/maestro/captureMessage.yml

@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Capture Message"
+- runFlow: utils/assertEventIdVisible.yml

dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml

@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Unhandled Promise Rejection"
+- runFlow: utils/assertEventIdVisible.yml

dev-packages/e2e-tests/maestro/close.yml

@@ -0,0 +1,7 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Close"
+
+- assertNotVisible:
+    id: "eventId"

dev-packages/e2e-tests/maestro/crash.yml

@@ -0,0 +1,7 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Crash"
+
+- launchApp
+- assertVisible: "E2E Tests Ready"

dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml

@@ -0,0 +1,10 @@
+appId: ${APP_ID}
+---
+- extendedWaitUntil:
+    visible:
+      id: "eventId"
+    timeout: 600_000 # 10 minutes
+
+- copyTextFrom:
+    id: "eventId"
+- assertTrue: ${maestro.copiedText}

dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml

@@ -0,0 +1,10 @@
+appId: ${APP_ID}
+---
+- launchApp:
+    clearState: true
+    arguments:
+      sentryAuthToken: ${SENTRY_AUTH_TOKEN}
+
+- extendedWaitUntil:
+    visible: "E2E Tests Ready"
+    timeout: 120_000 # 2 minutes

dev-packages/e2e-tests/package.json

@@ -6,7 +6,8 @@
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc --project tsconfig.build.json",
-    "test:e2e:runner": "NODE_OPTIONS=--experimental-vm-modules jest"
+    "test:ios": "./cli.mjs ios --create --build --test",
+    "test:android": "./cli.mjs android --create --build --test"
   },
   "license": "MIT",
   "devDependencies": {
@@ -23,12 +24,17 @@
     "jest": "^29.7.0",
     "react": "18.3.1",
     "react-native": "0.75.4",
+    "react-native-launch-arguments": "^4.0.2",
     "typescript": "4.9.5",
     "webdriverio": "^8.27.0"
   },
   "dependencies": {
     "minimist": "1.2.8",
+    "p-retry": "^6.2.0",
     "semver": "7.6.3",
     "xcode": "3.0.1"
+  },
+  "peerDependencies": {
+    "react-native-launch-arguments": "^4.0.2"
   }
 }