Decompose Android E2E tasks, retry all flaky steps

invertase · Aug 14, 2020 · f1b9af9 · f1b9af9
1 parent bd1661c
commit f1b9af9
Show file tree

Hide file tree

Showing 3 changed files with 83 additions and 43 deletions.
diff --git a/.github/workflows/tests_e2e.yml b/.github/workflows/tests_e2e.yml
@@ -47,72 +47,112 @@ jobs:
           export DETOX_DISABLE_POSTINSTALL=1
           yarn --no-audit --prefer-offline
 
-      # OpenJDK1.8 is the default in GitHub macOS 10.15 runner, setup should not be required
-      # TODO verify that it is installed
-      - name: JDK check
-        run: java -fullversion
-
-      # - name: Setup JDK8
-      #   uses: actions/setup-java@v1
-      #   with:
-      #     java-version: '8'
-      #     architecture: 'x64'
+      - name: Verify JDK 1.8
+        # OpenJDK1.8 is the default in GitHub macOS 10.15 runner, setup is not required
+        # Run a check that exits with error unless it is 1.8 version to future-proof against unexpected upgrades
+        run: java -fullversion 2>&1 | grep '1.8'
+        shell: bash
 
       - name: Build Android App
         run: cd tests/android && ./gradlew assembleDebug assembleAndroidTest -DtestBuildType=debug -Dorg.gradle.daemon=false
 
-      - name: Download Emulator Image
+      - name: Pre-fetch Javascript bundle
+        # Prebuild the bundle so that's fast when the app starts.
         run: |
-          echo "y" | $ANDROID_HOME/tools/bin/sdkmanager --install "system-images;android-28;google_apis;x86_64"
-          echo "no" | $ANDROID_HOME/tools/bin/avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-28;google_apis;x86_64' -g google_apis
-          $ANDROID_HOME/emulator/emulator -list-avds
+          nohup yarn run tests:packager:jet &
+          printf 'Waiting for packager to come online'
+          until curl --output /dev/null --silent --head --fail http://localhost:8081/status; do
+            printf '.'
+            sleep 2
+          done
+          echo "Packager is online!"
+          curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=android&dev=true&minify=false&inlineSourceMap=true"
 
+      - name: Download Emulator Image
+        # This can fail on network request, wrap with retry
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          retry_wait_seconds: 60
+          max_attempts: 3
+          command: echo "y" | $ANDROID_HOME/tools/bin/sdkmanager --install "system-images;android-28;google_apis;x86_64"
+
+      - name: Create Emulator
+        run: echo "no" | $ANDROID_HOME/tools/bin/avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-28;google_apis;x86_64' -g google_apis
+
+      # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes
       - name: Start Android Emulator
-        timeout-minutes: 15
+        id: emu1
+        timeout-minutes: 5
         continue-on-error: true
         run: |
           echo "Starting emulator"
           nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
           $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Start Android Emulator Retry 1
+        id: emu2
+        if: steps.emu1.outcome=='failure'
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          echo "Starting emulator, second attempt"
           $ANDROID_HOME/platform-tools/adb devices
-          $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
-          $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
-          echo "Emulator started"
+          sudo killall -9 qemu-system-x86_64-headless || true
+          sleep 2
+          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
+          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Start Android Emulator Retry 2
+        id: emu3
+        if: steps.emu2.outcome=='failure'
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          echo "Starting emulator, third attempt"
+          $ANDROID_HOME/platform-tools/adb devices
+          sudo killall -9 qemu-system-x86_64-headless || true
+          sleep 2
+          nohup $ANDROID_HOME/emulator/emulator -avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 &
+          $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
+
+      - name: Emulator Status
+        if: always()
+        run: |
+          if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then
+             echo "Emulator Started"
+          else
+             exit 1
+          fi
 
       - name: Detox Test
-        timeout-minutes: 40
-        # Prebuild the bundle so that's fast when the app starts.
         # Detox uses Espresso to choreograph steps in reaction to UI events, so we need to send a stream of taps.
-        # But if you send them before the app is ready, there is a chance react-native comes alive with an ACTION_UP and no ACTION_DOWN, which hangs everything.
-        # So we wait 180 seconds, which seems to be enough for the app to come up.
+        timeout-minutes: 40
         run: |
-          nohup yarn run tests:packager:jet &
-          printf 'Waiting for packager to come online'
-          until curl --output /dev/null --silent --head --fail http://localhost:8081/status; do
-            printf '.'
-            sleep 2
-          done
-          echo "Packager is online!"
-          curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=android&dev=true&minify=false&inlineSourceMap=true"
-          nohup sh -c "sleep 180 && until false; do $ANDROID_HOME/platform-tools/adb shell input tap 100 800; sleep 0.1; done" &
+          $ANDROID_HOME/platform-tools/adb devices
+          $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
+          $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
+          $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
+          nohup sh -c "until false; do $ANDROID_HOME/platform-tools/adb shell input tap 100 800; sleep 0.2; done" &
           nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" &
-          cd tests
-          ./node_modules/.bin/nyc ./node_modules/.bin/detox test --loglevel trace --configuration android.emu.debug --cleanup
+          yarn tests:android:test-cover --cleanup
         shell: bash
 
       - name: Submit Coverage
-        run: |
-          ./node_modules/.bin/codecov
-        shell: bash
+        # This can fail on timeouts etc, wrap with retry
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          retry_wait_seconds: 60
+          max_attempts: 3
+          command: ./node_modules/.bin/codecov
 
       - name: Compress Emulator Log
         if: always()
-        run: |
-            gzip -9 adb-log.txt
+        run: gzip -9 adb-log.txt
         shell: bash
 
-      - name: Upload Logs
+      - name: Upload Emulator Log
         uses: actions/upload-artifact@v2
         if: always()
         with:

diff --git a/package.json b/package.json
@@ -25,7 +25,7 @@
     "tests:android:test": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug",
     "tests:android:test:debug": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug --inspect",
     "tests:android:test-reuse": "cd tests && ./node_modules/.bin/detox test --configuration android.emu.debug --reuse",
-    "tests:android:test-cover": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --configuration android.emu.debug",
+    "tests:android:test-cover": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --loglevel trace --configuration android.emu.debug",
     "tests:android:test-cover-reuse": "cd tests && ./node_modules/.bin/nyc ./node_modules/.bin/detox test --configuration android.emu.debug --reuse",
     "tests:ios:build": "cd tests && ./node_modules/.bin/detox build --configuration ios.sim.debug",
     "tests:ios:build-release": "cd tests && ./node_modules/.bin/detox build --configuration ios.sim.release",
@@ -99,4 +99,4 @@
       "**/superstruct/**"
     ]
   }
-}
+}
diff --git a/tests/package.json b/tests/package.json
@@ -89,4 +89,4 @@
       }
     }
   }
-}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -89,4 +89,4 @@ @@
           }
         }
       }
-    }
+    }