Merge pull request #1604 from bugsnag/tms/app-hangs-bs

Run App Hang test scenarios with Appium 1

.buildkite/pipeline.full.yml

@@ -24,6 +24,8 @@ steps:
         run: cocoa-maze-runner-bitbar
         service-ports: true
         command:
+          # PLAT-11155: App hang scenarios run on BrowserStack
+          - "--exclude=features/app_hangs.feature"
           - "--exclude=features/[e-z].*.feature$"
           - "--app=@/app/build/ipa_url_bb.txt"
           - "--farm=bb"
@@ -79,6 +81,8 @@ steps:
           - "--no-tunnel"
           - "--aws-public-ip"
           - "--fail-fast"
+          # PLAT-11155: App hang scenarios run on BrowserStack
+          - "--exclude=features/app_hangs.feature"
           - "--exclude=features/[e-z].*.feature$"
     concurrency: 25
     concurrency_group: 'bitbar'
@@ -139,6 +143,8 @@ steps:
           - "--no-tunnel"
           - "--aws-public-ip"
           - "--fail-fast"
+          # PLAT-11155: App hang scenarios run on BrowserStack
+          - "--exclude=features/app_hangs.feature"
           - "--exclude=features/[e-z].*.feature$"
     concurrency: 25
     concurrency_group: 'bitbar'
@@ -199,6 +205,8 @@ steps:
           - "--no-tunnel"
           - "--aws-public-ip"
           - "--fail-fast"
+          # PLAT-11155: App hang scenarios run on BrowserStack
+          - "--exclude=features/app_hangs.feature"
           - "--exclude=features/[e-z].*.feature$"
     concurrency: 25
     concurrency_group: 'bitbar'
@@ -241,6 +249,118 @@ steps:
   #
   # BrowserStack
   #
+  # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now
+  - label: ':browserstack: iOS 15 app hang tests'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 10
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        command:
+          - "--app=@build/ipa_url_bs.txt"
+          - "--farm=bs"
+          - "--device=IOS_15"
+          - "--appium-version=1.21.0"
+          - "features/app_hangs.feature"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now
+  - label: ':browserstack: iOS 14 app hang tests'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 10
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        command:
+          - "--app=@build/ipa_url_bs.txt"
+          - "--farm=bs"
+          - "--device=IOS_14"
+          - "--appium-version=1.21.0"
+          - "features/app_hangs.feature"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now
+  - label: ':browserstack: iOS 13 app hang tests'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 10
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        command:
+          - "--app=@build/ipa_url_bs.txt"
+          - "--farm=bs"
+          - "--device=IOS_13"
+          - "--appium-version=1.21.0"
+          - "features/app_hangs.feature"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
+  # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now
+  - label: ':browserstack: iOS 12 app hang tests'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 10
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        command:
+          - "--app=@build/ipa_url_bs.txt"
+          - "--farm=bs"
+          - "--device=IOS_12"
+          - "--appium-version=1.21.0"
+          - "features/app_hangs.feature"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
   - label: ':browserstack: iOS 11 E2E tests batch 1'
     skip: "https://smartbear.atlassian.net/browse/PLAT-11154"
     depends_on:

.buildkite/pipeline.yml

@@ -316,6 +316,8 @@ steps:
           - "--no-tunnel"
           - "--aws-public-ip"
           - "--fail-fast"
+          # PLAT-11155: App hang scenarios run on BrowserStack
+          - "--exclude=features/app_hangs.feature"
           - "--exclude=features/[e-z].*.feature$"
     concurrency: 25
     concurrency_group: 'bitbar'
@@ -355,6 +357,34 @@ steps:
         - exit_status: -1  # Agent was lost
           limit: 2
 
+  # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now
+  - label: ':browserstack: iOS 16 app hang tests'
+    depends_on:
+      - cocoa_fixture
+    timeout_in_minutes: 10
+    agents:
+      queue: opensource
+    plugins:
+      artifacts#v1.5.0:
+        download: "features/fixtures/ios/output/ipa_url_bs.txt"
+        upload: "maze_output/failed/**/*"
+      docker-compose#v3.7.0:
+        pull: cocoa-maze-runner
+        run: cocoa-maze-runner
+        command:
+          - "--app=@build/ipa_url_bs.txt"
+          - "--farm=bs"
+          - "--device=IOS_16"
+          - "--appium-version=1.21.0"
+          - "features/app_hangs.feature"
+    concurrency: 5
+    concurrency_group: 'browserstack-app'
+    concurrency_method: eager
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
+
   - label: ':bitbar: iOS 15 barebone tests'
     depends_on:
       - cocoa_fixture

.gitignore

@@ -32,3 +32,4 @@ Package.resolved
 /docs/
 /infer-out
 /oclint.json
+bb.ready

features/app_hangs.feature

@@ -1,4 +1,3 @@
-@skip # https://smartbear.atlassian.net/browse/PLAT-11155
 Feature: App hangs
 
   Background: