From 800ce32309606872352ce9934dbd6f20fa04abe7 Mon Sep 17 00:00:00 2001 From: Steve Kirkland-Walton Date: Mon, 6 Nov 2023 18:57:19 +0000 Subject: [PATCH 1/3] Skip app_hangs.feature on BitBar (Appium 2) --- .buildkite/pipeline.yml | 2 ++ .gitignore | 1 + features/app_hangs.feature | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 161921db4..11439a3f4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -305,6 +305,8 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" + # App hang scenarios run on BrowserStack, see PLAT-11155 + - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" concurrency: 25 concurrency_group: 'bitbar' diff --git a/.gitignore b/.gitignore index 001e898bd..da7e5f018 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ Package.resolved /docs/ /infer-out /oclint.json +bb.ready diff --git a/features/app_hangs.feature b/features/app_hangs.feature index a2c91dbca..8f31de417 100644 --- a/features/app_hangs.feature +++ b/features/app_hangs.feature @@ -1,4 +1,3 @@ -@skip # https://smartbear.atlassian.net/browse/PLAT-11155 Feature: App hangs Background: From 7a429153b6c3bdd2f98a5bc80feba163be365921 Mon Sep 17 00:00:00 2001 From: Steve Kirkland-Walton Date: Mon, 6 Nov 2023 19:34:25 +0000 Subject: [PATCH 2/3] Run app hang tests on Appium 1 (BrowserStack) [full ci] --- .buildkite/pipeline.full.yml | 120 +++++++++++++++++++++++++++++++++++ .buildkite/pipeline.yml | 30 ++++++++- 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.full.yml b/.buildkite/pipeline.full.yml index 52ddff668..37d6e60f4 100644 --- a/.buildkite/pipeline.full.yml +++ b/.buildkite/pipeline.full.yml @@ -24,6 +24,8 @@ steps: run: cocoa-maze-runner-bitbar service-ports: true command: + # PLAT-11155: App hang scenarios run on BrowserStack + - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" - "--app=@/app/build/ipa_url_bb.txt" - "--farm=bb" @@ -79,6 +81,8 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" + # PLAT-11155: App hang scenarios run on BrowserStack + - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" concurrency: 25 concurrency_group: 'bitbar' @@ -139,6 +143,8 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" + # PLAT-11155: App hang scenarios run on BrowserStack + - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" concurrency: 25 concurrency_group: 'bitbar' @@ -199,6 +205,8 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" + # PLAT-11155: App hang scenarios run on BrowserStack + - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" concurrency: 25 concurrency_group: 'bitbar' @@ -241,6 +249,118 @@ steps: # # BrowserStack # + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':browserstack: iOS 15 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + command: + - "--app=@build/ipa_url_bs.txt" + - "--farm=bs" + - "--device=IOS_15" + - "--appium-version=1.22.0" + - "features/app_hangs.feature" + concurrency: 5 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':browserstack: iOS 14 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + command: + - "--app=@build/ipa_url_bs.txt" + - "--farm=bs" + - "--device=IOS_14" + - "--appium-version=1.22.0" + - "features/app_hangs.feature" + concurrency: 5 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':browserstack: iOS 13 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + command: + - "--app=@build/ipa_url_bs.txt" + - "--farm=bs" + - "--device=IOS_13" + - "--appium-version=1.22.0" + - "features/app_hangs.feature" + concurrency: 5 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':browserstack: iOS 12 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + command: + - "--app=@build/ipa_url_bs.txt" + - "--farm=bs" + - "--device=IOS_12" + - "--appium-version=1.22.0" + - "features/app_hangs.feature" + concurrency: 5 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - label: ':browserstack: iOS 11 E2E tests batch 1' skip: "https://smartbear.atlassian.net/browse/PLAT-11154" depends_on: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 11439a3f4..93faa1f1c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -305,7 +305,7 @@ steps: - "--no-tunnel" - "--aws-public-ip" - "--fail-fast" - # App hang scenarios run on BrowserStack, see PLAT-11155 + # PLAT-11155: App hang scenarios run on BrowserStack - "--exclude=features/app_hangs.feature" - "--exclude=features/[e-z].*.feature$" concurrency: 25 @@ -346,6 +346,34 @@ steps: - exit_status: -1 # Agent was lost limit: 2 + # PLAT-11155: App hang tests run on BrowserStack (Appium 1.x) for now + - label: ':browserstack: iOS 16 app hang tests' + depends_on: + - cocoa_fixture + timeout_in_minutes: 10 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + command: + - "--app=@build/ipa_url_bs.txt" + - "--farm=bs" + - "--device=IOS_16" + - "--appium-version=1.22.0" + - "features/app_hangs.feature" + concurrency: 5 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - label: ':bitbar: iOS 15 barebone tests' depends_on: - cocoa_fixture From 6c72cd90a45870c9db3f6fbf38771fa824bc7203 Mon Sep 17 00:00:00 2001 From: Steve Kirkland-Walton Date: Tue, 7 Nov 2023 09:44:31 +0000 Subject: [PATCH 3/3] Use Appium 1.21.0, not 1.22.0 to prevent app hang scenario problems [full ci] --- .buildkite/pipeline.full.yml | 8 ++++---- .buildkite/pipeline.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.full.yml b/.buildkite/pipeline.full.yml index 37d6e60f4..d4eea6b16 100644 --- a/.buildkite/pipeline.full.yml +++ b/.buildkite/pipeline.full.yml @@ -267,7 +267,7 @@ steps: - "--app=@build/ipa_url_bs.txt" - "--farm=bs" - "--device=IOS_15" - - "--appium-version=1.22.0" + - "--appium-version=1.21.0" - "features/app_hangs.feature" concurrency: 5 concurrency_group: 'browserstack-app' @@ -295,7 +295,7 @@ steps: - "--app=@build/ipa_url_bs.txt" - "--farm=bs" - "--device=IOS_14" - - "--appium-version=1.22.0" + - "--appium-version=1.21.0" - "features/app_hangs.feature" concurrency: 5 concurrency_group: 'browserstack-app' @@ -323,7 +323,7 @@ steps: - "--app=@build/ipa_url_bs.txt" - "--farm=bs" - "--device=IOS_13" - - "--appium-version=1.22.0" + - "--appium-version=1.21.0" - "features/app_hangs.feature" concurrency: 5 concurrency_group: 'browserstack-app' @@ -351,7 +351,7 @@ steps: - "--app=@build/ipa_url_bs.txt" - "--farm=bs" - "--device=IOS_12" - - "--appium-version=1.22.0" + - "--appium-version=1.21.0" - "features/app_hangs.feature" concurrency: 5 concurrency_group: 'browserstack-app' diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 93faa1f1c..21708f4c4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -364,7 +364,7 @@ steps: - "--app=@build/ipa_url_bs.txt" - "--farm=bs" - "--device=IOS_16" - - "--appium-version=1.22.0" + - "--appium-version=1.21.0" - "features/app_hangs.feature" concurrency: 5 concurrency_group: 'browserstack-app'