diff --git a/.buildkite/browser-pipeline.full.yml b/.buildkite/browser-pipeline.full.yml index 577b55fc8..967c31e4e 100644 --- a/.buildkite/browser-pipeline.full.yml +++ b/.buildkite/browser-pipeline.full.yml @@ -29,6 +29,7 @@ steps: run: browser-maze-runner-bs use-aliases: true command: + - --https - --farm=bs - --browser={{ matrix }} artifacts#v1.5.0: diff --git a/.buildkite/browser-pipeline.yml b/.buildkite/browser-pipeline.yml index 4667222fe..a5f33f6f9 100644 --- a/.buildkite/browser-pipeline.yml +++ b/.buildkite/browser-pipeline.yml @@ -46,6 +46,7 @@ steps: use-aliases: true service-ports: true command: + - --https - --farm=bb - --browser={{ matrix.browser }}_{{ matrix.version }} - --appium-version=1.22 @@ -78,6 +79,7 @@ steps: run: browser-maze-runner-legacy use-aliases: true command: + - --https - --farm=bs - --browser={{ matrix.browser }}_{{ matrix.version }} artifacts#v1.5.0: @@ -97,5 +99,31 @@ steps: adjustments: - with: { browser: firefox, version: 60 } - with: { browser: edge, version: 80 } - - with: { browser: safari, version: 11 } - with: { browser: edge, version: latest } + + # BrowserStack non-https + - label: ":browserstack: :{{ matrix.browser }}: {{ matrix.version }} Browser non-https tests${EXTRA_STEP_LABEL}" + depends_on: "browser-maze-runner-legacy-${USE_CDN_BUILD}" + timeout_in_minutes: 30 + plugins: + docker-compose#v4.12.0: + pull: browser-maze-runner-legacy + run: browser-maze-runner-legacy + use-aliases: true + command: + - --farm=bs + - --browser={{ matrix.browser }}_{{ matrix.version }} + artifacts#v1.5.0: + upload: + - "./test/browser/maze_output/failed/**/*" + test-collector#v1.10.2: + files: "reports/TEST-*.xml" + format: "junit" + branch: "^main|next$$" + api-token-env-name: "BROWSER_PERFORMANCE_BUILDKITE_ANALYTICS_TOKEN" + concurrency: 2 + concurrency_group: "browserstack" + matrix: + setup: + browser: [safari] + version: [11] \ No newline at end of file diff --git a/test/browser/Gemfile.lock b/test/browser/Gemfile.lock deleted file mode 100644 index 113de12f7..000000000 --- a/test/browser/Gemfile.lock +++ /dev/null @@ -1,128 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - appium_lib (12.0.1) - appium_lib_core (~> 5.0) - nokogiri (~> 1.8, >= 1.8.1) - tomlrb (>= 1.1, < 3.0) - appium_lib_core (5.4.0) - faye-websocket (~> 0.11.0) - selenium-webdriver (~> 4.2, < 4.6) - bugsnag (6.27.1) - concurrent-ruby (~> 1.0) - bugsnag-maze-runner (9.13.0) - appium_lib (~> 12.0.0) - appium_lib_core (~> 5.4.0) - bugsnag (~> 6.24) - cucumber (~> 7.1) - cucumber-expressions (~> 6.0.0) - curb (~> 1.0.5) - dogstatsd-ruby (~> 5.5.0) - json_schemer (~> 0.2.24) - optimist (~> 3.0.1) - os (~> 1.0.0) - rack (~> 2.2) - rake (~> 12.3.3) - rubyzip (~> 2.3.2) - selenium-webdriver (~> 4.0) - test-unit (~> 3.5.2) - webrick (~> 1.7.0) - builder (3.3.0) - childprocess (4.1.0) - concurrent-ruby (1.3.4) - cucumber (7.1.0) - builder (~> 3.2, >= 3.2.4) - cucumber-core (~> 10.1, >= 10.1.0) - cucumber-create-meta (~> 6.0, >= 6.0.1) - cucumber-cucumber-expressions (~> 14.0, >= 14.0.0) - cucumber-gherkin (~> 22.0, >= 22.0.0) - cucumber-html-formatter (~> 17.0, >= 17.0.0) - cucumber-messages (~> 17.1, >= 17.1.1) - cucumber-wire (~> 6.2, >= 6.2.0) - diff-lcs (~> 1.4, >= 1.4.4) - mime-types (~> 3.3, >= 3.3.1) - multi_test (~> 0.1, >= 0.1.2) - sys-uname (~> 1.2, >= 1.2.2) - cucumber-core (10.1.1) - cucumber-gherkin (~> 22.0, >= 22.0.0) - cucumber-messages (~> 17.1, >= 17.1.1) - cucumber-tag-expressions (~> 4.1, >= 4.1.0) - cucumber-create-meta (6.0.4) - cucumber-messages (~> 17.1, >= 17.1.1) - sys-uname (~> 1.2, >= 1.2.2) - cucumber-cucumber-expressions (14.0.0) - cucumber-expressions (6.0.1) - cucumber-gherkin (22.0.0) - cucumber-messages (~> 17.1, >= 17.1.1) - cucumber-html-formatter (17.0.0) - cucumber-messages (~> 17.1, >= 17.1.0) - cucumber-messages (17.1.1) - cucumber-tag-expressions (4.1.0) - cucumber-wire (6.2.1) - cucumber-core (~> 10.1, >= 10.1.0) - cucumber-cucumber-expressions (~> 14.0, >= 14.0.0) - curb (1.0.5) - diff-lcs (1.5.1) - dogstatsd-ruby (5.5.0) - ecma-re-validator (0.4.0) - regexp_parser (~> 2.2) - eventmachine (1.2.7) - faye-websocket (0.11.3) - eventmachine (>= 0.12.0) - websocket-driver (>= 0.5.1) - ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86_64-darwin) - hana (1.3.7) - json_schemer (0.2.25) - ecma-re-validator (~> 0.3) - hana (~> 1.3) - regexp_parser (~> 2.0) - simpleidn (~> 0.2) - uri_template (~> 0.7) - mime-types (3.5.2) - mime-types-data (~> 3.2015) - mime-types-data (3.2024.0806) - multi_test (0.1.2) - nokogiri (1.16.7-arm64-darwin) - racc (~> 1.4) - nokogiri (1.16.7-x86_64-darwin) - racc (~> 1.4) - optimist (3.0.1) - os (1.0.1) - power_assert (2.0.3) - racc (1.8.1) - rack (2.2.9) - rake (12.3.3) - regexp_parser (2.9.2) - rexml (3.3.5) - strscan - rubyzip (2.3.2) - selenium-webdriver (4.5.0) - childprocess (>= 0.5, < 5.0) - rexml (~> 3.2, >= 3.2.5) - rubyzip (>= 1.2.2, < 3.0) - websocket (~> 1.0) - simpleidn (0.2.3) - strscan (3.1.0) - sys-uname (1.3.0) - ffi (~> 1.1) - test-unit (3.5.9) - power_assert - tomlrb (2.0.3) - uri_template (0.7.0) - webrick (1.7.0) - websocket (1.2.11) - websocket-driver (0.7.6) - websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.5) - -PLATFORMS - arm64-darwin-21 - arm64-darwin-23 - x86_64-darwin-19 - -DEPENDENCIES - bugsnag-maze-runner (~> 9.0) - -BUNDLED WITH - 2.4.8 diff --git a/test/browser/TESTING.md b/test/browser/TESTING.md index 08beec343..8567aca51 100644 --- a/test/browser/TESTING.md +++ b/test/browser/TESTING.md @@ -11,13 +11,13 @@ brew install chromedriver To run all the tests, run the following in `test/browser`: ```sh -bundle exec maze-runner --farm=local --browser=chrome +bundle exec maze-runner --https --farm=local --browser=chrome ``` Or to run a single feature file: ```sh -bundle exec maze-runner --farm=local --browser=chrome features/device.feature +bundle exec maze-runner --https --farm=local --browser=chrome features/device.feature ``` ## Running the end-to-end tests with BrowserStack @@ -34,11 +34,11 @@ The browsers available to test on are the keys in [`browsers.yml`](https://githu To run all the tests, run the following in `test/browser`: ```sh -bundle exec maze-runner --farm=bs --browser=chrome_latest +bundle exec maze-runner --https --farm=bs --browser=chrome_latest ``` Or to run a single feature file: ```sh -bundle exec maze-runner --farm=bs --browser=chrome_latest features/manual-spans.feature +bundle exec maze-runner --https --farm=bs --browser=chrome_latest features/manual-spans.feature ``` diff --git a/test/browser/features/angular.feature b/test/browser/features/angular.feature index 2973a5cad..3e13a291c 100644 --- a/test/browser/features/angular.feature +++ b/test/browser/features/angular.feature @@ -5,7 +5,7 @@ Feature: Angular Scenario: Angular route change spans are automatically instrumented - Given I navigate to the test URL "/angular/dist" + Given I navigate to the test URL "/docs/angular/dist" And the element "customers" is present And I click the element "customers" When I wait to receive 1 trace @@ -25,7 +25,7 @@ Feature: Angular | bugsnag.browser.page.route_change.trigger | stringValue | imperative | Scenario: Route with a custom URL matcher - Given I navigate to the test URL "/angular/dist" + Given I navigate to the test URL "/docs/angular/dist" And the element "customMatching" is present And I click the element "customMatching" When I wait to receive 1 trace diff --git a/test/browser/features/custom-attributes.feature b/test/browser/features/custom-attributes.feature index e027661bb..55a8e1cd9 100644 --- a/test/browser/features/custom-attributes.feature +++ b/test/browser/features/custom-attributes.feature @@ -1,7 +1,7 @@ Feature: Custom attributes Scenario: Custom attributes can be added to spans - Given I navigate to the test URL "/custom-attributes" + Given I navigate to the test URL "/docs/custom-attributes" And I wait for 1 span Then a span named "Custom/CustomAttributesScenario" contains the attributes: diff --git a/test/browser/features/device-id.feature b/test/browser/features/device-id.feature index 6b35d6603..8163cacab 100644 --- a/test/browser/features/device-id.feature +++ b/test/browser/features/device-id.feature @@ -2,13 +2,13 @@ Feature: Device ID persistence Scenario: persisted device ID is used if present # load the page and persist an ID - Given I navigate to the test URL "/retry-scenario" + Given I navigate to the test URL "/docs/retry-scenario" And I wait to receive a sampling request And I store the device ID "c1234567890abcdefghijklmnop" # reload the page to prove the ID was persisted and not just available in # memory somehow - When I navigate to the test URL "/retry-scenario" + When I navigate to the test URL "/docs/retry-scenario" And I wait to receive a sampling request And I click the element "send-first-span" And I wait for 1 span diff --git a/test/browser/features/enabled-release-stages.feature b/test/browser/features/enabled-release-stages.feature index 245b92fae..d819b59c2 100644 --- a/test/browser/features/enabled-release-stages.feature +++ b/test/browser/features/enabled-release-stages.feature @@ -1,11 +1,11 @@ Feature: Configuration Scenario: Delivery is prevented when releaseStage is not in enabledReleaseStages - Given I navigate to the test URL "/enabled-release-stages-disabled" + Given I navigate to the test URL "/docs/enabled-release-stages-disabled" Then I should receive no traces Scenario: Delivery is allowed when releaseStage is present in enabledReleaseStages - Given I navigate to the test URL "/enabled-release-stages" + Given I navigate to the test URL "/docs/enabled-release-stages" And I wait to receive at least 1 trace Then the trace payload field "resourceSpans.0.scopeSpans.0.spans.0.name" equals "Custom/Should send" And the trace payload field "resourceSpans.0.resource" string attribute "deployment.environment" equals "test" diff --git a/test/browser/features/error-correlation.feature b/test/browser/features/error-correlation.feature index 8a7cd416c..4880e6826 100644 --- a/test/browser/features/error-correlation.feature +++ b/test/browser/features/error-correlation.feature @@ -1,7 +1,7 @@ Feature: Error Correlation Scenario: Error does not include the correlation property when no spans are open - Given I navigate to the test URL "/error-correlation" + Given I navigate to the test URL "/docs/error-correlation" And I wait to receive a sampling request And I click the element "send-error" And I wait to receive an error @@ -9,7 +9,7 @@ Feature: Error Correlation And the error payload field "events.0.correlation" is null Scenario: Reported errors include the current trace and span id - Given I navigate to the test URL "/error-correlation" + Given I navigate to the test URL "/docs/error-correlation" And I wait to receive a sampling request Then I click the element "start-span" And I wait for 1 second @@ -33,7 +33,7 @@ Feature: Error Correlation And the error payload field "events.0.correlation.traceId" equals the stored value "traceId" Scenario: Reported errors do not include the trace and span id if the span is not the current context - Given I navigate to the test URL "/error-correlation" + Given I navigate to the test URL "/docs/error-correlation" And I wait to receive a sampling request Then I click the element "start-span" And I wait for 1 second diff --git a/test/browser/features/fixed-sampling-probability.feature b/test/browser/features/fixed-sampling-probability.feature index af375f220..3bf9278cd 100644 --- a/test/browser/features/fixed-sampling-probability.feature +++ b/test/browser/features/fixed-sampling-probability.feature @@ -1,7 +1,7 @@ Feature: Fixed Sampling Probability Scenario: Uses configured sampling probability - Given I navigate to the test URL "/fixed-sampling-probability" + Given I navigate to the test URL "/docs/fixed-sampling-probability" And I enter unmanaged traces mode When I click the element "send-span" And I wait to receive 1 trace diff --git a/test/browser/features/fixtures/packages/angular/src/index.html b/test/browser/features/fixtures/packages/angular/src/index.html index e2c0b6b82..6523fb01a 100644 --- a/test/browser/features/fixtures/packages/angular/src/index.html +++ b/test/browser/features/fixtures/packages/angular/src/index.html @@ -2,9 +2,9 @@ - + Angular - + diff --git a/test/browser/features/fixtures/packages/react-router/index.html b/test/browser/features/fixtures/packages/react-router/index.html index f3f0a2bb1..b01d0bfbd 100644 --- a/test/browser/features/fixtures/packages/react-router/index.html +++ b/test/browser/features/fixtures/packages/react-router/index.html @@ -2,7 +2,7 @@ - + React router diff --git a/test/browser/features/fixtures/packages/react-router/src/index.jsx b/test/browser/features/fixtures/packages/react-router/src/index.jsx index b0034814d..518236a81 100644 --- a/test/browser/features/fixtures/packages/react-router/src/index.jsx +++ b/test/browser/features/fixtures/packages/react-router/src/index.jsx @@ -13,7 +13,7 @@ const parameters = new URLSearchParams(window.location.search) const apiKey = parameters.get('api_key') const endpoint = parameters.get('endpoint') -const basename = '/react-router' +const basename = '/docs/react-router' function Root() { return ( diff --git a/test/browser/features/fixtures/packages/resource-load-spans/index.html b/test/browser/features/fixtures/packages/resource-load-spans/index.html index fbe5994ac..dd8f6ac8c 100644 --- a/test/browser/features/fixtures/packages/resource-load-spans/index.html +++ b/test/browser/features/fixtures/packages/resource-load-spans/index.html @@ -2,11 +2,11 @@ - + - + Resource load spans diff --git a/test/browser/features/fixtures/packages/resource-load-spans/src/index.js b/test/browser/features/fixtures/packages/resource-load-spans/src/index.js index b2ec76a61..cf1ad579d 100644 --- a/test/browser/features/fixtures/packages/resource-load-spans/src/index.js +++ b/test/browser/features/fixtures/packages/resource-load-spans/src/index.js @@ -12,7 +12,7 @@ document.getElementById("end-span").addEventListener("click", () => { const node = document.getElementById("image-container") const img = new Image() node.appendChild(img) - img.src = "/favicon.png?height=100&width=100" + img.src = "/docs/favicon.png?height=100&width=100" img.onload = () => { span.end() } diff --git a/test/browser/features/fixtures/packages/route-change-spans/index.html b/test/browser/features/fixtures/packages/route-change-spans/index.html index 20b94cb16..4a5eaa0da 100644 --- a/test/browser/features/fixtures/packages/route-change-spans/index.html +++ b/test/browser/features/fixtures/packages/route-change-spans/index.html @@ -2,7 +2,7 @@ - + Route change spans diff --git a/test/browser/features/fixtures/packages/route-change-spans/src/app.jsx b/test/browser/features/fixtures/packages/route-change-spans/src/app.jsx index 5aa12fdd9..e8b95b96a 100644 --- a/test/browser/features/fixtures/packages/route-change-spans/src/app.jsx +++ b/test/browser/features/fixtures/packages/route-change-spans/src/app.jsx @@ -12,7 +12,7 @@ function Navigation() {