diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 161921db4..635ce3afc 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -11,7 +11,9 @@ steps: key: cocoa_fixture timeout_in_minutes: 30 agents: - queue: macos-12-arm + queue: macos-13-arm + env: + DEVELOPER_DIR: /Applications/Xcode14.3.app artifact_paths: - features/fixtures/ios/output/iOSTestApp.ipa - features/fixtures/macos/output/macOSTestApp.zip @@ -26,7 +28,7 @@ steps: - label: Static framework and Swift Package Manager builds timeout_in_minutes: 10 agents: - queue: macos-12-arm + queue: macos-13-arm commands: - make build_swift - make build_ios_static @@ -34,7 +36,7 @@ steps: - label: Carthage timeout_in_minutes: 15 agents: - queue: macos-12-arm + queue: macos-13-arm commands: - ./scripts/build-carthage.sh plugins: @@ -91,6 +93,15 @@ steps: artifact_paths: - logs/* + - label: iOS 17 unit tests + timeout_in_minutes: 10 + agents: + queue: macos-13-arm + commands: + - ./scripts/run-unit-tests.sh PLATFORM=iOS OS=17.0 + artifact_paths: + - logs/* + - label: iOS 15 unit tests timeout_in_minutes: 10 agents: @@ -467,6 +478,68 @@ steps: # # BrowserStack # + - label: ':browserstack: iOS 17 E2E tests batch 1' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + service-ports: true + command: + - --app=@/app/build/ipa_url_bs.txt + - --farm=bs + - --device=IOS_17 + - --appium-version=1.18.0 + - --a11y-locator + - --fail-fast + - --exclude=features/[e-z].*.feature$ + - --order=random + concurrency: 25 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + + - label: ':browserstack: iOS 17 E2E tests batch 2' + depends_on: + - cocoa_fixture + timeout_in_minutes: 60 + agents: + queue: opensource + plugins: + artifacts#v1.5.0: + download: "features/fixtures/ios/output/ipa_url_bs.txt" + upload: "maze_output/failed/**/*" + docker-compose#v3.7.0: + pull: cocoa-maze-runner + run: cocoa-maze-runner + service-ports: true + command: + - --app=@/app/build/ipa_url_bs.txt + - --farm=bs + - --device=IOS_17 + - --appium-version=1.18.0 + - --a11y-locator + - --fail-fast + - --exclude=features/[a-d].*.feature$ + - --order=random + concurrency: 25 + concurrency_group: 'browserstack-app' + concurrency_method: eager + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - label: ':browserstack: iOS 11 barebone tests' skip: "https://smartbear.atlassian.net/browse/PLAT-11154" depends_on: diff --git a/BugsnagNetworkRequestPlugin/BugsnagNetworkRequestPlugin.xcodeproj/project.pbxproj b/BugsnagNetworkRequestPlugin/BugsnagNetworkRequestPlugin.xcodeproj/project.pbxproj index 1110195f5..fff8a89fe 100644 --- a/BugsnagNetworkRequestPlugin/BugsnagNetworkRequestPlugin.xcodeproj/project.pbxproj +++ b/BugsnagNetworkRequestPlugin/BugsnagNetworkRequestPlugin.xcodeproj/project.pbxproj @@ -888,6 +888,7 @@ CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = 372ZUL2ZB7; INFOPLIST_FILE = "../Tests/TestHost-iOS/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -905,6 +906,7 @@ CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = 372ZUL2ZB7; INFOPLIST_FILE = "../Tests/TestHost-iOS/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -1250,6 +1252,7 @@ DYLIB_INSTALL_NAME_BASE = "@rpath"; INFOPLIST_FILE = BugsnagNetworkRequestPlugin/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -1273,6 +1276,7 @@ DYLIB_INSTALL_NAME_BASE = "@rpath"; INFOPLIST_FILE = BugsnagNetworkRequestPlugin/Info.plist; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -1293,6 +1297,7 @@ GCC_WARN_PEDANTIC = NO; GCC_WARN_UNUSED_PARAMETER = NO; INFOPLIST_FILE = BugsnagNetworkRequestPluginTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -1313,6 +1318,7 @@ GCC_WARN_PEDANTIC = NO; GCC_WARN_UNUSED_PARAMETER = NO; INFOPLIST_FILE = BugsnagNetworkRequestPluginTests/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", diff --git a/Gemfile b/Gemfile index 427043d09..95566ec28 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,7 @@ gem 'cocoapods' # A reference to Maze Runner is only needed for running tests locally and if committed it must be # portable for CI, e.g. a specific release. However, leaving it commented out would mean quicker CI. -gem 'bugsnag-maze-runner', '~> 8.0' +gem 'bugsnag-maze-runner', '~> 8.8' # Use a specific branch #gem 'bugsnag-maze-runner', git: 'https://github.com/bugsnag/maze-runner', branch: 'master' diff --git a/Gemfile.lock b/Gemfile.lock index 1c00ea98a..d50a61457 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -21,9 +21,9 @@ GEM faye-websocket (~> 0.11.0) selenium-webdriver (~> 4.2, < 4.6) atomos (0.1.3) - bugsnag (6.25.2) + bugsnag (6.26.0) concurrent-ruby (~> 1.0) - bugsnag-maze-runner (8.1.4) + bugsnag-maze-runner (8.8.0) appium_lib (~> 12.0.0) appium_lib_core (~> 5.4.0) bugsnag (~> 6.24) @@ -122,7 +122,7 @@ GEM ethon (0.16.0) ffi (>= 1.15.0) eventmachine (1.2.7) - faye-websocket (0.11.2) + faye-websocket (0.11.3) eventmachine (>= 0.12.0) websocket-driver (>= 0.5.1) ffi (1.15.5) @@ -140,25 +140,25 @@ GEM regexp_parser (~> 2.0) simpleidn (~> 0.2) uri_template (~> 0.7) - mime-types (3.4.1) + mime-types (3.5.1) mime-types-data (~> 3.2015) - mime-types-data (3.2023.0218.1) + mime-types-data (3.2023.1003) minitest (5.18.1) molinillo (0.8.0) multi_test (0.1.2) nanaimo (0.3.0) nap (1.1.0) netrc (0.11.0) - nokogiri (1.15.3-x86_64-darwin) + nokogiri (1.15.4-x86_64-darwin) racc (~> 1.4) optimist (3.0.1) os (1.0.1) power_assert (2.0.3) public_suffix (4.0.7) racc (1.7.1) - rack (2.2.7) + rack (2.2.8) rake (12.3.3) - regexp_parser (2.8.1) + regexp_parser (2.8.2) rexml (3.2.5) ruby-macho (2.5.1) rubyzip (2.3.2) @@ -183,8 +183,8 @@ GEM unf_ext (0.0.8.2) uri_template (0.7.0) webrick (1.7.0) - websocket (1.2.9) - websocket-driver (0.7.5) + websocket (1.2.10) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xcodeproj (1.22.0) @@ -198,9 +198,10 @@ GEM PLATFORMS x86_64-darwin-19 x86_64-darwin-20 + x86_64-darwin-21 DEPENDENCIES - bugsnag-maze-runner (~> 8.0) + bugsnag-maze-runner (~> 8.8) cocoapods BUNDLED WITH diff --git a/Makefile b/Makefile index 842568a78..352c5ea02 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,12 @@ else else ifeq ($(PLATFORM),iOS) SDK?=iphonesimulator - DEVICE?=iPhone 8 + ifeq ($(shell expr $(OS) \>= 17.0), 1) + DEVICE?=iPhone 14 + else + DEVICE?=iPhone 8 + endif + DESTINATION?=platform=iOS Simulator,name=$(DEVICE),OS=$(OS) RELEASE_DIR=Release-iphoneos else diff --git a/Tests/BugsnagTests/BSGInternalErrorReporterTests.m b/Tests/BugsnagTests/BSGInternalErrorReporterTests.m index 37aeea86e..2948d5154 100644 --- a/Tests/BugsnagTests/BSGInternalErrorReporterTests.m +++ b/Tests/BugsnagTests/BSGInternalErrorReporterTests.m @@ -63,14 +63,17 @@ - (void)testEventWithException { NSException *exception = nil; @try { - NSLog(@"%@", @[][0]); + [[NSException exceptionWithName:NSRangeException + reason:@"Something is out of range" + userInfo:nil] raise]; } @catch (NSException *e) { exception = e; } BugsnagEvent *event = [reporter eventWithException:exception diagnostics:nil groupingHash:@"test"]; XCTAssertEqualObjects(event.errors[0].errorClass, @"NSRangeException"); - XCTAssertEqualObjects(event.errors[0].errorMessage, @"*** -[__NSArray0 objectAtIndex:]: index 0 beyond bounds for empty NSArray"); + XCTAssertEqualObjects(event.errors[0].errorMessage, @"Something is out of range"); + XCTAssertEqualObjects(event.groupingHash, @"test"); XCTAssertEqualObjects(event.threads, @[]); XCTAssertGreaterThan(event.errors[0].stacktrace.count, 0); diff --git a/Tests/BugsnagTests/BugsnagStackframeTest.m b/Tests/BugsnagTests/BugsnagStackframeTest.m index c647c2385..bb584fd86 100644 --- a/Tests/BugsnagTests/BugsnagStackframeTest.m +++ b/Tests/BugsnagTests/BugsnagStackframeTest.m @@ -253,6 +253,10 @@ - (void)testRealCallStackSymbols { // This frame is not in any known image return; } + if ([stackframe.method isEqualToString: @"start_sim"]) { + // This frame is part of the simulator environment + return; + } #endif XCTAssertNotNil(stackframe.machoUuid); XCTAssertNotNil(stackframe.machoVmAddress); diff --git a/features/barebone_tests.feature b/features/barebone_tests.feature index 497eea7c7..691eeb534 100644 --- a/features/barebone_tests.feature +++ b/features/barebone_tests.feature @@ -68,7 +68,7 @@ Feature: Barebone tests And the event "metaData.error.nsexception.userInfo.date" equals "2001-01-01 00:00:00 +0000" And the event "metaData.error.nsexception.userInfo.NSUnderlyingError" matches "Error Domain=ErrorDomain Code=0" And the event "metaData.error.nsexception.userInfo.scenario" equals "BareboneTestHandledScenario" - And the event "metaData.error.reason" equals "-[__NSSingleObjectArrayI objectAtIndex:]: index 1 beyond bounds [0 .. 0]" + And the event "metaData.error.reason" equals "Something is out of range" And the event "metaData.error.type" equals "nsexception" And the event "metaData.usage" is null And the event "metaData.user.email" is null @@ -103,7 +103,7 @@ Feature: Barebone tests | notify | rangeException | And the event does not contain the feature flag "nope" And the exception "errorClass" equals "NSRangeException" - And the exception "message" equals "-[__NSSingleObjectArrayI objectAtIndex:]: index 1 beyond bounds [0 .. 0]" + And the exception "message" equals "Something is out of range" And the exception "type" equals "cocoa" And the error payload field "events.0.app.dsymUUIDs" is a non-empty array And the error payload field "events.0.app.duration" is a number @@ -199,7 +199,7 @@ Feature: Barebone tests And the event "metaData.error.nsexception.userInfo.date" equals "2001-01-01 00:00:00 +0000" And the event "metaData.error.nsexception.userInfo.NSUnderlyingError" matches "Error Domain=ErrorDomain Code=0" And the event "metaData.error.nsexception.userInfo.scenario" equals "BareboneTestUnhandledErrorScenario" - And the event "metaData.error.reason" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray" + And the event "metaData.error.reason" equals "Something is out of range" And the event "metaData.error.type" equals "nsexception" And the event "metaData.usage" is null And the event "metaData.user.email" is null @@ -227,7 +227,7 @@ Feature: Barebone tests | featureFlag | variant | | Testing | | And the exception "errorClass" equals "NSRangeException" - And the exception "message" equals "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray" + And the exception "message" equals "Something is out of range" And the exception "type" equals "cocoa" And the error payload field "events.0.app.dsymUUIDs" is a non-empty array And the error payload field "events.0.app.duration" is a number @@ -244,6 +244,7 @@ Feature: Barebone tests @skip_macos @skip_ios_16 # https://smartbear.atlassian.net/browse/PLAT-9724 + @skip_ios_17 Scenario: Barebone test: Out Of Memory When I run "OOMScenario" diff --git a/features/delivery.feature b/features/delivery.feature index 6855ffa90..08af636e5 100644 --- a/features/delivery.feature +++ b/features/delivery.feature @@ -46,7 +46,7 @@ Feature: Delivery of errors And I configure Bugsnag for "OldHandledErrorScenario" And I wait to receive an error And I wait for the fixture to process the response - # The error should now have been deleted + # The error should now have been deleted And I kill and relaunch the app And I clear the error queue And I configure Bugsnag for "OldHandledErrorScenario" @@ -157,6 +157,7 @@ Feature: Delivery of errors And the event "usage.system.stringCharsTruncated" is not null And the event "usage.system.stringsTruncated" is not null + @skip_ios_17 Scenario Outline: Attempt Delivery On Crash When I set the app to "" mode And I run "AttemptDeliveryOnCrashScenario" @@ -174,7 +175,31 @@ Feature: Delivery of errors And I wait to receive 2 sessions Then I should receive no error Examples: - | scenario_mode | error_type | error_class | message | - | NSException | nsexception | NSRangeException | *** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray | - | SwiftFatalError | mach | Fatal error | Unexpectedly found nil while unwrapping an Optional value | - | BadAccess | mach | EXC_BAD_ACCESS | Attempted to dereference garbage pointer 0x20. | + | scenario_mode | error_type | error_class | message | + | NSException | nsexception | NSRangeException | Something is out of range | + | SwiftFatalError | mach | Fatal error | Unexpectedly found nil while unwrapping an Optional value | + | BadAccess | mach | EXC_BAD_ACCESS | Attempted to dereference garbage pointer 0x20. | + + @skip_below_ios_17 + @skip_macos + Scenario Outline: Attempt Delivery On Crash iOS 17 + When I set the app to "" mode + And I run "AttemptDeliveryOnCrashScenario" + And I wait to receive an error + Then the error is valid for the error reporting API + And the event "context" equals "OnSendError" + And the exception "errorClass" equals "" + And the exception "message" equals "" + And the event "metaData.error.type" equals "" + And the event "unhandled" is true + And the event "usage.config.attemptDeliveryOnCrash" is true + And I discard the oldest error + And I relaunch the app after a crash + And I configure Bugsnag for "AttemptDeliveryOnCrashScenario" + And I wait to receive 2 sessions + Then I should receive no error + Examples: + | scenario_mode | error_type | error_class | message | + | NSException | nsexception | NSRangeException | Something is out of range | + | SwiftFatalError | mach | Fatal error | Unexpectedly found nil while unwrapping an Optional value | + | BadAccess | mach | EXC_BAD_ACCESS | Attempted to dereference garbage pointer 0x20. | diff --git a/features/fixtures/shared/scenarios/AttemptDeliveryOnCrashScenario.swift b/features/fixtures/shared/scenarios/AttemptDeliveryOnCrashScenario.swift index c3ccf1ae4..5dd2f1b03 100644 --- a/features/fixtures/shared/scenarios/AttemptDeliveryOnCrashScenario.swift +++ b/features/fixtures/shared/scenarios/AttemptDeliveryOnCrashScenario.swift @@ -28,7 +28,14 @@ class AttemptDeliveryOnCrashScenario: Scenario { break case "NSException": - NSArray().object(at: 42) + NSException( + name: .rangeException, + reason: "Something is out of range", + userInfo: [ + "date": Date(timeIntervalSinceReferenceDate: 0), + "scenario": "BareboneTestUnhandledErrorScenario", + NSUnderlyingErrorKey: NSError(domain: "ErrorDomain", code: 0)]) + .raise() break case "SwiftFatalError": diff --git a/features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift b/features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift index 763a83822..ae7e85198 100644 --- a/features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift +++ b/features/fixtures/shared/scenarios/BareboneTestHandledScenario.swift @@ -89,7 +89,7 @@ class BareboneTestHandledScenario: Scenario { """, key: "shouldBeTruncated", section: "Other") Bugsnag.notify(NSException(name: .rangeException, - reason: "-[__NSSingleObjectArrayI objectAtIndex:]: index 1 beyond bounds [0 .. 0]", + reason: "Something is out of range", userInfo: ["date": Date(timeIntervalSinceReferenceDate: 0), "scenario": "BareboneTestHandledScenario", NSUnderlyingErrorKey: NSError(domain: "ErrorDomain", code: 0)])) { diff --git a/features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift b/features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift index 1c3c4d5f3..689505e39 100644 --- a/features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift +++ b/features/fixtures/shared/scenarios/BareboneTestUnhandledErrorScenario.swift @@ -33,7 +33,7 @@ class BareboneTestUnhandledErrorScenario: Scenario { // Manually constructing an exception to verify handling of userInfo NSException( name: .rangeException, - reason: "*** -[__NSArray0 objectAtIndex:]: index 42 beyond bounds for empty NSArray", + reason: "Something is out of range", userInfo: [ "date": Date(timeIntervalSinceReferenceDate: 0), "scenario": "BareboneTestUnhandledErrorScenario", diff --git a/features/fixtures/shared/scenarios/RecrashScenarios.mm b/features/fixtures/shared/scenarios/RecrashScenarios.mm index 784bf81a1..da496aa9b 100644 --- a/features/fixtures/shared/scenarios/RecrashScenarios.mm +++ b/features/fixtures/shared/scenarios/RecrashScenarios.mm @@ -12,7 +12,10 @@ #import #define THROW_CPP_EXCEPTION throw std::runtime_error("err") -#define THROW_OBJC_EXCEPTION [@[] objectAtIndex:42] +#define THROW_OBJC_EXCEPTION \ + [[NSException exceptionWithName:NSRangeException \ + reason:@"Something is out of range" \ + userInfo:nil] raise] #define CAUSE_MACH_EXCEPTION volatile int *ptr = NULL; *ptr = 42 #define RAISE_SIGNAL abort() diff --git a/features/support/env.rb b/features/support/env.rb index e4e4bd9d5..72b2c6012 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -77,6 +77,10 @@ def skip_between(os, version_lo, version_hi) skip_between('ios', 16, 16.99) end +Before('@skip_ios_17') do |_scenario| + skip_between('ios', 17, 17.99) +end + Before('@skip_below_ios_11') do |_scenario| skip_below('ios', 11) end @@ -85,6 +89,10 @@ def skip_between(os, version_lo, version_hi) skip_below('ios', 13) end +Before('@skip_below_ios_13') do |_scenario| + skip_below('ios', 17) +end + Before('@skip_below_macos_10_15') do |_scenario| skip_below('macos', 10.15) end