From 546a181f1c125109df0c09949d9150dd1eac4868 Mon Sep 17 00:00:00 2001 From: Konstantin Svist Date: Thu, 27 Apr 2023 13:21:05 -0700 Subject: [PATCH] Bringing integration tests back to life * Updated default iOS test device from iphone8/12 to iphone13pro/15.7 (#2366) FTL recently deprecated iphone8/12, so integration tests have been failing * Updated local copy of FTL catalog to recent version * Replaced iOS integration test app EarlGreyExample with FlankExampleTest EarlGreyExample is failing on newer OS/xcode. Current copy is based on EarlGreyExample v1, which is no longer maintained. V2 is pretty hard to get up and running, so switching to FlankExampleTest is the easiest way to unblock integration tests. --- docs/index.md | 4 +- .../test/kotlin/integration/DumpShardsIT.kt | 4 +- .../test/kotlin/integration/LegacyResultIT.kt | 2 +- .../resources/cases/all_test_filtered_ios.yml | 4 +- .../test/resources/cases/dump_shards_ios.yml | 4 +- .../cases/flank_ios_single_legacy.yml | 4 +- .../compare/AllTestFilteredIT-ios-compare | 8 +- .../ios/EarlGreyExample/flank.ios.yml | 4 +- test_runner/flank.ios.yml | 4 +- .../main/kotlin/ftl/config/FtlConstants.kt | 4 +- .../src/main/resources/ios_catalog.json | 411 ++++++++---------- .../test/kotlin/ftl/args/IosArgsFileTest.kt | 2 +- .../src/test/kotlin/ftl/args/IosArgsTest.kt | 36 +- .../kotlin/ftl/args/ValidateIosArgsTest.kt | 4 +- .../firebase/test/ios/IosRunCommandTest.kt | 4 +- .../src/test/kotlin/ftl/doctor/DoctorTest.kt | 2 +- .../kotlin/ftl/fixtures/flank.ios.gcs.yml | 4 +- .../test/kotlin/ftl/fixtures/flank.ios.yml | 4 +- .../test/kotlin/ftl/fixtures/flank2.ios.yml | 4 +- .../src/test/kotlin/ftl/ios/IosCatalogTest.kt | 2 +- .../ftl/reports/utils/ReportManagerTest.kt | 4 +- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 2 +- 22 files changed, 232 insertions(+), 289 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5e76f54b43..cb860361be 100644 --- a/docs/index.md +++ b/docs/index.md @@ -152,8 +152,8 @@ gcloud: ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. # device: - # - model: iphone8 - # version: 12.0 + # - model: iphone13pro + # version: 15.7 # locale: en # orientation: portrait # - model: iphonex diff --git a/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt b/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt index b4d40d8ef1..cacd0960bd 100644 --- a/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt +++ b/integration_tests/src/test/kotlin/integration/DumpShardsIT.kt @@ -89,8 +89,8 @@ class DumpShardsIT { assertThat(shards.count()).isEqualTo(2) shards.first().values.flatten().let { firstShard -> - assertThat(firstShard.count()).isEqualTo(8) - assertThat(firstShard).contains("EarlGreyExampleSwiftTests/testWithCustomFailureHandler") + assertThat(firstShard.count()).isEqualTo(2) + assertThat(firstShard).contains("FlankExampleTests/test2") } } } diff --git a/integration_tests/src/test/kotlin/integration/LegacyResultIT.kt b/integration_tests/src/test/kotlin/integration/LegacyResultIT.kt index cb10b75ae4..1d9d2c7ed1 100644 --- a/integration_tests/src/test/kotlin/integration/LegacyResultIT.kt +++ b/integration_tests/src/test/kotlin/integration/LegacyResultIT.kt @@ -93,7 +93,7 @@ class LegacyResultIT { val testSuiteOverview = outputReport.firstTestSuiteOverview testSuiteOverview.assertTestCountMatches( - total = 17, + total = 4, skipped = 0 ) } diff --git a/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml b/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml index 3b48dbc340..0bf2f317a9 100644 --- a/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml +++ b/integration_tests/src/test/resources/cases/all_test_filtered_ios.yml @@ -1,6 +1,6 @@ gcloud: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip - xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.zip + xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.xctestrun flank: test-targets: - nonExisting/Class diff --git a/integration_tests/src/test/resources/cases/dump_shards_ios.yml b/integration_tests/src/test/resources/cases/dump_shards_ios.yml index 0d63e2f0f1..e8fabbfe16 100644 --- a/integration_tests/src/test/resources/cases/dump_shards_ios.yml +++ b/integration_tests/src/test/resources/cases/dump_shards_ios.yml @@ -1,6 +1,6 @@ gcloud: - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip - xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun + test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.zip + xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.xctestrun flank: disable-sharding: false diff --git a/integration_tests/src/test/resources/cases/flank_ios_single_legacy.yml b/integration_tests/src/test/resources/cases/flank_ios_single_legacy.yml index c3c7bb8fd6..3be055a7f2 100644 --- a/integration_tests/src/test/resources/cases/flank_ios_single_legacy.yml +++ b/integration_tests/src/test/resources/cases/flank_ios_single_legacy.yml @@ -1,6 +1,6 @@ gcloud: - test: "../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip" - xctestrun-file: "../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun" + test: "../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.zip" + xctestrun-file: "../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/FlankExample/FlankExample.xctestrun" flank: output-report: json diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare index f1c7415c1a..1784fbb3de 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare @@ -9,12 +9,12 @@ IosArgs network-profile: null results-history-name: null # iOS gcloud - test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/EarlGreyExample\/EarlGreyExample.zip - xctestrun-file: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/EarlGreyExample\/EarlGreyExampleSwiftTests.xctestrun + test: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/FlankExample\/FlankExample.zip + xctestrun-file: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/FlankExample\/FlankExample.xctestrun xcode-version: null device: - - model: iphone8 - version: 13.6 + - model: iphone13pro + version: 15.7 locale: en orientation: portrait num-flaky-test-attempts: 0 diff --git a/test_projects/ios/EarlGreyExample/flank.ios.yml b/test_projects/ios/EarlGreyExample/flank.ios.yml index 6ad796b4e9..e02e93e6af 100644 --- a/test_projects/ios/EarlGreyExample/flank.ios.yml +++ b/test_projects/ios/EarlGreyExample/flank.ios.yml @@ -3,7 +3,7 @@ gcloud: test: ./earlgrey_example.zip xctestrun-file: ./dd_tmp/Build/Products/EarlGreyExampleSwiftTests_iphoneos13.4-arm64e.xctestrun device: - - model: iphone8 - version: 12.0 + - model: iphone13pro + version: 15.7 orientation: portrait locale: en_US diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 69164ea5e7..a6e24f2522 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -88,8 +88,8 @@ gcloud: ## This flag may be repeated to specify multiple devices. ## The four device dimensions are: model, version, locale, and orientation. # device: - # - model: iphone8 - # version: 12.0 + # - model: iphone13pro + # version: 15.7 # locale: en # orientation: portrait # - model: iphonex diff --git a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt index 4fc00421bf..aaeba826a7 100644 --- a/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt +++ b/test_runner/src/main/kotlin/ftl/config/FtlConstants.kt @@ -16,8 +16,8 @@ object FtlConstants { const val localhost = "http://localhost:8080" const val defaultLocale = "en" const val defaultOrientation = "portrait" - const val defaultIosModel = "iphone8" - const val defaultIosVersion = "13.6" + const val defaultIosModel = "iphone13pro" + const val defaultIosVersion = "15.7" const val defaultAndroidModel = "NexusLowRes" const val defaultAndroidVersion = "28" const val defaultIosConfig = "./flank.ios.yml" diff --git a/test_runner/src/main/resources/ios_catalog.json b/test_runner/src/main/resources/ios_catalog.json index a2473fc793..8633bae6ae 100644 --- a/test_runner/src/main/resources/ios_catalog.json +++ b/test_runner/src/main/resources/ios_catalog.json @@ -27,14 +27,17 @@ "formFactor": "TABLET", "id": "ipad5", "name": "iPad (5th generation)", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_LOW", + "versionId": "15.4" + } + ], "screenDensity": 264, "screenX": 1536, "screenY": 2048, "supportedVersionIds": [ - "14.1" - ], - "tags": [ - "deprecated=14.1" + "15.4" ] }, { @@ -63,55 +66,17 @@ "formFactor": "TABLET", "id": "ipadmini4", "name": "iPad mini 4", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_LOW", + "versionId": "15.4" + } + ], "screenDensity": 326, "screenX": 1536, "screenY": 2048, "supportedVersionIds": [ - "14.1" - ], - "tags": [ - "deprecated=14.1" - ] - }, - { - "deviceCapabilities": [ - "accelerometer", - "arm64", - "armv6", - "armv7", - "auto-focus-camera", - "bluetooth-le", - "front-facing-camera", - "gamekit", - "gyroscope", - "location-services", - "magnetometer", - "metal", - "microphone", - "opengles-1", - "opengles-2", - "opengles-3", - "peer-peer", - "still-camera", - "video-camera", - "wifi", - "arkit", - "camera-flash", - "gps", - "healthkit", - "nfc", - "sms", - "telephony" - ], - "formFactor": "PHONE", - "id": "iphone11", - "name": "iPhone 11", - "screenDensity": 326, - "screenX": 828, - "screenY": 1792, - "supportedVersionIds": [ - "13.3", - "13.6" + "15.4" ] }, { @@ -147,16 +112,30 @@ "formFactor": "PHONE", "id": "iphone11pro", "name": "iPhone 11 Pro", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "14.7" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "16.2" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_LOW", + "versionId": "16.3" + } + ], "screenDensity": 458, "screenX": 1125, "screenY": 2436, "supportedVersionIds": [ - "13.3", - "14.1", - "14.7" + "14.7", + "16.2", + "16.3" ], "tags": [ - "deprecated=14.1" + "deprecated=16.2" ] }, { @@ -190,16 +169,19 @@ "telephony" ], "formFactor": "PHONE", - "id": "iphone7plus", - "name": "iPhone 7 Plus", - "screenDensity": 401, - "screenX": 1080, - "screenY": 1920, - "supportedVersionIds": [ - "14.1" + "id": "iphone12pro", + "name": "iPhone 12 Pro", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "14.8" + } ], - "tags": [ - "deprecated=14.1" + "screenDensity": 460, + "screenX": 1170, + "screenY": 2532, + "supportedVersionIds": [ + "14.8" ] }, { @@ -233,21 +215,26 @@ "telephony" ], "formFactor": "PHONE", - "id": "iphone8", - "name": "iPhone 8", - "screenDensity": 326, - "screenX": 750, - "screenY": 1334, + "id": "iphone13pro", + "name": "iPhone 13 Pro", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_LOW", + "versionId": "15.2" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "15.7" + } + ], + "screenDensity": 460, + "screenX": 1170, + "screenY": 2532, "supportedVersionIds": [ - "11.4", - "12.0", - "12.4", - "13.6", - "14.1", - "14.7" + "15.2", + "15.7" ], "tags": [ - "deprecated=14.1", "default" ] }, @@ -282,94 +269,37 @@ "telephony" ], "formFactor": "PHONE", - "id": "iphone8plus", - "name": "iPhone 8 Plus", - "screenDensity": 401, - "screenX": 1080, - "screenY": 1920, - "supportedVersionIds": [ - "12.0" - ] - }, - { - "deviceCapabilities": [ - "accelerometer", - "arm64", - "armv6", - "armv7", - "auto-focus-camera", - "bluetooth-le", - "front-facing-camera", - "gamekit", - "gyroscope", - "location-services", - "magnetometer", - "metal", - "microphone", - "opengles-1", - "opengles-2", - "opengles-3", - "peer-peer", - "still-camera", - "video-camera", - "wifi", - "arkit", - "camera-flash", - "gps", - "healthkit", - "nfc", - "sms", - "telephony" + "id": "iphone8", + "name": "iPhone 8", + "perVersionInfo": [ + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "14.7" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "15.7" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "16.2" + }, + { + "deviceCapacity": "DEVICE_CAPACITY_MEDIUM", + "versionId": "16.3" + } ], - "formFactor": "PHONE", - "id": "iphonexr", - "name": "iPhone XR", "screenDensity": 326, - "screenX": 828, - "screenY": 1792, + "screenX": 750, + "screenY": 1334, "supportedVersionIds": [ - "12.4", - "13.2" - ] - }, - { - "deviceCapabilities": [ - "accelerometer", - "arm64", - "armv6", - "armv7", - "auto-focus-camera", - "bluetooth-le", - "front-facing-camera", - "gamekit", - "gyroscope", - "location-services", - "magnetometer", - "metal", - "microphone", - "opengles-1", - "opengles-2", - "opengles-3", - "peer-peer", - "still-camera", - "video-camera", - "wifi", - "arkit", - "camera-flash", - "gps", - "healthkit", - "nfc", - "sms", - "telephony" + "14.7", + "15.7", + "16.2", + "16.3" ], - "formFactor": "PHONE", - "id": "iphonexsmax", - "name": "iPhone XS Max", - "screenDensity": 458, - "screenX": 1242, - "screenY": 2688, - "supportedVersionIds": [ - "12.1" + "tags": [ + "deprecated=16.2" ] } ], @@ -3267,143 +3197,156 @@ }, "versions": [ { - "id": "10.3", - "majorVersion": 10, - "minorVersion": 3, + "id": "13.2", + "majorVersion": 13, + "minorVersion": 2, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "11.2", - "majorVersion": 11, - "minorVersion": 2, + "id": "13.3", + "majorVersion": 13, + "minorVersion": 3, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "11.4", - "majorVersion": 11, - "minorVersion": 4, + "id": "13.6", + "majorVersion": 13, + "minorVersion": 6, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "12.0", - "majorVersion": 12, + "id": "14.1", + "majorVersion": 14, + "minorVersion": 1, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "12.1", - "majorVersion": 12, - "minorVersion": 1, + "id": "14.7", + "majorVersion": 14, + "minorVersion": 7, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "12.2", - "majorVersion": 12, - "minorVersion": 2, + "id": "14.8", + "majorVersion": 14, + "minorVersion": 8, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "12.5.1", + "13.4.1", + "14.2" ] }, { - "id": "12.3", - "majorVersion": 12, - "minorVersion": 3, + "id": "15.0", + "majorVersion": 15, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "13.4.1", + "14.2" ] }, { - "id": "12.4", - "majorVersion": 12, - "minorVersion": 4, + "id": "15.1", + "majorVersion": 15, + "minorVersion": 1, "supportedXcodeVersionIds": [ - "10.3", - "11.7", - "12.5" + "13.4.1", + "14.2" ] }, { - "id": "13.2", - "majorVersion": 13, + "id": "15.2", + "majorVersion": 15, "minorVersion": 2, "supportedXcodeVersionIds": [ - "11.7", - "12.5" + "13.4.1", + "14.2" ] }, { - "id": "13.3", - "majorVersion": 13, - "minorVersion": 3, + "id": "15.4", + "majorVersion": 15, + "minorVersion": 4, "supportedXcodeVersionIds": [ - "11.7", - "12.5" + "13.4.1", + "14.2" ] }, { - "id": "13.6", - "majorVersion": 13, - "minorVersion": 6, + "id": "15.7", + "majorVersion": 15, + "minorVersion": 7, "supportedXcodeVersionIds": [ - "11.7", - "12.5" + "13.4.1", + "14.2" + ], + "tags": [ + "default" ] }, { - "id": "14.1", - "majorVersion": 14, + "id": "16.0", + "majorVersion": 16, + "supportedXcodeVersionIds": [ + "14.2" + ] + }, + { + "id": "16.1", + "majorVersion": 16, "minorVersion": 1, "supportedXcodeVersionIds": [ - "12.5" + "14.2" ] }, { - "id": "14.7", - "majorVersion": 14, - "minorVersion": 7, + "id": "16.2", + "majorVersion": 16, + "minorVersion": 2, "supportedXcodeVersionIds": [ - "12.5" - ], - "tags": [ - "default" + "14.2" + ] + }, + { + "id": "16.3", + "majorVersion": 16, + "minorVersion": 3, + "supportedXcodeVersionIds": [ + "14.2" ] } ], "xcodeVersions": [ { - "version": "10.3" + "version": "12.5.1" }, { - "version": "11.7" + "version": "13.4.1" }, { + "version": "14.2", "tags": [ "default" - ], - "version": "12.5" + ] } ] -} \ No newline at end of file +} diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt index 8978376556..1c2f139c89 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsFileTest.kt @@ -50,7 +50,7 @@ class IosArgsFileTest { assert( devices, listOf( - Device("iphone8", "11.2", "en_US", "portrait") + Device("iphone13pro", "15.7", "en_US", "portrait") ) ) assert(maxTestShards, 1) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 7953bd7b33..fcd8e9bea8 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -74,12 +74,12 @@ class IosArgsTest { xctestrun-file: $xctestrunFile xcode-version: 9.2 device: - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 locale: c orientation: default - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 locale: c orientation: default num-flaky-test-attempts: 4 @@ -137,8 +137,8 @@ flank: @Test fun `args invalidDeviceExits`() { - assertThrowsWithMessage(Throwable::class, "iOS 11.2 on iphonexsmax is not supported\nSupported version ids for 'iphonexsmax': 12.1") { - val invalidDevice = mutableListOf(Device("iphonexsmax", "11.2")) + assertThrowsWithMessage(Throwable::class, "iOS 1.0 on iphone13pro is not supported\nSupported version ids for 'iphone13pro': 15.2, 15.7") { + val invalidDevice = mutableListOf(Device("iphone13pro", "1.0")) createIosArgs( config = defaultIosConfig().apply { common.gcloud.devices = invalidDevice @@ -190,7 +190,7 @@ flank: // IosGcloudYml assert(xctestrunZip, testAbsolutePath) assert(xctestrunFile, xctestrunFileAbsolutePath) - val device = Device("iphone8", "11.2", "c", "default") + val device = Device("iphone13pro", "15.7", "c", "default") assert(xcodeVersion ?: "", "9.2") assert(devices, listOf(device, device)) @@ -235,12 +235,12 @@ IosArgs xctestrun-file: $xctestrunFileAbsolutePath xcode-version: 9.2 device: - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 locale: c orientation: default - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 locale: c orientation: default num-flaky-test-attempts: 4 @@ -315,8 +315,8 @@ IosArgs xctestrun-file: $xctestrunFileAbsolutePath xcode-version: null device: - - model: iphone8 - version: 13.6 + - model: iphone13pro + version: 15.7 locale: en orientation: portrait num-flaky-test-attempts: 0 @@ -386,7 +386,7 @@ IosArgs // IosGcloudYml assert(xctestrunZip, testAbsolutePath) assert(xctestrunFile, xctestrunFileAbsolutePath) - assert(devices, listOf(Device("iphone8", "13.6"))) + assert(devices, listOf(Device("iphone13pro", "15.7"))) assert(flakyTestAttempts, 0) // FlankYml @@ -766,7 +766,7 @@ IosArgs @Test fun `cli device`() { val cli = IosRunCommand() - CommandLine(cli).parseArgs("--device=model=iphone8,version=12.0,locale=zh_CN,orientation=default") + CommandLine(cli).parseArgs("--device=model=iphone13pro,version=15.7,locale=zh_CN,orientation=default") val yaml = """ gcloud: @@ -779,7 +779,7 @@ IosArgs assertThat(defaultDevices.size).isEqualTo(1) val args = IosArgs.load(yaml, cli) - val expectedDevice = Device("iphone8", "12.0", "zh_CN", "default") + val expectedDevice = Device("iphone13pro", "15.7", "zh_CN", "default") val actualDevices = args.devices assertThat(actualDevices.first()).isEqualTo(expectedDevice) assertThat(actualDevices.size).isEqualTo(1) @@ -788,7 +788,7 @@ IosArgs @Test fun `cli device repeat`() { val cli = IosRunCommand() - val deviceCmd = "--device=model=iphone8,version=12.0,locale=zh_CN,orientation=default" + val deviceCmd = "--device=model=iphone13pro,version=15.7,locale=zh_CN,orientation=default" CommandLine(cli).parseArgs(deviceCmd, deviceCmd) val yaml = """ @@ -797,7 +797,7 @@ IosArgs xctestrun-file: $testPath """ val args = IosArgs.load(yaml, cli) - val expectedDevice = Device("iphone8", "12.0", "zh_CN", "default") + val expectedDevice = Device("iphone13pro", "15.7", "zh_CN", "default") val actualDevices = args.devices assertThat(actualDevices.size).isEqualTo(2) assertThat(actualDevices[0]).isEqualTo(expectedDevice) diff --git a/test_runner/src/test/kotlin/ftl/args/ValidateIosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/ValidateIosArgsTest.kt index 3cd81318d5..e7757f5f9e 100644 --- a/test_runner/src/test/kotlin/ftl/args/ValidateIosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/ValidateIosArgsTest.kt @@ -12,8 +12,8 @@ class ValidateIosArgsTest { @Test fun supportedDevice() { assertThat(isDeviceSupported("bogus", "11.2", projectId)).isFalse() - assertThat(isDeviceSupported("iphone8", "bogus", projectId)).isFalse() - assertThat(isDeviceSupported("iphone8", "12.0", projectId)).isTrue() + assertThat(isDeviceSupported("iphone13pro", "bogus", projectId)).isFalse() + assertThat(isDeviceSupported("iphone13pro", "15.7", projectId)).isTrue() } @Test diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 1dac921de8..4afc8033ed 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -239,9 +239,9 @@ class IosRunCommandTest { @Test fun `device parse`() { val cmd = IosRunCommand() - CommandLine(cmd).parseArgs("--device=model=iphone8,version=11.2,locale=zh_CN,orientation=default") + CommandLine(cmd).parseArgs("--device=model=iphone13pro,version=15.7,locale=zh_CN,orientation=default") - val expectedDevice = Device("iphone8", "11.2", "zh_CN", "default") + val expectedDevice = Device("iphone13pro", "15.7", "zh_CN", "default") assertThat(cmd.config.common.gcloud.devices?.size).isEqualTo(1) assertThat(cmd.config.common.gcloud.devices?.first()).isEqualTo(expectedDevice) } diff --git a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt index cbbe9dc15e..678400ecc4 100644 --- a/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt +++ b/test_runner/src/test/kotlin/ftl/doctor/DoctorTest.kt @@ -137,7 +137,7 @@ Warning: Version should be string gcloud -> device[Nexus5] -> version[23] @Test fun iosDoctorTest() { val lint = validateYaml(IosArgs, Paths.get("src/test/kotlin/ftl/fixtures/flank.ios.yml")) - val expected = "Warning: Version should be string gcloud -> device[iphone8] -> version[11.2]" + val expected = "Warning: Version should be string gcloud -> device[iphone13pro] -> version[15.7]" assertEquals(expected, lint.summary()) } diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml index e92a7cd060..3557cd7806 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.gcs.yml @@ -9,8 +9,8 @@ gcloud: timeout: 60m async: true device: - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 orientation: portrait locale: en_US diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml index 3964265612..5c1254f960 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank.ios.yml @@ -6,8 +6,8 @@ gcloud: timeout: 60m async: true device: - - model: iphone8 - version: 11.2 + - model: iphone13pro + version: 15.7 orientation: portrait locale: en_US diff --git a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml index 02d70b61e0..a1ce6a92e1 100644 --- a/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml +++ b/test_runner/src/test/kotlin/ftl/fixtures/flank2.ios.yml @@ -6,8 +6,8 @@ gcloud: timeout: 60m async: true device: - - model: iphone8 - version: 13.6 + - model: iphone13pro + version: 15.7 orientation: portrait locale: en_US diff --git a/test_runner/src/test/kotlin/ftl/ios/IosCatalogTest.kt b/test_runner/src/test/kotlin/ftl/ios/IosCatalogTest.kt index 5097ef9a56..f9755347d2 100644 --- a/test_runner/src/test/kotlin/ftl/ios/IosCatalogTest.kt +++ b/test_runner/src/test/kotlin/ftl/ios/IosCatalogTest.kt @@ -17,7 +17,7 @@ class IosCatalogTest { @Test fun supportedXcode() { - assertThat(IosCatalog.supportedXcode("10.3", projectId)).isTrue() + assertThat(IosCatalog.supportedXcode("14.2", projectId)).isTrue() assertThat(IosCatalog.supportedXcode("0.1", projectId)).isFalse() } diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index d8f8dbf32b..87896f8b9d 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -204,7 +204,7 @@ class ReportManagerTest { assumeFalse(isWindows) // TODO investigate as to why the pathing fails here completely val legacyPath = File("results/2020-08-06_12-08-55.641213_jGpY/matrix_0/NexusLowRes-28-en-portrait/test_result_1.xml") - val iosPath = File("results/test_dir/shard_0/iphone8-12.0-en-portrait/test_result_0.xml") + val iosPath = File("results/test_dir/shard_0/iphone13pro-15.7-en-portrait/test_result_0.xml") assertEquals( "2020-08-06_12-08-55.641213_jGpY/matrix_0", legacyPath.getMatrixPath("2020-08-06_12-08-55.641213_jGpY") @@ -215,7 +215,7 @@ class ReportManagerTest { @Test fun `shouldn't contains multiple test_dir in MatrixPath`() { assumeFalse(isWindows) // TODO investigate as to why the pathing fails here completely - val path = File("results/test_dir/test_dir/shard_0/iphone8-12.0-en-portrait/test_result_0.xml") + val path = File("results/test_dir/test_dir/shard_0/iphone13-15.7-en-portrait/test_result_0.xml") assertEquals("test_dir/shard_0", path.getMatrixPath("test_dir")) } diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 110b90e146..bb9d566b4f 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -33,7 +33,7 @@ private const val FILE_NAME = "StandardOutputAndStandardError.txt" private const val SHARD = "shard_0" private const val MATRIX = "matrix_1" private const val ANDROID_DEVICE = "NexusLowRes-27-en-portrait-shard_0-rerun_1" -private const val IOS_DEVICE = "iphone8-12.0-en-portrait" +private const val IOS_DEVICE = "iphone13pro-15.7-en-portrait" private const val LOCAL_DIR = "results" private const val TEST_FILE_PATH = "any/path/that/is/possible" private const val FULL_IOS_PATH = "$OBJECT_NAME/$SHARD/$IOS_DEVICE/$TEST_FILE_PATH/$FILE_NAME"