From e95be72ae62032879038139a1487ae6c6fff7e4d Mon Sep 17 00:00:00 2001 From: Caroline Zhu Date: Wed, 6 Nov 2024 11:22:29 -0800 Subject: [PATCH] [Mobile] Add E2E BrowserStack tests for iOS tests (#22610) ### Description - Changes running the E2E iOS tests from running in App Center to running in BrowserStack - Steps for running locally can be found in the OneNote ### Motivation and Context - Follow-up of #22117 - App Center (the previous platform for running E2E mobile tests) is getting deprecated in 2025 ### Misc info Additional build steps were required to get the necessary testing artifacts for BrowserStack. App Center consumed an entire folder, while BrowserStack requests the following: 1. a ZIP file of all the tests 2. an IPA file of the test app #### Flow Here is a rough outline of what is happening in the pipeline: 1. The build_and_assemble_apple_pods.py script builds the relevant frameworks (currently, this means packages for iOS and Mac) 4. The test_apple_packages.py script installs the necessary cocoapods for later steps 5. XCode task to build for testing builds the iOS target for the test app 6. Now that the test app and the tests have been built, we can zip them, creating the tests .zip file 7. To create the IPA file, we need to create a .plist XML file which is generated by the generate_plist.py script. - Attempts to use the Xcode@5 task to automatically generate the plist file failed. - Also, building for testing generates some plist files -- these cannot be used to export an IPA file. 8. We run the Xcode task to build an .xcarchive file, which is required for creating an IPA file. 9. We use xcodebuild in a script step to build an IPA file with the xcarchive and plist files from the last two steps. 10. Finally, we can run the tests using the BrowserStack script. --------- Co-authored-by: Scott McKay Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- .../generate_ipa_export_options_plist.py | 54 +++++++++++++ .../templates/android-java-api-aar-test.yml | 4 +- .../templates/install-appcenter.yml | 12 --- .../stages/mac-ios-packaging-build-stage.yml | 75 ++++++++++++++++--- .../upload_and_run_browserstack_tests.py | 18 +++-- 5 files changed, 129 insertions(+), 34 deletions(-) create mode 100644 onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py delete mode 100644 tools/ci_build/github/azure-pipelines/templates/install-appcenter.yml diff --git a/onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py b/onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py new file mode 100644 index 0000000000000..4e5329dd5b09a --- /dev/null +++ b/onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py @@ -0,0 +1,54 @@ +import argparse + +plist_file_content = """ + + + + + method + development + teamID + {team_id} + provisioningProfiles + + ai.onnxruntime.tests.ios-package-test + {provisioning_profile_uuid} + + signingStyle + manual + + +""" +if __name__ == "__main__": + # handle cli args + parser = argparse.ArgumentParser( + "Generates a PList file to the relevant destination. This PList file contains the properties to allow a user to generate an IPA file for the ios-package-test. " + ) + + parser.add_argument("--dest_file", type=str, help="Path to output the PList file to.", required=True) + parser.add_argument( + "--apple_team_id", + type=str, + help="The Team ID associated with the provisioning profile. You should be able to find this from the Apple developer portal under Membership.", + required=True, + ) + parser.add_argument( + "--provisioning_profile_uuid", + type=str, + help="The Provisioning Profile UUID, which can be found in the .mobileprovision file. ", + required=True, + ) + + args = parser.parse_args() + + formatted_plist = plist_file_content.format( + team_id=args.apple_team_id, provisioning_profile_uuid=args.provisioning_profile_uuid + ) + + with open(args.dest_file, "w") as file: + file.write(formatted_plist) + + print("Wrote plist file to ", args.dest_file) + print() + print("Contents of file:") + print(formatted_plist) diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml index e162365c40ce7..13236453f9906 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml @@ -77,8 +77,8 @@ jobs: pip install requests python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \ --test_platform espresso \ - --app_apk_path "debug/app-debug.apk" \ - --test_apk_path "androidTest/debug/app-debug-androidTest.apk" \ + --app_path "debug/app-debug.apk" \ + --test_path "androidTest/debug/app-debug-androidTest.apk" \ --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0" displayName: Run E2E tests using Browserstack workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk diff --git a/tools/ci_build/github/azure-pipelines/templates/install-appcenter.yml b/tools/ci_build/github/azure-pipelines/templates/install-appcenter.yml deleted file mode 100644 index 51be73d4c658a..0000000000000 --- a/tools/ci_build/github/azure-pipelines/templates/install-appcenter.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Install appcenter CLI - -parameters: -- name: appcenterVersion - type: string - default: "2.13.7" - -steps: -- bash: | - set -e -x - npm install -g appcenter-cli@${{ parameters.appcenterVersion }} - displayName: Install appcenter CLI ${{ parameters.appcenterVersion }} diff --git a/tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml b/tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml index a3b6bc1025267..b6a214154e680 100644 --- a/tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/stages/mac-ios-packaging-build-stage.yml @@ -70,8 +70,6 @@ stages: parameters: xcodeVersion: $(xcodeVersion) - - template: ../install-appcenter.yml - - script: | pip install -r tools/ci_build/github/apple/ios_packaging/requirements.txt displayName: "Install Python requirements" @@ -100,6 +98,8 @@ stages: --prepare_test_project_only displayName: "Assemble test project for App Center" + # Xcode tasks require absolute paths because it searches for the paths and files relative to + # the root directory and not relative to the working directory - task: Xcode@5 inputs: actions: 'build-for-testing' @@ -107,8 +107,6 @@ stages: xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/apple_package_test.xcworkspace' sdk: 'iphoneos' scheme: 'ios_package_test' - xcodeVersion: 'specifyPath' - xcodeDeveloperDir: '/Applications/Xcode_${{ variables.xcodeVersion }}.app/Contents/Developer' signingOption: 'manual' signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)' provisioningProfileUuid: '$(APPLE_PROV_PROFILE_UUID)' @@ -117,16 +115,69 @@ stages: useXcpretty: false # xcpretty can hide useful error output so we will disable it displayName: 'Build App Center iPhone arm64 tests' + - script: | + zip -r --symlinks $(Build.ArtifactStagingDirectory)/package_tests.zip ios_package_testUITests-Runner.app + workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData/Build/Products/Debug-iphoneos' + displayName: "Create .zip file of the tests" + + - script: | + python $(Build.SourcesDirectory)/onnxruntime/test/platform/apple/generate_ipa_export_options_plist.py \ + --dest_file "exportOptions.plist" \ + --apple_team_id $(APPLE_TEAM_ID) \ + --provisioning_profile_uuid $(APPLE_PROV_PROFILE_UUID) + workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/' + displayName: "Generate .plist file for the .ipa file" + + # Task only generates an .xcarchive file if the plist export options are included, but does + # not produce an IPA file. + # Source code: https://github.com/microsoft/azure-pipelines-tasks/blob/master/Tasks/XcodeV5/xcode.ts + - task: Xcode@5 + inputs: + actions: 'archive' + xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/apple_package_test.xcworkspace' + packageApp: true + archivePath: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/' + exportOptions: 'plist' + exportOptionsPlist: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/exportOptions.plist' + configuration: 'Debug' + sdk: 'iphoneos' + scheme: 'ios_package_test' + args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData' + workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/' + useXcpretty: false + displayName: 'Create archive for the .ipa file' + + # Use script step because exporting the .ipa file using the Xcode@5 task was too brittle (Xcode@5 is designed + # to handle both the .xcarchive step and the .ipa step in the same step -- ran into countless issues with signing + # and the .plist file) + - script: | + xcodebuild -exportArchive \ + -archivePath ios_package_test.xcarchive \ + -exportOptionsPlist exportOptions.plist \ + -exportPath $(Build.ArtifactStagingDirectory)/test_ipa + workingDirectory: '$(Build.BinariesDirectory)/app_center_test/apple_package_test/' + displayName: "Create .ipa file" + + # Publish the BrowserStack artifacts first so that if the next step fails, the artifacts will still be published + # so that users can attempt to locally debug + - publish: "$(Build.ArtifactStagingDirectory)" + artifact: "browserstack_test_artifacts" + displayName: "Publish BrowserStack test artifacts" + - script: | set -e -x - appcenter test run xcuitest \ - --app "AI-Frameworks/ORT-Mobile-iOS" \ - --devices $(app_center_test_devices) \ - --test-series "master" \ - --locale "en_US" \ - --build-dir $(Build.BinariesDirectory)/app_center_test/apple_package_test/DerivedData/Build/Products/Debug-iphoneos \ - --token $(app_center_api_token) - displayName: "Run E2E tests on App Center" + pip install requests + python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \ + --test_platform xcuitest \ + --app_path "$(Build.ArtifactStagingDirectory)/test_ipa/ios_package_test.ipa" \ + --test_path "$(Build.ArtifactStagingDirectory)/package_tests.zip" \ + --devices "iPhone 15-17" + displayName: Run E2E tests using Browserstack + workingDirectory: $(Build.BinariesDirectory)/app_center_test/apple_package_test + timeoutInMinutes: 15 + env: + BROWSERSTACK_ID: $(browserstack_username) + BROWSERSTACK_TOKEN: $(browserstack_access_key) - script: | set -e -x diff --git a/tools/python/upload_and_run_browserstack_tests.py b/tools/python/upload_and_run_browserstack_tests.py index 8751368e1b2fc..9b812da156707 100644 --- a/tools/python/upload_and_run_browserstack_tests.py +++ b/tools/python/upload_and_run_browserstack_tests.py @@ -78,22 +78,24 @@ def build_query_loop(build_id, test_platform, id, token): "--test_platform", type=str, help="Testing platform", choices=["espresso", "xcuitest"], required=True ) parser.add_argument( - "--app_apk_path", + "--app_path", type=Path, help=( - "Path to the app APK. " - "Typically, the app APK is in " + "Path to the app file. " + "For Android, typically, the app file (the APK) is in " "{build_output_dir}/android_test/android/app/build/outputs/apk/debug/app-debug.apk" + ". For iOS, you will have to build an IPA file from the test app, which is built from the .xcarchive path" ), required=True, ) parser.add_argument( - "--test_apk_path", + "--test_path", type=Path, help=( - "Path to the test APK. " + "Path to the test suite file. " "Typically, the test APK is in " "{build_output_dir}/android_test/android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk" + ". For iOS, you will have to create a .zip of the tests. After manually building the tests, the tests that you need to zip will be in {{Xcode DerivedData Folder Path}}/Build/Products" ), required=True, ) @@ -102,7 +104,7 @@ def build_query_loop(build_id, test_platform, id, token): type=str, nargs="+", help="List of devices to run the tests on. For more info, " - "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices", + "see https://www.browserstack.com/docs/app-automate/espresso/specify-devices (Android) or https://www.browserstack.com/docs/app-automate/xcuitest/specify-devices (iOS)", required=True, ) @@ -121,13 +123,13 @@ def build_query_loop(build_id, test_platform, id, token): # Upload the app and test suites upload_app_json = upload_apk_parse_json( f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/app", - args.app_apk_path, + args.app_path, browserstack_id, browserstack_token, ) upload_test_json = upload_apk_parse_json( f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/test-suite", - args.test_apk_path, + args.test_path, browserstack_id, browserstack_token, )