From a2b433b531b5585c12b956073965f7553da41304 Mon Sep 17 00:00:00 2001 From: vraspar Date: Sat, 27 Jul 2024 23:49:16 -0700 Subject: [PATCH 1/7] Refactor build_apple_framework.py to support macOS framework structure --- .../github/apple/build_apple_framework.py | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index 3cd7a3af70622..749bc97898488 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -89,18 +89,48 @@ def _build_for_apple_sysroot( pathlib.Path(framework_dir).mkdir(parents=True, exist_ok=True) # copy the Info.plist, framework_info.json, and header files - shutil.copy(info_plist_path, framework_dir) - shutil.copy(framework_info_path, os.path.dirname(framework_dir)) - header_dir = os.path.join(framework_dir, "Headers") - pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) - for _header in headers: - shutil.copy(_header, header_dir) - - # use lipo to create a fat ort library - lipo_command = ["lipo", "-create"] - lipo_command += ort_libs - lipo_command += ["-output", os.path.join(framework_dir, "onnxruntime")] - subprocess.run(lipo_command, shell=False, check=True) + + # macos requires different framework structure: + # https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html + if sysroot == "macosx": + # create headers and resources directory + header_dir = os.path.join(framework_dir, "Versions", "A", "Headers") + resource_dir = os.path.join(framework_dir, "Versions", "A", "Resources") + pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) + pathlib.Path(resource_dir).mkdir(parents=True, exist_ok=True) + + shutil.copy(info_plist_path, resource_dir) + shutil.copy(framework_info_path, os.path.dirname(framework_dir)) + + for _header in headers: + shutil.copy(_header, header_dir) + + # use lipo to create a fat ort library + lipo_command = ["lipo", "-create"] + lipo_command += ort_libs + lipo_command += ["-output", os.path.join(framework_dir, "Versions", "A", "onnxruntime")] + subprocess.run(lipo_command, shell=False, check=True) + + # create the symbolic link + pathlib.Path(os.path.join(framework_dir, "Versions", "Current")).symlink_to("A", target_is_directory=True) + pathlib.Path(os.path.join(framework_dir, "Headers")).symlink_to("Versions/Current/Headers") + pathlib.Path(os.path.join(framework_dir, "Resources")).symlink_to("Versions/Current/Resources") + pathlib.Path(os.path.join(framework_dir, "onnxruntime")).symlink_to("Versions/Current/onnxruntime") + + else: + shutil.copy(info_plist_path, framework_dir) + shutil.copy(framework_info_path, os.path.dirname(framework_dir)) + header_dir = os.path.join(framework_dir, "Headers") + pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) + + for _header in headers: + shutil.copy(_header, header_dir) + + # use lipo to create a fat ort library + lipo_command = ["lipo", "-create"] + lipo_command += ort_libs + lipo_command += ["-output", os.path.join(framework_dir, "onnxruntime")] + subprocess.run(lipo_command, shell=False, check=True) return framework_dir @@ -163,6 +193,7 @@ def _build_package(args): public_headers_path = os.path.join(os.path.dirname(framework_dir), "onnxruntime.framework", "Headers") # create the folder for xcframework and copy the LICENSE and framework_info.json file + # changes here xcframework_dir = os.path.join(build_dir, "framework_out") pathlib.Path(xcframework_dir).mkdir(parents=True, exist_ok=True) shutil.copy(os.path.join(REPO_DIR, "LICENSE"), xcframework_dir) From 633ae76df2b8966bd11de3818702497fe07e312a Mon Sep 17 00:00:00 2001 From: vraspar Date: Mon, 29 Jul 2024 11:38:08 -0700 Subject: [PATCH 2/7] Update zip command to store symbolic links --- .../ci_build/github/apple/assemble_apple_packaging_artifacts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh index 317048506ac67..c796763a639ad 100755 --- a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh +++ b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh @@ -26,7 +26,7 @@ PODSPEC_BASENAME="${POD_NAME}.podspec" pushd "${BINARIES_STAGING_DIR}/${POD_NAME}" # assemble the files in the artifacts staging directory -zip -r "${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME}" ./* --exclude "${PODSPEC_BASENAME}" +zip -r -y "${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME}" ./* --exclude "${PODSPEC_BASENAME}" cp "${PODSPEC_BASENAME}" "${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME}" popd From 2e790c6430a6821fabc43e54bb08d18c595790aa Mon Sep 17 00:00:00 2001 From: vraspar Date: Mon, 29 Jul 2024 14:05:52 -0700 Subject: [PATCH 3/7] symlink correctly --- tools/ci_build/github/apple/build_apple_framework.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index 749bc97898488..b92baa3da6872 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -113,8 +113,12 @@ def _build_for_apple_sysroot( # create the symbolic link pathlib.Path(os.path.join(framework_dir, "Versions", "Current")).symlink_to("A", target_is_directory=True) - pathlib.Path(os.path.join(framework_dir, "Headers")).symlink_to("Versions/Current/Headers") - pathlib.Path(os.path.join(framework_dir, "Resources")).symlink_to("Versions/Current/Resources") + pathlib.Path(os.path.join(framework_dir, "Headers")).symlink_to( + "Versions/Current/Headers", target_is_directory=True + ) + pathlib.Path(os.path.join(framework_dir, "Resources")).symlink_to( + "Versions/Current/Resources", target_is_directory=True + ) pathlib.Path(os.path.join(framework_dir, "onnxruntime")).symlink_to("Versions/Current/onnxruntime") else: From e8e6b31b138b0754ff2ed9d4888261c1e75748ae Mon Sep 17 00:00:00 2001 From: vraspar Date: Thu, 1 Aug 2024 13:01:41 -0700 Subject: [PATCH 4/7] Refactor build_and_assemble_apple_pods.py for debugging --- .../ci_build/github/apple/build_and_assemble_apple_pods.py | 6 ++++++ tools/ci_build/github/apple/build_apple_framework.py | 7 +++++++ tools/ci_build/github/apple/c/assemble_c_pod_package.py | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 5014ba11d983d..9410462a01f3a 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -116,6 +116,9 @@ def main(): run(build_apple_framework_args) + print("framework Out: ", str(build_dir / "framework_out")) + run(["ls", "-LR", str(build_dir / "framework_out")]) + if args.test: test_apple_packages_args = [ sys.executable, @@ -148,6 +151,9 @@ def main(): package_variant=package_variant, ) + print("C pod dir:", c_pod_staging_dir) + run(["ls", "-LR", str(c_pod_staging_dir)]) + if args.test: test_c_pod_args = ["pod", "lib", "lint", "--verbose"] diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index b92baa3da6872..1c6ef7562518b 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -216,6 +216,13 @@ def _build_package(args): subprocess.run(build_xcframework_cmd, shell=False, check=True, cwd=REPO_DIR) + # For debugging + print("XCFramework dir:", xcframework_dir) + result = subprocess.run( + ["ls", "-lR"], shell=False, check=True, cwd=xcframework_dir, stdout=subprocess.PIPE, text=True + ) + print(result.stdout) + def parse_args(): parser = argparse.ArgumentParser( diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index ca4f01cf65bd9..dc4e14872267b 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -66,8 +66,8 @@ def assemble_c_pod_package( print("Warning: staging directory already exists", file=sys.stderr) # copy the necessary files to the staging directory - shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) - shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True) + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True, symlinks=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True, symlinks=True) copy_repo_relative_to_dir(["LICENSE"], staging_dir) # generate the podspec file from the template From 3f2632e7c3eca795389a0dd468eeb3c98ca7446e Mon Sep 17 00:00:00 2001 From: vraspar Date: Thu, 1 Aug 2024 23:02:52 -0700 Subject: [PATCH 5/7] add symlinks before zipping the files and cleanup --- .../assemble_apple_packaging_artifacts.sh | 23 +++++++++++++++++++ .../apple/build_and_assemble_apple_pods.py | 6 ----- .../github/apple/build_apple_framework.py | 7 ------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh index c796763a639ad..b8f5d0a869aa0 100755 --- a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh +++ b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh @@ -23,6 +23,29 @@ ORT_POD_VERSION=${4:?${USAGE_TEXT}} POD_ARCHIVE_BASENAME="pod-archive-${POD_NAME}-${ORT_POD_VERSION}.zip" PODSPEC_BASENAME="${POD_NAME}.podspec" +# Check for directories starting with "macos" and create symlinks if necessary +for MACOS_DIR in "${BINARIES_STAGING_DIR}/${POD_NAME}/onnxruntime.xcframework/macos"*; do + if [ -d "${MACOS_DIR}" ]; then + echo "Creating symlinks for ${MACOS_DIR}" + pushd "${MACOS_DIR}/onnxruntime.framework" + + rm -rf Headers Resources onnxruntime + rm -rf Versions/Current + + ln -sfn A Versions/Current + ln -sfn Versions/Current/Headers Headers + ln -sfn Versions/Current/Resources Resources + ln -sfn Versions/Current/onnxruntime onnxruntime + + popd + + fi +done + + +echo "Contents of ${BINARIES_STAGING_DIR}/${POD_NAME}:" +ls -lR "${BINARIES_STAGING_DIR}/${POD_NAME}" + pushd "${BINARIES_STAGING_DIR}/${POD_NAME}" # assemble the files in the artifacts staging directory diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py index 9410462a01f3a..5014ba11d983d 100755 --- a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -116,9 +116,6 @@ def main(): run(build_apple_framework_args) - print("framework Out: ", str(build_dir / "framework_out")) - run(["ls", "-LR", str(build_dir / "framework_out")]) - if args.test: test_apple_packages_args = [ sys.executable, @@ -151,9 +148,6 @@ def main(): package_variant=package_variant, ) - print("C pod dir:", c_pod_staging_dir) - run(["ls", "-LR", str(c_pod_staging_dir)]) - if args.test: test_c_pod_args = ["pod", "lib", "lint", "--verbose"] diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index 1c6ef7562518b..b92baa3da6872 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -216,13 +216,6 @@ def _build_package(args): subprocess.run(build_xcframework_cmd, shell=False, check=True, cwd=REPO_DIR) - # For debugging - print("XCFramework dir:", xcframework_dir) - result = subprocess.run( - ["ls", "-lR"], shell=False, check=True, cwd=xcframework_dir, stdout=subprocess.PIPE, text=True - ) - print(result.stdout) - def parse_args(): parser = argparse.ArgumentParser( From 27b2006a19cb0188b49a46cc05dfa7a188e19740 Mon Sep 17 00:00:00 2001 From: vraspar Date: Fri, 2 Aug 2024 01:30:17 -0700 Subject: [PATCH 6/7] remove changes for assemble pods and set symlinks for macabi --- tools/ci_build/github/apple/build_apple_framework.py | 2 +- tools/ci_build/github/apple/c/assemble_c_pod_package.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index b92baa3da6872..f4982ffa5d7e2 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -92,7 +92,7 @@ def _build_for_apple_sysroot( # macos requires different framework structure: # https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html - if sysroot == "macosx": + if sysroot == "macosx" or sysroot == "macabi": # create headers and resources directory header_dir = os.path.join(framework_dir, "Versions", "A", "Headers") resource_dir = os.path.join(framework_dir, "Versions", "A", "Resources") diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index dc4e14872267b..ca4f01cf65bd9 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -66,8 +66,8 @@ def assemble_c_pod_package( print("Warning: staging directory already exists", file=sys.stderr) # copy the necessary files to the staging directory - shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True, symlinks=True) - shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True, symlinks=True) + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True) copy_repo_relative_to_dir(["LICENSE"], staging_dir) # generate the podspec file from the template From 9b27f009a8805a8e11922da34959d9a7bbf4949a Mon Sep 17 00:00:00 2001 From: vraspar Date: Fri, 2 Aug 2024 14:51:54 -0700 Subject: [PATCH 7/7] Add comments --- .../github/apple/assemble_apple_packaging_artifacts.sh | 5 ++++- tools/ci_build/github/apple/build_apple_framework.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh index b8f5d0a869aa0..f96227a750346 100755 --- a/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh +++ b/tools/ci_build/github/apple/assemble_apple_packaging_artifacts.sh @@ -23,7 +23,10 @@ ORT_POD_VERSION=${4:?${USAGE_TEXT}} POD_ARCHIVE_BASENAME="pod-archive-${POD_NAME}-${ORT_POD_VERSION}.zip" PODSPEC_BASENAME="${POD_NAME}.podspec" -# Check for directories starting with "macos" and create symlinks if necessary + +# Macos requires a different structure for the framework +# This will create the necessary symlinks for the macos framework before packaging +# Adding the symlinks here rather than in the build script ensures that symlinks are not lost for MACOS_DIR in "${BINARIES_STAGING_DIR}/${POD_NAME}/onnxruntime.xcframework/macos"*; do if [ -d "${MACOS_DIR}" ]; then echo "Creating symlinks for ${MACOS_DIR}" diff --git a/tools/ci_build/github/apple/build_apple_framework.py b/tools/ci_build/github/apple/build_apple_framework.py index f4982ffa5d7e2..7270bdd56523c 100644 --- a/tools/ci_build/github/apple/build_apple_framework.py +++ b/tools/ci_build/github/apple/build_apple_framework.py @@ -197,7 +197,6 @@ def _build_package(args): public_headers_path = os.path.join(os.path.dirname(framework_dir), "onnxruntime.framework", "Headers") # create the folder for xcframework and copy the LICENSE and framework_info.json file - # changes here xcframework_dir = os.path.join(build_dir, "framework_out") pathlib.Path(xcframework_dir).mkdir(parents=True, exist_ok=True) shutil.copy(os.path.join(REPO_DIR, "LICENSE"), xcframework_dir)