From 2a2cc01328d43437af857c962253488313a7940c Mon Sep 17 00:00:00 2001 From: poinwater Date: Fri, 31 May 2024 22:44:01 -0700 Subject: [PATCH] Add macOS arm64 cicd --- .github/workflows/make_wheel_macOS_arm64.sh | 24 +++++++++++-------- .github/workflows/release.yml | 22 ++++++++++++++--- README.md | 11 ++++----- setup.py | 6 ----- .../kernel_tests/redis_table_ops_test.py | 12 ++++++++++ .../kernel_tests/redis_table_variable_test.py | 4 ++-- tensorflow_recommenders_addons/version.py | 2 +- 7 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/make_wheel_macOS_arm64.sh b/.github/workflows/make_wheel_macOS_arm64.sh index bac62cafe..2a415e185 100644 --- a/.github/workflows/make_wheel_macOS_arm64.sh +++ b/.github/workflows/make_wheel_macOS_arm64.sh @@ -1,15 +1,13 @@ #!/bin/bash # # Making wheel for macOS arm64 architecture -# Requirements: -# MacOS Monterey 12.0.0 +, Tensorflow-macos 2.6.0 - 2.11.0, ARM64 Apple Silicon, Bazel 5.1.1 -# Please don't install tensorflow-metal, it may cause incorrect GPU devices detection issue. set -e -x export TF_NEED_CUDA=0 -if [ -z $HOROVOD_VERSION ] ; then - export HOROVOD_VERSION='0.28.1' -fi +export IGNORE_HKV="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/hkv_hashtable_ops_test.py" +export IGNORE_REDIS="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py" +export IGNORE_REDIS_VAR="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py" +export USE_BAZEL_VERSION='5.1.1' # For TensorFlow version 2.12 or earlier: export PROTOBUF_VERSION=3.19.6 @@ -31,8 +29,6 @@ python -m pip install \ protobuf~=$PROTOBUF_VERSION $TF_NAME==$TF_VERSION python configure.py -# Setting DYLD_LIBRARY_PATH to help delocate finding tensorflow after the rpath invalidation -export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(python -c 'import configure; print(configure.get_tf_shared_lib_dir())') bazel build \ --cpu=darwin_arm64 \ @@ -44,5 +40,13 @@ bazel build \ --test_output=errors \ build_pip_pkg -bazel-bin/build_pip_pkg artifacts "--plat-name macosx_11_0_arm64 $NIGHTLY_FLAG" -delocate-wheel -w wheelhouse -v --ignore-missing-dependencies artifacts/*.whl \ No newline at end of file +bazel-bin/build_pip_pkg artifacts "--plat-name macosx_12_0_arm64 $NIGHTLY_FLAG" +delocate-wheel -w wheelhouse -v --ignore-missing-dependencies artifacts/*.whl + +# Test +pip install --default-timeout=1000 -r tools/install_deps/pytest.txt +cp ./bazel-bin/tensorflow_recommenders_addons/dynamic_embedding/core/_*_ops.so ./tensorflow_recommenders_addons/dynamic_embedding/core/ +python -m pytest -v -s --functions-durations=20 --modules-durations=5 $IGNORE_HKV $IGNORE_REDIS $IGNORE_REDIS_VAR $SKIP_CUSTOM_OP_TESTS ./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/ + +# Clean +bazel clean \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4215515f..7e46eac10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,11 +42,11 @@ jobs: strategy: matrix: # TODO: add back 'windows-latest' when it can be compiled. - os: ['macos-12', 'ubuntu-20.04'] + os: ['macos-14', 'macos-12', 'ubuntu-20.04'] py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] tf-version: ['2.11.0', '2.15.1'] tf-need-cuda: ['0'] - cpu: ['x86'] + cpu: ['x86', 'arm64'] exclude: # excludes cuda on macOS - tf-version: '2.11.0' @@ -55,6 +55,16 @@ jobs: py-version: '3.7' - tf-version: '2.15.1' py-version: '3.8' + - os: 'macos-14' + cpu: 'x86' + - os: 'ubuntu-20.04' + cpu: 'arm64' + - os: 'macos-12' + cpu: 'arm64' + - py-version: '3.7' + cpu: 'arm64' + - py-version: '3.8' + cpu: 'arm64' fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -184,7 +194,7 @@ jobs: py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] tf-version: ['2.15.1'] tf-need-cuda: ['0', '1'] - cpu: ['x86'] + cpu: ['x86', 'arm64'] exclude: # excludes cuda on macOS - os: 'macOS' @@ -195,6 +205,12 @@ jobs: py-version: '3.7' - tf-version: '2.15.1' py-version: '3.8' + - os: 'Linux' + cpu: 'arm64' + - py-version: '3.7' + cpu: 'arm64' + - py-version: '3.8' + cpu: 'arm64' fail-fast: false if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || github.event_name == 'release' steps: diff --git a/README.md b/README.md index 9b524f98d..963ad93ff 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ makes building, evaluating, and serving sophisticated recommenders models easy. See approved TensorFlow RFC #[313](https://github.com/tensorflow/community/pull/313). Those contributions will be complementary to TensorFlow Core and TensorFlow Recommenders etc. -For Apple silicon(M1), please refer to [Apple Silicon Support](#apple-silicon-support-beta-release). +For Apple silicon(M1), please refer to [Apple Silicon Support](#apple-silicon-support). ## Main Features @@ -200,14 +200,12 @@ pytest -s tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/h Requirements: - macOS 12.0.0+ -- Python 3.9 -- tensorflow-macos 2.9.0 +- tensorflow 2.15.1 - bazel 5.1.1 -The natively supported TensorFlow is maintained by Apple. Please see the instruction [Get started with tensorflow-metal](https://developer.apple.com/metal/tensorflow-plugin/) to install the Tensorflow on apple silicon devices. -**Install TFRA on Apple Silicon via PIP** +**Install TFRA on Apple Silicon via Pypi** ```sh python -m pip install tensorflow-recommenders-addons --no-deps ``` @@ -219,7 +217,7 @@ python -m pip install tensorflow-recommenders-addons --no-deps brew install bazelisk # Build wheel from source -PY_VERSION=3.9.0 TF_VERSION=2.9.0 TF_NEED_CUDA="0" sh .github/workflows/make_wheel_macOS_arm64.sh +TF_VERSION=2.15.1 TF_NEED_CUDA="0" sh .github/workflows/make_wheel_macOS_arm64.sh # Install the wheel python -m pip install --no-deps ./artifacts/*.whl @@ -231,6 +229,7 @@ The Apple silicon version of TFRA doesn't support: * Data type **float16** * Synchronous training based on **Horovod** +* HierarchicalKV (HKV) * `save_to_file_system` * `load_from_file_system` * `warm_start_util` diff --git a/setup.py b/setup.py index fe1134541..50475f4d7 100644 --- a/setup.py +++ b/setup.py @@ -55,12 +55,6 @@ def get_project_name_version(): project_name = "tensorflow-recommenders-addons" version["tf_project_name"] = "tensorflow" - - is_macos_arm64 = (platform.machine() == "arm64" - and platform.system() == "Darwin") - if is_macos_arm64 and Version(os.getenv("TF_VERSION")) <= Version("2.12.0"): - version["tf_project_name"] = "tensorflow-macos" - if os.getenv("TF_NEED_CUDA", "0") == "1": project_name = project_name + "-gpu" diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py index 83c7caf5a..585dc8568 100755 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py @@ -366,6 +366,10 @@ def _convert(v, t): return np.array(v).astype(_type_converter(t)) for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list): + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16: + continue id += 1 with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: @@ -724,6 +728,10 @@ def test_training_save_restore(self): dim_list, [10], ): + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16: + continue id += 1 save_dir = os.path.join(self.get_temp_dir(), "save_restore") save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") @@ -819,6 +827,10 @@ def test_training_save_restore_by_files(self): [10], [10], ): + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16: + continue id += 1 save_dir = os.path.join(self.get_temp_dir(), "save_restore") save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash") diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py index a17fc1a7c..951483de9 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py @@ -368,7 +368,7 @@ def _convert(v, t): id += 1 # Skip float16 tests if the platform is macOS arm64 architecture if is_macos() and is_arm64(): - if value_dtype == dtypes.half: + if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16: continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: @@ -445,7 +445,7 @@ def _convert(v, t): id += 1 # Skip float16 tests if the platform is macOS arm64 architecture if is_macos() and is_arm64(): - if value_dtype == dtypes.half: + if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16: continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: diff --git a/tensorflow_recommenders_addons/version.py b/tensorflow_recommenders_addons/version.py index f460f36d9..0170db78e 100644 --- a/tensorflow_recommenders_addons/version.py +++ b/tensorflow_recommenders_addons/version.py @@ -28,7 +28,7 @@ def is_arm64(): # Required TensorFlow version [min, max] if (is_macos() and is_arm64()): - MIN_TF_VERSION = "2.11.0" + MIN_TF_VERSION = "2.15.1" MAX_TF_VERSION = "2.15.1" else: MIN_TF_VERSION = "2.11.0"