Skip to content

Commit

Permalink
Add macOS arm64 cicd
Browse files Browse the repository at this point in the history
  • Loading branch information
poinwater authored and rhdong committed Jun 1, 2024
1 parent 2d0f704 commit 2a2cc01
Show file tree
Hide file tree
Showing 7 changed files with 53 additions and 28 deletions.
24 changes: 14 additions & 10 deletions .github/workflows/make_wheel_macOS_arm64.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
#!/bin/bash
#
# Making wheel for macOS arm64 architecture
# Requirements:
# MacOS Monterey 12.0.0 +, Tensorflow-macos 2.6.0 - 2.11.0, ARM64 Apple Silicon, Bazel 5.1.1
# Please don't install tensorflow-metal, it may cause incorrect GPU devices detection issue.
set -e -x

export TF_NEED_CUDA=0
if [ -z $HOROVOD_VERSION ] ; then
export HOROVOD_VERSION='0.28.1'
fi
export IGNORE_HKV="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/hkv_hashtable_ops_test.py"
export IGNORE_REDIS="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_ops_test.py"
export IGNORE_REDIS_VAR="--ignore=./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py"
export USE_BAZEL_VERSION='5.1.1'

# For TensorFlow version 2.12 or earlier:
export PROTOBUF_VERSION=3.19.6
Expand All @@ -31,8 +29,6 @@ python -m pip install \
protobuf~=$PROTOBUF_VERSION $TF_NAME==$TF_VERSION

python configure.py
# Setting DYLD_LIBRARY_PATH to help delocate finding tensorflow after the rpath invalidation
export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(python -c 'import configure; print(configure.get_tf_shared_lib_dir())')

bazel build \
--cpu=darwin_arm64 \
Expand All @@ -44,5 +40,13 @@ bazel build \
--test_output=errors \
build_pip_pkg

bazel-bin/build_pip_pkg artifacts "--plat-name macosx_11_0_arm64 $NIGHTLY_FLAG"
delocate-wheel -w wheelhouse -v --ignore-missing-dependencies artifacts/*.whl
bazel-bin/build_pip_pkg artifacts "--plat-name macosx_12_0_arm64 $NIGHTLY_FLAG"
delocate-wheel -w wheelhouse -v --ignore-missing-dependencies artifacts/*.whl

# Test
pip install --default-timeout=1000 -r tools/install_deps/pytest.txt
cp ./bazel-bin/tensorflow_recommenders_addons/dynamic_embedding/core/_*_ops.so ./tensorflow_recommenders_addons/dynamic_embedding/core/
python -m pytest -v -s --functions-durations=20 --modules-durations=5 $IGNORE_HKV $IGNORE_REDIS $IGNORE_REDIS_VAR $SKIP_CUSTOM_OP_TESTS ./tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/

# Clean
bazel clean
22 changes: 19 additions & 3 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ jobs:
strategy:
matrix:
# TODO: add back 'windows-latest' when it can be compiled.
os: ['macos-12', 'ubuntu-20.04']
os: ['macos-14', 'macos-12', 'ubuntu-20.04']
py-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
tf-version: ['2.11.0', '2.15.1']
tf-need-cuda: ['0']
cpu: ['x86']
cpu: ['x86', 'arm64']
exclude:
# excludes cuda on macOS
- tf-version: '2.11.0'
Expand All @@ -55,6 +55,16 @@ jobs:
py-version: '3.7'
- tf-version: '2.15.1'
py-version: '3.8'
- os: 'macos-14'
cpu: 'x86'
- os: 'ubuntu-20.04'
cpu: 'arm64'
- os: 'macos-12'
cpu: 'arm64'
- py-version: '3.7'
cpu: 'arm64'
- py-version: '3.8'
cpu: 'arm64'
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
Expand Down Expand Up @@ -184,7 +194,7 @@ jobs:
py-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
tf-version: ['2.15.1']
tf-need-cuda: ['0', '1']
cpu: ['x86']
cpu: ['x86', 'arm64']
exclude:
# excludes cuda on macOS
- os: 'macOS'
Expand All @@ -195,6 +205,12 @@ jobs:
py-version: '3.7'
- tf-version: '2.15.1'
py-version: '3.8'
- os: 'Linux'
cpu: 'arm64'
- py-version: '3.7'
cpu: 'arm64'
- py-version: '3.8'
cpu: 'arm64'
fail-fast: false
if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || github.event_name == 'release'
steps:
Expand Down
11 changes: 5 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ makes building, evaluating, and serving sophisticated recommenders models easy.
See approved TensorFlow RFC #[313](https://github.com/tensorflow/community/pull/313).
Those contributions will be complementary to TensorFlow Core and TensorFlow Recommenders etc.

For Apple silicon(M1), please refer to [Apple Silicon Support](#apple-silicon-support-beta-release).
For Apple silicon(M1), please refer to [Apple Silicon Support](#apple-silicon-support).

## Main Features

Expand Down Expand Up @@ -200,14 +200,12 @@ pytest -s tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/h
Requirements:

- macOS 12.0.0+
- Python 3.9
- tensorflow-macos 2.9.0
- tensorflow 2.15.1
- bazel 5.1.1

The natively supported TensorFlow is maintained by Apple. Please see the instruction [Get started with tensorflow-metal](https://developer.apple.com/metal/tensorflow-plugin/) to install the Tensorflow on apple silicon devices.


**Install TFRA on Apple Silicon via PIP**
**Install TFRA on Apple Silicon via Pypi**
```sh
python -m pip install tensorflow-recommenders-addons --no-deps
```
Expand All @@ -219,7 +217,7 @@ python -m pip install tensorflow-recommenders-addons --no-deps
brew install bazelisk

# Build wheel from source
PY_VERSION=3.9.0 TF_VERSION=2.9.0 TF_NEED_CUDA="0" sh .github/workflows/make_wheel_macOS_arm64.sh
TF_VERSION=2.15.1 TF_NEED_CUDA="0" sh .github/workflows/make_wheel_macOS_arm64.sh

# Install the wheel
python -m pip install --no-deps ./artifacts/*.whl
Expand All @@ -231,6 +229,7 @@ The Apple silicon version of TFRA doesn't support:

* Data type **float16**
* Synchronous training based on **Horovod**
* HierarchicalKV (HKV)
* `save_to_file_system`
* `load_from_file_system`
* `warm_start_util`
Expand Down
6 changes: 0 additions & 6 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,6 @@ def get_project_name_version():

project_name = "tensorflow-recommenders-addons"
version["tf_project_name"] = "tensorflow"

is_macos_arm64 = (platform.machine() == "arm64"
and platform.system() == "Darwin")
if is_macos_arm64 and Version(os.getenv("TF_VERSION")) <= Version("2.12.0"):
version["tf_project_name"] = "tensorflow-macos"

if os.getenv("TF_NEED_CUDA", "0") == "1":
project_name = project_name + "-gpu"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,10 @@ def _convert(v, t):
return np.array(v).astype(_type_converter(t))

for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list):
# Skip float16 tests if the platform is macOS arm64 architecture
if is_macos() and is_arm64():
if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16:
continue
id += 1
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()) as sess:
Expand Down Expand Up @@ -724,6 +728,10 @@ def test_training_save_restore(self):
dim_list,
[10],
):
# Skip float16 tests if the platform is macOS arm64 architecture
if is_macos() and is_arm64():
if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16:
continue
id += 1
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
Expand Down Expand Up @@ -819,6 +827,10 @@ def test_training_save_restore_by_files(self):
[10],
[10],
):
# Skip float16 tests if the platform is macOS arm64 architecture
if is_macos() and is_arm64():
if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16:
continue
id += 1
save_dir = os.path.join(self.get_temp_dir(), "save_restore")
save_path = os.path.join(tempfile.mkdtemp(prefix=save_dir), "hash")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ def _convert(v, t):
id += 1
# Skip float16 tests if the platform is macOS arm64 architecture
if is_macos() and is_arm64():
if value_dtype == dtypes.half:
if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16:
continue
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()) as sess:
Expand Down Expand Up @@ -445,7 +445,7 @@ def _convert(v, t):
id += 1
# Skip float16 tests if the platform is macOS arm64 architecture
if is_macos() and is_arm64():
if value_dtype == dtypes.half:
if value_dtype == dtypes.half or value_dtype == dtypes.bfloat16:
continue
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()) as sess:
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_recommenders_addons/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def is_arm64():

# Required TensorFlow version [min, max]
if (is_macos() and is_arm64()):
MIN_TF_VERSION = "2.11.0"
MIN_TF_VERSION = "2.15.1"
MAX_TF_VERSION = "2.15.1"
else:
MIN_TF_VERSION = "2.11.0"
Expand Down

0 comments on commit 2a2cc01

Please sign in to comment.