From 301ad5d77f4fe7130e73db26e11d7c77f3464159 Mon Sep 17 00:00:00 2001 From: Shanshan Yu Date: Wed, 9 Feb 2022 19:58:34 -0800 Subject: [PATCH] [feat]support Mac m1 devices --- ...wheel_Linux.sh => make_wheel_Linux_x86.sh} | 0 ...l_Windows.sh => make_wheel_Windows_x86.sh} | 0 .github/workflows/make_wheel_macOS_arm64.sh | 28 ++++++++++++++ ...wheel_macOS.sh => make_wheel_macOS_x86.sh} | 0 .github/workflows/release.yml | 27 ++++++++++--- configure.py | 32 ++++++++++++++-- .../dynamic_embedding_variable_test.py | 9 +++++ .../kernel_tests/redis_table_variable_test.py | 25 +++++------- .../python/ops/dynamic_embedding_variable.py | 6 +++ .../utils/check_platform.py | 38 +++++++++++++++++++ 10 files changed, 140 insertions(+), 25 deletions(-) rename .github/workflows/{make_wheel_Linux.sh => make_wheel_Linux_x86.sh} (100%) rename .github/workflows/{make_wheel_Windows.sh => make_wheel_Windows_x86.sh} (100%) create mode 100644 .github/workflows/make_wheel_macOS_arm64.sh rename .github/workflows/{make_wheel_macOS.sh => make_wheel_macOS_x86.sh} (100%) create mode 100644 tensorflow_recommenders_addons/utils/check_platform.py diff --git a/.github/workflows/make_wheel_Linux.sh b/.github/workflows/make_wheel_Linux_x86.sh similarity index 100% rename from .github/workflows/make_wheel_Linux.sh rename to .github/workflows/make_wheel_Linux_x86.sh diff --git a/.github/workflows/make_wheel_Windows.sh b/.github/workflows/make_wheel_Windows_x86.sh similarity index 100% rename from .github/workflows/make_wheel_Windows.sh rename to .github/workflows/make_wheel_Windows_x86.sh diff --git a/.github/workflows/make_wheel_macOS_arm64.sh b/.github/workflows/make_wheel_macOS_arm64.sh new file mode 100644 index 000000000..090d7a216 --- /dev/null +++ b/.github/workflows/make_wheel_macOS_arm64.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Making wheel for macOS arm64 architecture +# Requirements: +# MacOS Monterey 12.0.0 +, Tensorflow-macos 2.5.0, ARM64 Apple Silicon, Bazel 4.1.0 + +# Please don't install tensorflow-metal, it may cause incorrect GPU devices detection issue. +set -e -x + +# Install CPU version +export TF_NEED_CUDA=0 + +python --version + +python configure.py + +bazel build \ + --cpu=darwin_arm64 \ + --copt -mmacosx-version-min=12.0 \ + --linkopt -mmacosx-version-min=12.0 \ + --noshow_progress \ + --noshow_loading_progress \ + --verbose_failures \ + --test_output=errors \ + build_pip_pkg + +# Output the wheel file to the artifacts directory +bazel-bin/build_pip_pkg artifacts "--plat-name macosx_12_0_arm64 $NIGHTLY_FLAG" +delocate-wheel -w wheelhouse artifacts/*.whl diff --git a/.github/workflows/make_wheel_macOS.sh b/.github/workflows/make_wheel_macOS_x86.sh similarity index 100% rename from .github/workflows/make_wheel_macOS.sh rename to .github/workflows/make_wheel_macOS_x86.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f870d07aa..4b7afe32d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,15 +42,23 @@ jobs: strategy: matrix: # TODO: add back 'windows-latest' when it can be compiled. - os: ['macos-latest', 'ubuntu-18.04'] + os: ['macos-10.15', 'ubuntu-18.04'] py-version: ['3.7', '3.8', '3.9'] tf-version: ['2.5.1', '2.7.0'] tf-need-cuda: ['1', '0'] tf-cuda-version: ['10.0', '11.2'] tf-cudnn-version: ['7.6', '8.1'] + cpu: ['x86'] +# TODO(poinwater): add macOS CI once GitHub supports macOS 12.0.0 + +# include: +# - os: 'macos-11' +# cpu: 'arm64' +# tf-version: '2.5.0' +# py-version: '3.9' +# tf-need-cuda: '0' exclude: # excludes cuda on macOS - - os: 'macos-latest' + - os: 'macos-10.15' tf-need-cuda: '1' # excludes TF2.5.1 with cuda 10.0 - tf-version: '2.5.1' @@ -98,8 +106,9 @@ jobs: TF_CUDA_VERSION: ${{ matrix.tf-cuda-version }} TF_CUDNN_VERSION: ${{ matrix.tf-cudnn-version }} NIGHTLY_TIME: ${{ steps.author-date.outputs.result }} + CPU: ${{ matrix.cpu }} shell: bash - run: bash .github/workflows/make_wheel_${OS}.sh + run: bash .github/workflows/make_wheel_${OS}_${CPU}.sh - uses: haya14busa/action-cond@v1 id: device with: @@ -108,7 +117,7 @@ jobs: if_false: "cpu" - uses: actions/upload-artifact@v1 with: - name: ${{ runner.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-wheel + name: ${{ runner.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-${{ matrix.cpu }}-wheel path: wheelhouse upload-wheels: name: Publish wheels to PyPi @@ -121,6 +130,14 @@ jobs: py-version: ['3.7', '3.8', '3.9'] tf-version: ['2.5.1', '2.7.0'] tf-need-cuda: ['1', '0'] + cpu: ['x86'] +# TODO(poinwater): add macOS CI once GitHub supports macOS 12.0.0 + +# include: +# - os: 'macOS' +# cpu: 'arm64' +# tf-version: '2.5.0' +# py-version: '3.9' +# tf-need-cuda: '0' exclude: # excludes cuda on macOS - os: 'macOS' @@ -136,7 +153,7 @@ jobs: if_false: "cpu" - uses: actions/download-artifact@v1 with: - name: ${{ matrix.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-wheel + name: ${{ matrix.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-${{ matrix.cpu }}-wheel path: ./dist - run: | set -e -x diff --git a/configure.py b/configure.py index ee7c5bfa8..553a93fdd 100644 --- a/configure.py +++ b/configure.py @@ -27,18 +27,25 @@ # Maping TensorFlow version to valid Bazel version. def _VALID_BAZEL_VERSION(tf_version): - if tf_version < "2.0.0": + if is_macos() and is_arm64(): + target_bazel = "4.1.0" + logging.warn( + 'Only Bazel version greater than 4.1.0 supports macOS arm64 platform.') + return target_bazel + elif tf_version < "2.0.0": + target_bazel = "0.26.1" logging.warn( 'There is only limited support for TensorFlow under version 2.0.0 ' 'because its Bazel version, and requiring users to make some Bazel script changes ' 'refering to the previous COMMIT to compile properly by themselves.') - return "0.26.1" + return target_bazel elif tf_version >= "2.0.0": + target_bazel = "3.7.2" logging.info( 'To ensure code compatibility with Bazel rules_foreign_cc component, ' 'we specify Bazel version greater than 3.7.2 ' 'for Tensorflow versions greater than 2.0.0.') - return "3.7.2" + return target_bazel else: raise ValueError('Unsupport TensorFlow version {}.'.format(tf_version)) @@ -160,16 +167,31 @@ def get_tf_version_integer(): return int(tf_version_num) -def check_bazel_version(): +def _get_installed_and_valid_bazel_version(): stream = os.popen('bazel version |grep label') output = stream.read() installed_bazel_version = str(output).split(":")[1].strip() valid_bazel_version = _VALID_BAZEL_VERSION(tf.__version__) + return installed_bazel_version, valid_bazel_version + + +def check_bazel_version(): + installed_bazel_version, valid_bazel_version = _get_installed_and_valid_bazel_version( + ) if installed_bazel_version != valid_bazel_version: raise ValueError('Bazel version is {}, but {} is needed.'.format( installed_bazel_version, valid_bazel_version)) +def check_bazel_version_for_macOS_arm64(): + installed_bazel_version, valid_bazel_version = _get_installed_and_valid_bazel_version( + ) + if installed_bazel_version < valid_bazel_version: + raise ValueError( + 'Bazel version is {}. For macOS arm64 platform, Bazel version must be at least {}.' + .format(installed_bazel_version, valid_bazel_version)) + + def extract_tf_header(): tf_header_dir = get_tf_header_dir() tf_version_integer = get_tf_version_integer() @@ -193,6 +215,8 @@ def create_build_configuration(): os.remove(_TFRA_BAZELRC) if is_linux(): check_bazel_version() + if is_macos() and is_arm64(): + check_bazel_version_for_macOS_arm64() extract_tf_header() logging.disable(logging.WARNING) diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py index ba163bbc8..f2c5447c0 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py @@ -27,6 +27,7 @@ import tempfile from tensorflow_recommenders_addons import dynamic_embedding as de +from tensorflow_recommenders_addons.utils.check_platform import is_macos, is_arm64 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session @@ -387,6 +388,10 @@ def _convert(v, t): for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list): id += 1 + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half: + continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: keys = constant_op.constant( @@ -456,6 +461,10 @@ def _convert(v, t): for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list): id += 1 + # Skip float16 tests if the platform is macOS arm64 archtecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half: + continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: base_keys = constant_op.constant( diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py index 7abb244be..66a2e9114 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py @@ -31,6 +31,7 @@ import tempfile from tensorflow_recommenders_addons import dynamic_embedding as de +from tensorflow_recommenders_addons.utils.check_platform import is_windows, is_macos, is_arm64, is_linux, is_raspi_arm from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session @@ -355,6 +356,10 @@ def _convert(v, t): for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list): id += 1 + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half: + continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: keys = constant_op.constant( @@ -427,6 +432,10 @@ def _convert(v, t): for (key_dtype, value_dtype), dim in itertools.product(kv_list, dim_list): id += 1 + # Skip float16 tests if the platform is macOS arm64 architecture + if is_macos() and is_arm64(): + if value_dtype == dtypes.half: + continue with self.session(config=default_config, use_gpu=test_util.is_gpu_available()) as sess: keys = constant_op.constant( @@ -1815,22 +1824,6 @@ def test_unable_connect_to_redis(self): self.evaluate(table.size()) -def is_macos(): - return platform.system() == "Darwin" - - -def is_windows(): - return platform.system() == "Windows" - - -def is_linux(): - return platform.system() == "Linux" - - -def is_raspi_arm(): - return os.uname()[4] == "armv7l" - - if __name__ == "__main__": if is_windows() == False: if os.popen("which redis-server").read() == '': diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/ops/dynamic_embedding_variable.py b/tensorflow_recommenders_addons/dynamic_embedding/python/ops/dynamic_embedding_variable.py index e511ee3fd..f2886d37e 100644 --- a/tensorflow_recommenders_addons/dynamic_embedding/python/ops/dynamic_embedding_variable.py +++ b/tensorflow_recommenders_addons/dynamic_embedding/python/ops/dynamic_embedding_variable.py @@ -25,6 +25,7 @@ import functools from tensorflow_recommenders_addons import dynamic_embedding as de +from tensorflow_recommenders_addons.utils.check_platform import is_macos, is_arm64 try: from tensorflow.python.util import _pywrap_util_port as pywrap @@ -305,6 +306,11 @@ def _get_default_devices(): [dtypes.int64, dtypes.int64], [dtypes.int32, dtypes.float32], ] + if is_macos() and is_arm64(): + if value_dtype == dtypes.half: + raise TypeError(""" + float16 value dtype is not supported on macOS with ARM64 architecture. Please try another type. + """) if [key_dtype, value_dtype] not in valid_dtype_list: raise TypeError( "key-value dtype ({}-{}) is not support! The valid dtypes are \n{}\n". diff --git a/tensorflow_recommenders_addons/utils/check_platform.py b/tensorflow_recommenders_addons/utils/check_platform.py new file mode 100644 index 000000000..57137a394 --- /dev/null +++ b/tensorflow_recommenders_addons/utils/check_platform.py @@ -0,0 +1,38 @@ +# Copyright 2022 The TensorFlow Recommenders-Addons Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Check the system and architecture types +""" +import platform + + +def is_macos(): + return platform.system() == "Darwin" + + +def is_windows(): + return platform.system() == "Windows" + + +def is_linux(): + return platform.system() == "Linux" + + +def is_arm64(): + return platform.machine() == "arm64" + + +def is_raspi_arm(): + return os.uname()[4] == "armv7l"