From 64ec2e5c7399ec7b5b14d1b5199e4cac4d0fa669 Mon Sep 17 00:00:00 2001
From: rhdong <rhdong2017@gmail.com>
Date: Thu, 9 Mar 2023 16:23:10 +0800
Subject: [PATCH] [CI] update CI docker to ubuntu20.04
- Simplify the CI pipeline by removing some unnecessary testing items.
- Fix the hiredis dependency to tag v1.1.0
---
.github/workflows/backport.yml | 2 +-
.github/workflows/make_wheel_Linux_x86.sh | 2 +-
.github/workflows/release.yml | 31 ++---
WORKSPACE | 2 +-
.../dynamic_embedding_ops_test.py | 12 +-
.../dynamic_embedding_optimizer_test.py | 2 +-
.../dynamic_embedding_variable_test.py | 6 +-
.../kernel_tests/redis_table_variable_test.py | 6 +-
tools/check_python_format.py | 4 +-
...20.04-manylinux2014-multipython.Dockerfile | 115 ++++++++++++++++++
tools/docker/install/install_deb_packages.sh | 2 +-
tools/docker/install/install_latest_clang.sh | 15 +--
tools/docker/install/install_nccl.sh | 2 +-
.../install_pip_packages_by_version.sh | 2 +-
tools/docker/sanity_check.Dockerfile | 1 +
15 files changed, 150 insertions(+), 54 deletions(-)
create mode 100644 tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 59dad26a0..493dc97c8 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -7,7 +7,7 @@ on:
jobs:
backport:
- runs-on: ubuntu-18.04
+ runs-on: ubuntu-20.04
name: Backport
steps:
- name: Backport Bot
diff --git a/.github/workflows/make_wheel_Linux_x86.sh b/.github/workflows/make_wheel_Linux_x86.sh
index c0be3a96d..8c9d95eaa 100644
--- a/.github/workflows/make_wheel_Linux_x86.sh
+++ b/.github/workflows/make_wheel_Linux_x86.sh
@@ -12,7 +12,7 @@ else
fi
if [ $TF_VERSION == "2.6.3" ] || [ $TF_VERSION == "2.8.3" ] ; then
- export BUILD_IMAGE="tfra/nosla-cuda11.2-cudnn8.1-ubuntu18.04-manylinux2010-multipython"
+ export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython"
export TF_CUDA_VERSION="11.2"
export TF_CUDNN_VERSION="8.1"
elif [ $TF_VERSION == "2.4.1" ] ; then
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7bfef6815..0755106fd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -23,7 +23,7 @@ env:
jobs:
test-with-bazel:
name: Test with bazel
- runs-on: ubuntu-18.04
+ runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
@@ -44,12 +44,12 @@ jobs:
strategy:
matrix:
# TODO: add back 'windows-latest' when it can be compiled.
- os: ['macos-12', 'ubuntu-18.04']
+ os: ['macos-12', 'ubuntu-20.04']
py-version: ['3.7', '3.8', '3.9']
tf-version: ['2.6.3', '2.8.3']
tf-need-cuda: ['1', '0']
- tf-cuda-version: ['10.0', '11.2']
- tf-cudnn-version: ['7.6', '8.1']
+ tf-cuda-version: ['11.2']
+ tf-cudnn-version: ['8.1']
cpu: ['x86']
# TODO(poinwater): add macOS CI once GitHub supports macOS 12.0.0 +
# include:
@@ -62,23 +62,6 @@ jobs:
# excludes cuda on macOS
- os: 'macos-12'
tf-need-cuda: '1'
- # excludes TF2.6.3 with cuda 10.0
- - tf-version: '2.6.3'
- tf-cuda-version: '10.0'
- # excludes TF2.8.3 with cuda 10.0
- - tf-version: '2.8.3'
- tf-cuda-version: '10.0'
- # excludes cudnn 8 with cuda 10.0
- - tf-cuda-version: '11.2'
- tf-cudnn-version: '7.6'
- # excludes cudnn 7 with cuda 11.0
- - tf-cuda-version: '10.0'
- tf-cudnn-version: '8.1'
- # excludes some unnecessary step on no cuda
- - tf-need-cuda: '0'
- tf-cuda-version: '10.0'
- - tf-need-cuda: '0'
- tf-cudnn-version: '7.6'
fail-fast: false
steps:
- uses: actions/github-script@0.3.0
@@ -97,7 +80,7 @@ jobs:
python-version: ${{ matrix.py-version }}
- name: Setup Bazel
# Ubuntu bazel is run inside of the docker image
- if: matrix.os != 'ubuntu-18.04'
+ if: matrix.os != 'ubuntu-20.04'
run: bash tools/install_deps/install_bazelisk.sh ./
- name: Build wheels
env:
@@ -124,7 +107,7 @@ jobs:
upload-wheels:
name: Publish wheels to PyPi
needs: [release-wheel, test-with-bazel]
- runs-on: ubuntu-18.04
+ runs-on: ubuntu-20.04
strategy:
matrix:
# TODO: add back 'Windows' when it can be compiled.
@@ -168,7 +151,7 @@ jobs:
upload-dev-container:
name: Upload dev container to DockerHub
needs: [release-wheel, test-with-bazel]
- runs-on: ubuntu-18.04
+ runs-on: ubuntu-20.04
strategy:
matrix:
py-version: ['3.7', '3.8', '3.9']
diff --git a/WORKSPACE b/WORKSPACE
index 7df12b577..4cf968237 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -41,9 +41,9 @@ http_archive(
new_git_repository(
name = "hiredis",
- branch = "master",
build_file = "//build_deps/toolchains/redis:hiredis.BUILD",
remote = "https://github.com/redis/hiredis.git",
+ tag = "v1.1.0",
)
http_archive(
diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py
index 1ade02b52..168594964 100644
--- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py
+++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_ops_test.py
@@ -66,7 +66,7 @@ def _type_converter(tf_type):
mapper = {
dtypes.int32: np.int32,
dtypes.int64: np.int64,
- dtypes.float32: np.float,
+ dtypes.float32: float,
dtypes.float64: np.float64,
}
return mapper[tf_type]
@@ -383,9 +383,8 @@ def test_higher_rank(self):
initializer=2.0,
dim=dim,
)
- ids = np.random.randint(2**31,
- size=np.prod(ids_shape),
- dtype=np.int).reshape(ids_shape)
+ ids = np.random.randint(2**31, size=np.prod(ids_shape),
+ dtype=int).reshape(ids_shape)
ids = constant_op.constant(ids, dtype=dtypes.int64)
simple = params.lookup(ids)
self.evaluate(params.upsert(ids, simple))
@@ -414,9 +413,8 @@ def test_static_shape_checking(self):
params_nn = variable_scope.get_variable("n",
shape=[100, dim],
use_resource=False)
- ids = np.random.randint(2**31,
- size=np.prod(ids_shape),
- dtype=np.int).reshape(ids_shape)
+ ids = np.random.randint(2**31, size=np.prod(ids_shape),
+ dtype=int).reshape(ids_shape)
ids = constant_op.constant(ids, dtype=dtypes.int64)
embedding_test = de.embedding_lookup(params, ids)
diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py
index 16ea7122c..973765d71 100644
--- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py
+++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_optimizer_test.py
@@ -62,7 +62,7 @@ def _type_converter(tf_type):
mapper = {
dtypes.int32: np.int32,
dtypes.int64: np.int64,
- dtypes.float32: np.float,
+ dtypes.float32: float,
dtypes.float64: np.float64,
}
return mapper[tf_type]
diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py
index 61f597eba..f1af27609 100644
--- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py
+++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/dynamic_embedding_variable_test.py
@@ -71,13 +71,13 @@ def _type_converter(tf_type):
mapper = {
dtypes.int32: np.int32,
dtypes.int64: np.int64,
- dtypes.float32: np.float,
+ dtypes.float32: float,
dtypes.float64: np.float64,
- dtypes.string: np.str,
+ dtypes.string: str,
dtypes.half: np.float16,
dtypes.bfloat16: dtypes.bfloat16.as_numpy_dtype,
dtypes.int8: np.int8,
- dtypes.bool: np.bool,
+ dtypes.bool: bool,
}
return mapper[tf_type]
diff --git a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py
index 174f43565..0b9122a3c 100644
--- a/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py
+++ b/tensorflow_recommenders_addons/dynamic_embedding/python/kernel_tests/redis_table_variable_test.py
@@ -71,12 +71,12 @@ def _type_converter(tf_type):
mapper = {
dtypes.int32: np.int32,
dtypes.int64: np.int64,
- dtypes.float32: np.float,
+ dtypes.float32: float,
dtypes.float64: np.float64,
- dtypes.string: np.str,
+ dtypes.string: str,
dtypes.half: np.float16,
dtypes.int8: np.int8,
- dtypes.bool: np.bool,
+ dtypes.bool: bool,
}
return mapper[tf_type]
diff --git a/tools/check_python_format.py b/tools/check_python_format.py
index 2b0b7ee12..ccd7056d5 100644
--- a/tools/check_python_format.py
+++ b/tools/check_python_format.py
@@ -19,7 +19,9 @@ def _run_format():
if files_changed:
print("Some files have changed.")
- print("Please use 'yapf --style=google -ri ./**/*.py' before commit.")
+ print(
+ "Please use 'find . -name '*.py' -print0 | xargs -0 yapf --style=./.yapf -ir' before commit."
+ )
else:
print("No formatting needed.")
diff --git a/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile b/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile
new file mode 100644
index 000000000..7b9bc3abd
--- /dev/null
+++ b/tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile
@@ -0,0 +1,115 @@
+# Dockerfile to build a manylinux 2010 compliant cross-compiler.
+#
+# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
+# glibc (2.12) and system libstdc++ (4.4).
+#
+# To push a new version, run:
+# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile . \
+# --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython"
+# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython
+
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset
+
+RUN chmod 777 /tmp/
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y \
+ cpio \
+ file \
+ flex \
+ g++ \
+ make \
+ patch \
+ rpm2cpio \
+ unar \
+ wget \
+ xz-utils \
+ libjpeg-dev \
+ zlib1g-dev \
+ libgflags-dev \
+ libsnappy-dev \
+ libbz2-dev \
+ liblz4-dev \
+ libzstd-dev \
+ openssh-client \
+ && \
+ rm -rf /var/lib/apt/lists/*
+
+ADD devtoolset/fixlinks.sh fixlinks.sh
+ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
+ADD devtoolset/rpm-patch.sh rpm-patch.sh
+
+# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
+RUN /build_devtoolset.sh devtoolset-7 /dt7
+# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
+RUN /build_devtoolset.sh devtoolset-8 /dt8
+
+# TODO(klimek): Split up into two different docker images.
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04
+COPY --from=devtoolset /dt7 /dt7
+COPY --from=devtoolset /dt8 /dt8
+
+# Install TensorRT.
+RUN echo \
+ deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \
+ > /etc/apt/sources.list.d/nvidia-ml.list \
+ && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \
+ apt-get update && \
+ rm -rf /var/lib/apt/lists/*
+
+# Copy and run the install scripts.
+ARG DEBIAN_FRONTEND=noninteractive
+
+COPY install/install_bootstrap_deb_packages.sh /install/
+RUN /install/install_bootstrap_deb_packages.sh
+
+COPY install/install_deb_packages.sh /install/
+RUN /install/install_deb_packages.sh
+
+# Install additional packages needed for this image:
+# - dependencies to build Python from source
+# - patchelf, as it is required by auditwheel
+RUN apt-get update && apt-get install -y \
+ libbz2-dev \
+ libffi-dev \
+ libgdbm-dev \
+ libncurses5-dev \
+ libnss3-dev \
+ libreadline-dev \
+ patchelf \
+ gcc-multilib \
+ && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN chmod 777 /tmp/
+WORKDIR /tmp/
+
+COPY install/install_nccl.sh /install/
+RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
+
+COPY install/install_rocksdb.sh /install/
+RUN /install/install_rocksdb.sh "6.22.1"
+
+COPY install/install_bazel.sh /install/
+RUN /install/install_bazel.sh "5.1.1"
+
+COPY install/build_and_install_python.sh /install/
+RUN /install/build_and_install_python.sh "3.7.7"
+RUN /install/build_and_install_python.sh "3.8.2"
+RUN /install/build_and_install_python.sh "3.9.7"
+RUN /install/build_and_install_python.sh "3.10.6"
+
+COPY install/install_pip_packages_by_version.sh /install/
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.9"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.8"
+RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.7"
+
+COPY install/use_devtoolset_7.sh /install/
+RUN /install/use_devtoolset_7.sh
+
+COPY install/install_openmpi.sh /install/
+RUN /install/install_openmpi.sh "4.1.1"
+
+# clean
+RUN rm -rf /tmp/*
diff --git a/tools/docker/install/install_deb_packages.sh b/tools/docker/install/install_deb_packages.sh
index 9ad50e365..c5eeb28b0 100755
--- a/tools/docker/install/install_deb_packages.sh
+++ b/tools/docker/install/install_deb_packages.sh
@@ -55,7 +55,7 @@ apt-get install -y --no-install-recommends \
pkg-config \
python-dev \
python-setuptools \
- python-virtualenv \
+ python3-virtualenv \
python3-dev \
python3-setuptools \
rsync \
diff --git a/tools/docker/install/install_latest_clang.sh b/tools/docker/install/install_latest_clang.sh
index 827383e72..18eec53b4 100755
--- a/tools/docker/install/install_latest_clang.sh
+++ b/tools/docker/install/install_latest_clang.sh
@@ -1,5 +1,5 @@
#!/bin/bash -eu
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,11 +14,8 @@
# limitations under the License.
# ==============================================================================
-DIST="$(grep "DISTRIB_RELEASE" /etc/lsb-release |sed 's,.*=,,; s,\.,_,')"
-INSTALL_DIR="/clang_${CLANG_VERSION}"
-STORAGE="https://storage.googleapis.com/clang-builds-stable"
-mkdir -p "${INSTALL_DIR}"
-cd "${INSTALL_DIR}"
-wget "${STORAGE}/clang-ubuntu${DIST}/clang_${CLANG_VERSION}.tar.gz"
-tar xvzf clang_${CLANG_VERSION}.tar.gz
-rm clang_${CLANG_VERSION}.tar.gz
+DIST="$(grep "DISTRIB_CODENAME" /etc/lsb-release |sed 's,.*=,,')"
+wget -O - "https://apt.llvm.org/llvm-snapshot.gpg.key"| apt-key add -
+add-apt-repository "deb http://apt.llvm.org/${DIST}/ llvm-toolchain-${DIST}-8 main"
+apt-get update && apt-get install -y clang-8 && \
+ rm -rf /var/lib/apt/lists/*
diff --git a/tools/docker/install/install_nccl.sh b/tools/docker/install/install_nccl.sh
index 815ae2850..e379c225c 100755
--- a/tools/docker/install/install_nccl.sh
+++ b/tools/docker/install/install_nccl.sh
@@ -22,4 +22,4 @@ set -e
cd /install/
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-keyring_1.0-1_all.deb
dpkg -i cuda-keyring_1.0-1_all.deb
-apt-get update && apt-get install -y libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION}
+apt-get update && apt-get install -y --allow-change-held-packages libnccl2=${NCCL_VERSION} libnccl-dev=${NCCL_VERSION}
diff --git a/tools/docker/install/install_pip_packages_by_version.sh b/tools/docker/install/install_pip_packages_by_version.sh
index e4b74b48a..5259a55bb 100755
--- a/tools/docker/install/install_pip_packages_by_version.sh
+++ b/tools/docker/install/install_pip_packages_by_version.sh
@@ -36,7 +36,7 @@ PACKAGES=(
"werkzeug"
"bleach"
"markdown"
- "protobuf"
+ "protobuf==3.20.0"
"numpy"
"scipy"
"scikit-learn"
diff --git a/tools/docker/sanity_check.Dockerfile b/tools/docker/sanity_check.Dockerfile
index 4c9da8fb0..d054b9713 100644
--- a/tools/docker/sanity_check.Dockerfile
+++ b/tools/docker/sanity_check.Dockerfile
@@ -86,6 +86,7 @@ RUN touch /ok.txt
# Bazel code format
FROM alpine:3.11 as check-bazel-format
+RUN apk update && apk add bash
COPY ./tools/install_deps/buildifier.sh ./
RUN sh buildifier.sh