From beee8db574b749af79266c5dc7c5110d2990e5e6 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Sat, 27 Mar 2021 02:55:01 +0100 Subject: [PATCH 1/8] Add GPU support for ONNXRuntime. --- cuda.spec | 2 ++ cudnn-toolfile.spec | 22 ++++++++++++++++++++++ cudnn.spec | 26 ++++++++++++++++++++++++++ onnxruntime-toolfile.spec | 2 ++ onnxruntime.spec | 6 +++++- 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 cudnn-toolfile.spec create mode 100644 cudnn.spec diff --git a/cuda.spec b/cuda.spec index f330a9ad423..ffd642a6589 100644 --- a/cuda.spec +++ b/cuda.spec @@ -34,6 +34,7 @@ mkdir -p %{i}/lib64/stubs # package only the runtime static library mv %_builddir/build/lib64/libcudadevrt.a %{i}/lib64/ +mv %_builddir/build/lib64/libcudart_static.a %{i}/lib64/ rm -f %_builddir/build/lib64/lib*.a # package only the CUDA driver and NVML library stub @@ -101,6 +102,7 @@ ln -sf libnvidia-ptxjitcompiler.so.1 %{i} sed \ -e"/^TOP *=/s|= .*|= $CMS_INSTALL_PREFIX/%{pkgrel}|" \ -e's|$(_HERE_)|$(TOP)/bin|g' \ + -e's|$(TOP)/lib|$(TOP)/lib64|g' \ -e's|/$(_TARGET_DIR_)||g' \ -e's|$(_TARGET_SIZE_)|64|g' \ -i $RPM_INSTALL_PREFIX/%{pkgrel}/bin/nvcc.profile diff --git a/cudnn-toolfile.spec b/cudnn-toolfile.spec new file mode 100644 index 00000000000..e18c62045e1 --- /dev/null +++ b/cudnn-toolfile.spec @@ -0,0 +1,22 @@ +### RPM external cuda-toolfile 1.0 +Requires: cudnn +%prep + +%build + +%install +mkdir -p %i/etc/scram.d +cat << \EOF_TOOLFILE >%i/etc/scram.d/cudnn.xml + + + + + + + + + + +EOF_TOOLFILE + +## IMPORT scram-tools-post diff --git a/cudnn.spec b/cudnn.spec new file mode 100644 index 00000000000..537acd6f735 --- /dev/null +++ b/cudnn.spec @@ -0,0 +1,26 @@ +### RPM external cudnn 8.1.1.33 +## INITENV +PATH LD_LIBRARY_PATH %i/lib64 + +%define cudaver_maj 11.2 +%define cudnnver_maj %(echo %{realversion} | cut -f1,2,3 -d.) + +%ifarch x86_64 +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-x64-v%{realversion}.tgz +%endif +%ifarch ppc64le +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-ppc64le-v%{realversion}.tgz +%endif +%ifarch aarch64 +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-aarch64sbsa-v%{realversion}.tgz +%endif +Requires: cuda + +%prep +%setup -n cuda + +%build + +%install +rm %_builddir/cuda/lib64/*.a +mv %_builddir/cuda/* %{i}/ + diff --git a/onnxruntime-toolfile.spec b/onnxruntime-toolfile.spec index cac1d127593..dce225390ff 100644 --- a/onnxruntime-toolfile.spec +++ b/onnxruntime-toolfile.spec @@ -16,6 +16,8 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/onnxruntime.xml + + EOF_TOOLFILE diff --git a/onnxruntime.spec b/onnxruntime.spec index d13b02ebc27..b40df0e2801 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -7,6 +7,7 @@ Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&expo BuildRequires: cmake ninja Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11 +Requires: cuda cudnn %prep %setup -q -n %{n}-%{realversion} @@ -21,7 +22,10 @@ cmake ../%{n}-%{realversion}/cmake -GNinja \ -DCMAKE_INSTALL_LIBDIR=lib \ -Donnxruntime_ENABLE_PYTHON=ON \ -Donnxruntime_BUILD_SHARED_LIB=ON \ - -Donnxruntime_USE_CUDA=OFF \ + -Donnxruntime_USE_CUDA=ON \ + -Donnxruntime_CUDA_VERSION="${CUDA_VERSION}" \ + -Donnxruntime_CUDA_HOME="${CUDA_ROOT}" \ + -Donnxruntime_CUDNN_HOME="${CUDNN_ROOT}" \ -Donnxruntime_BUILD_CSHARP=OFF \ -Donnxruntime_USE_EIGEN_FOR_BLAS=ON \ -Donnxruntime_USE_OPENBLAS=OFF \ From ee8a694009739285bbc38c064b598ab2dce2ff55 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Thu, 1 Apr 2021 02:59:45 +0200 Subject: [PATCH 2/8] Avoid dependency on libcudart_static.a --- cuda.spec | 2 -- cudnn.spec | 13 +++++++++---- onnxruntime.spec | 3 +++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cuda.spec b/cuda.spec index ffd642a6589..f330a9ad423 100644 --- a/cuda.spec +++ b/cuda.spec @@ -34,7 +34,6 @@ mkdir -p %{i}/lib64/stubs # package only the runtime static library mv %_builddir/build/lib64/libcudadevrt.a %{i}/lib64/ -mv %_builddir/build/lib64/libcudart_static.a %{i}/lib64/ rm -f %_builddir/build/lib64/lib*.a # package only the CUDA driver and NVML library stub @@ -102,7 +101,6 @@ ln -sf libnvidia-ptxjitcompiler.so.1 %{i} sed \ -e"/^TOP *=/s|= .*|= $CMS_INSTALL_PREFIX/%{pkgrel}|" \ -e's|$(_HERE_)|$(TOP)/bin|g' \ - -e's|$(TOP)/lib|$(TOP)/lib64|g' \ -e's|/$(_TARGET_DIR_)||g' \ -e's|$(_TARGET_SIZE_)|64|g' \ -i $RPM_INSTALL_PREFIX/%{pkgrel}/bin/nvcc.profile diff --git a/cudnn.spec b/cudnn.spec index 537acd6f735..1b095881d7b 100644 --- a/cudnn.spec +++ b/cudnn.spec @@ -1,23 +1,28 @@ ### RPM external cudnn 8.1.1.33 ## INITENV +PATH LD_LIBRARY_PATH %i/lib64 -%define cudaver_maj 11.2 +%define cudaver 11.2 %define cudnnver_maj %(echo %{realversion} | cut -f1,2,3 -d.) %ifarch x86_64 -Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-x64-v%{realversion}.tgz +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver}-linux-x64-v%{realversion}.tgz %endif %ifarch ppc64le -Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-ppc64le-v%{realversion}.tgz +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver}-linux-ppc64le-v%{realversion}.tgz %endif %ifarch aarch64 -Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver_maj}-linux-aarch64sbsa-v%{realversion}.tgz +Source: https://developer.download.nvidia.com/compute/redist/cudnn/v%{cudnnver_maj}/cudnn-%{cudaver}-linux-aarch64sbsa-v%{realversion}.tgz %endif Requires: cuda %prep %setup -n cuda +if [ "${CUDA_VERSION%.*}" != %{cudaver} ]; then + echo 'Incompatible CUDA version in cudnn.spec!' + exit 1 +fi + %build %install diff --git a/onnxruntime.spec b/onnxruntime.spec index b40df0e2801..9704f486bd4 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -43,6 +43,9 @@ cmake ../%{n}-%{realversion}/cmake -GNinja \ -Donnxruntime_DISABLE_CONTRIB_OPS=OFF \ -Donnxruntime_USE_PREINSTALLED_PROTOBUF=ON \ -Donnxruntime_PREFER_SYSTEM_LIB=ON \ + -DCMAKE_CUDA_FLAGS="-cudart shared" \ + -DCMAKE_CUDA_RUNTIME_LIBRARY=Shared \ + -DCMAKE_TRY_COMPILE_PLATFORM_VARIABLES="CMAKE_CUDA_RUNTIME_LIBRARY" \ -DCMAKE_PREFIX_PATH="${ZLIB_ROOT};${LIBPNG_ROOT};${PROTOBUF_ROOT};${PY2_PYBIND11_ROOT}" ninja -v %{makeprocesses} From a33ced00f6e6b79b004ed5a404e14def5be531b5 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Thu, 1 Apr 2021 15:42:49 +0200 Subject: [PATCH 3/8] Update cmssw-tool-conf.spec --- cmssw-tool-conf.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 7e381035d16..82f6d422a0c 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -170,6 +170,7 @@ Requires: gperftools-toolfile Requires: cuda-toolfile Requires: alpaka-toolfile Requires: cupla-toolfile +Requires: cudnn-toolfile %ifnarch ppc64le Requires: libunwind-toolfile From 9cf8f971409d87304ece09080ecbba8531f6a4df Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Thu, 1 Apr 2021 15:57:29 +0200 Subject: [PATCH 4/8] Update cudnn-toolfile.spec --- cudnn-toolfile.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudnn-toolfile.spec b/cudnn-toolfile.spec index e18c62045e1..2aed88d3eb4 100644 --- a/cudnn-toolfile.spec +++ b/cudnn-toolfile.spec @@ -1,4 +1,4 @@ -### RPM external cuda-toolfile 1.0 +### RPM external cudnn-toolfile 1.0 Requires: cudnn %prep From 234805d5f3459a43300f83194c2811e1b92dc18a Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Fri, 2 Apr 2021 12:28:12 +0200 Subject: [PATCH 5/8] Update cudnn.spec --- cudnn.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cudnn.spec b/cudnn.spec index 1b095881d7b..48780246781 100644 --- a/cudnn.spec +++ b/cudnn.spec @@ -26,6 +26,6 @@ fi %build %install -rm %_builddir/cuda/lib64/*.a +rm -f %_builddir/cuda/lib64/*.a mv %_builddir/cuda/* %{i}/ From 807de83321c0b47cf29f93997deb9be2ea33b663 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Tue, 6 Apr 2021 20:11:28 +0200 Subject: [PATCH 6/8] Build ONNXRuntime w/ CUDA support only for x86_64. --- cmssw-tool-conf.spec | 2 +- onnxruntime-toolfile.spec | 2 ++ onnxruntime.spec | 10 +++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 82f6d422a0c..ac4fbf791b8 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -170,7 +170,6 @@ Requires: gperftools-toolfile Requires: cuda-toolfile Requires: alpaka-toolfile Requires: cupla-toolfile -Requires: cudnn-toolfile %ifnarch ppc64le Requires: libunwind-toolfile @@ -186,6 +185,7 @@ Requires: intel-vtune Requires: glibc-toolfile Requires: cmsmon-tools Requires: dip-toolfile +Requires: cudnn-toolfile %else Requires: tkonlinesw-fake-toolfile Requires: oracle-fake-toolfile diff --git a/onnxruntime-toolfile.spec b/onnxruntime-toolfile.spec index dce225390ff..dc3ea4cf586 100644 --- a/onnxruntime-toolfile.spec +++ b/onnxruntime-toolfile.spec @@ -16,8 +16,10 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/onnxruntime.xml +%ifarch x86_64 +%endif EOF_TOOLFILE diff --git a/onnxruntime.spec b/onnxruntime.spec index 9704f486bd4..efc0e6691b9 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -7,7 +7,9 @@ Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&expo BuildRequires: cmake ninja Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11 +%ifarch x86_64 Requires: cuda cudnn +%endif %prep %setup -q -n %{n}-%{realversion} @@ -15,6 +17,12 @@ Requires: cuda cudnn %build rm -rf ../build; mkdir ../build; cd ../build +%ifarch x86_64 +USE_CUDA=ON +%else +USE_CUDA=OFF +%endif + cmake ../%{n}-%{realversion}/cmake -GNinja \ -DPYTHON_EXECUTABLE=${PYTHON3_ROOT}/bin/python3 \ -DCMAKE_BUILD_TYPE=Release \ @@ -22,7 +30,7 @@ cmake ../%{n}-%{realversion}/cmake -GNinja \ -DCMAKE_INSTALL_LIBDIR=lib \ -Donnxruntime_ENABLE_PYTHON=ON \ -Donnxruntime_BUILD_SHARED_LIB=ON \ - -Donnxruntime_USE_CUDA=ON \ + -Donnxruntime_USE_CUDA=${USE_CUDA} \ -Donnxruntime_CUDA_VERSION="${CUDA_VERSION}" \ -Donnxruntime_CUDA_HOME="${CUDA_ROOT}" \ -Donnxruntime_CUDNN_HOME="${CUDNN_ROOT}" \ From 6a9a7bf8d5976c4efbb0cc1b7b54081b5521cd66 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Wed, 7 Apr 2021 11:57:08 +0200 Subject: [PATCH 7/8] Fix cuDNN path on ppc64le. --- cmssw-tool-conf.spec | 2 +- cudnn.spec | 6 ++++++ onnxruntime-toolfile.spec | 2 -- onnxruntime.spec | 6 ------ 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index ac4fbf791b8..82f6d422a0c 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -170,6 +170,7 @@ Requires: gperftools-toolfile Requires: cuda-toolfile Requires: alpaka-toolfile Requires: cupla-toolfile +Requires: cudnn-toolfile %ifnarch ppc64le Requires: libunwind-toolfile @@ -185,7 +186,6 @@ Requires: intel-vtune Requires: glibc-toolfile Requires: cmsmon-tools Requires: dip-toolfile -Requires: cudnn-toolfile %else Requires: tkonlinesw-fake-toolfile Requires: oracle-fake-toolfile diff --git a/cudnn.spec b/cudnn.spec index 48780246781..778b2f5f22e 100644 --- a/cudnn.spec +++ b/cudnn.spec @@ -26,6 +26,12 @@ fi %build %install +%ifarch ppc64le +rm -f %_builddir/cuda/targets/ppc64le-linux/lib/*.a +mv %_builddir/cuda/targets/ppc64le-linux/lib %{i}/lib64 +mv %_builddir/cuda/targets/ppc64le-linux/* %{i}/ +%else rm -f %_builddir/cuda/lib64/*.a mv %_builddir/cuda/* %{i}/ +%endif diff --git a/onnxruntime-toolfile.spec b/onnxruntime-toolfile.spec index dc3ea4cf586..dce225390ff 100644 --- a/onnxruntime-toolfile.spec +++ b/onnxruntime-toolfile.spec @@ -16,10 +16,8 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/onnxruntime.xml -%ifarch x86_64 -%endif EOF_TOOLFILE diff --git a/onnxruntime.spec b/onnxruntime.spec index efc0e6691b9..5c7621d8ce6 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -7,9 +7,7 @@ Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&expo BuildRequires: cmake ninja Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11 -%ifarch x86_64 Requires: cuda cudnn -%endif %prep %setup -q -n %{n}-%{realversion} @@ -17,11 +15,7 @@ Requires: cuda cudnn %build rm -rf ../build; mkdir ../build; cd ../build -%ifarch x86_64 USE_CUDA=ON -%else -USE_CUDA=OFF -%endif cmake ../%{n}-%{realversion}/cmake -GNinja \ -DPYTHON_EXECUTABLE=${PYTHON3_ROOT}/bin/python3 \ From 7cc7fb0bc4d9c7a6807f3bd5e95723230d48cac0 Mon Sep 17 00:00:00 2001 From: Huilin Qu Date: Thu, 8 Apr 2021 00:45:07 +0200 Subject: [PATCH 8/8] Do not build cudnn on slc7_aarch64. --- cmssw-tool-conf.spec | 3 +++ onnxruntime-toolfile.spec | 2 ++ onnxruntime.spec | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/cmssw-tool-conf.spec b/cmssw-tool-conf.spec index 82f6d422a0c..710e64b512d 100644 --- a/cmssw-tool-conf.spec +++ b/cmssw-tool-conf.spec @@ -170,7 +170,10 @@ Requires: gperftools-toolfile Requires: cuda-toolfile Requires: alpaka-toolfile Requires: cupla-toolfile + +%if "%{cmsos}" != "slc7_aarch64" Requires: cudnn-toolfile +%endif %ifnarch ppc64le Requires: libunwind-toolfile diff --git a/onnxruntime-toolfile.spec b/onnxruntime-toolfile.spec index dce225390ff..116f28ab527 100644 --- a/onnxruntime-toolfile.spec +++ b/onnxruntime-toolfile.spec @@ -16,8 +16,10 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/onnxruntime.xml +%if "%{cmsos}" != "slc7_aarch64" +%endif EOF_TOOLFILE diff --git a/onnxruntime.spec b/onnxruntime.spec index 5c7621d8ce6..89de3830034 100644 --- a/onnxruntime.spec +++ b/onnxruntime.spec @@ -7,7 +7,9 @@ Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&expo BuildRequires: cmake ninja Requires: protobuf py3-numpy py2-wheel py2-onnx zlib libpng py2-pybind11 +%if "%{cmsos}" != "slc7_aarch64" Requires: cuda cudnn +%endif %prep %setup -q -n %{n}-%{realversion} @@ -15,7 +17,11 @@ Requires: cuda cudnn %build rm -rf ../build; mkdir ../build; cd ../build +%if "%{cmsos}" != "slc7_aarch64" USE_CUDA=ON +%else +USE_CUDA=OFF +%endif cmake ../%{n}-%{realversion}/cmake -GNinja \ -DPYTHON_EXECUTABLE=${PYTHON3_ROOT}/bin/python3 \