From 14b51693c43fcc376576ceb347a7ca748fa43d32 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Wed, 4 May 2022 21:32:07 -0700 Subject: [PATCH] Enable ccache for cudfjni build in Docker (#10790) This PR enables ccache support for `./build.sh clean cudfjar`. ccache 4.6 is built during image creation because ccacheversion available via `yum install` does not cache nvcc-compiled binaries. It's enabled by default for build.sh and repeated no-change build ```bash PARALLEL_LEVEL=6 SKIP_TESTS=true time ./build.sh clean cudfjar ``` succeeds in 1.5 min on my machine. To disable set CCACHE_DISABLE=1 in the environment It's not enabled for ./java/ci/build-in-docker.sh Signed-off-by: Gera Shegalov Authors: - Gera Shegalov (https://github.com/gerashegalov) Approvers: - Jason Lowe (https://github.com/jlowe) URL: https://github.com/rapidsai/cudf/pull/10790 --- build.sh | 19 +++++++++++++++++-- java/ci/Dockerfile.centos7 | 20 +++++++++++++++++++- java/pom.xml | 2 ++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 48182ca1a6f..ab3bd0e7a89 100755 --- a/build.sh +++ b/build.sh @@ -112,16 +112,22 @@ function buildLibCudfJniInDocker { local localMavenRepo=${LOCAL_MAVEN_REPO:-"$HOME/.m2/repository"} local workspaceRepoDir="$workspaceDir/cudf" local workspaceMavenRepoDir="$workspaceDir/.m2/repository" + local workspaceCcacheDir="$workspaceDir/.ccache" mkdir -p "$CUDF_JAR_JAVA_BUILD_DIR/libcudf-cmake-build" + mkdir -p "$HOME/.ccache" "$HOME/.m2" nvidia-docker build \ -f java/ci/Dockerfile.centos7 \ --build-arg CUDA_VERSION=${cudaVersion} \ -t $imageName . nvidia-docker run -it -u $(id -u):$(id -g) --rm \ + -e PARALLEL_LEVEL \ + -e CCACHE_DISABLE \ + -e CCACHE_DIR="$workspaceCcacheDir" \ -v "/etc/group:/etc/group:ro" \ -v "/etc/passwd:/etc/passwd:ro" \ -v "/etc/shadow:/etc/shadow:ro" \ -v "/etc/sudoers.d:/etc/sudoers.d:ro" \ + -v "$HOME/.ccache:$workspaceCcacheDir:rw" \ -v "$REPODIR:$workspaceRepoDir:rw" \ -v "$localMavenRepo:$workspaceMavenRepoDir:rw" \ --workdir "$workspaceRepoDir/java/target/libcudf-cmake-build" \ @@ -129,11 +135,16 @@ function buildLibCudfJniInDocker { scl enable devtoolset-9 \ "cmake $workspaceRepoDir/cpp \ -G${CMAKE_GENERATOR} \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_LINKER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCUDA_STATIC_RUNTIME=ON \ -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ - -DCMAKE_INSTALL_PREFIX==/usr/local/rapids \ - -DUSE_NVTX=ON -DCUDF_USE_ARROW_STATIC=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local/rapids \ + -DUSE_NVTX=ON \ + -DCUDF_USE_ARROW_STATIC=ON \ -DCUDF_ENABLE_ARROW_S3=OFF \ -DBUILD_TESTS=OFF \ -DPER_THREAD_DEFAULT_STREAM=ON \ @@ -145,6 +156,10 @@ function buildLibCudfJniInDocker { -Dmaven.repo.local=$workspaceMavenRepoDir \ -DskipTests=${SKIP_TESTS:-false} \ -Dparallel.level=${PARALLEL_LEVEL} \ + -Dcmake.ccache.opts='-DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_LINKER_LAUNCHER=ccache' \ -DCUDF_CPP_BUILD_DIR=$workspaceRepoDir/java/target/libcudf-cmake-build \ -DCUDA_STATIC_RUNTIME=ON \ -DPER_THREAD_DEFAULT_STREAM=ON \ diff --git a/java/ci/Dockerfile.centos7 b/java/ci/Dockerfile.centos7 index dc8c0e4a95b..7993804554d 100644 --- a/java/ci/Dockerfile.centos7 +++ b/java/ci/Dockerfile.centos7 @@ -26,8 +26,9 @@ ARG CUDA_VERSION=11.5.0 FROM gpuci/cuda:$CUDA_VERSION-devel-centos7 ### Install basic requirements +ARG DEVTOOLSET_VERSION=9 RUN yum install -y centos-release-scl -RUN yum install -y devtoolset-9 epel-release +RUN yum install -y devtoolset-${DEVTOOLSET_VERSION} epel-release RUN yum install -y git zlib-devel maven tar wget patch ninja-build ## pre-create the CMAKE_INSTALL_PREFIX folder, set writable by any user for Jenkins @@ -37,4 +38,21 @@ ARG CMAKE_VERSION=3.22.3 RUN cd /usr/local/ && wget --quiet https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \ tar zxf cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \ rm cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz + ENV PATH /usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:$PATH + +ARG CCACHE_VERSION=4.6 +RUN cd /tmp && wget --quiet https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ + tar zxf ccache-${CCACHE_VERSION}.tar.gz && \ + rm ccache-${CCACHE_VERSION}.tar.gz && \ + cd ccache-${CCACHE_VERSION} && \ + mkdir build && \ + cd build && \ + scl enable devtoolset-${DEVTOOLSET_VERSION} \ + "cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DZSTD_FROM_INTERNET=ON \ + -DREDIS_STORAGE_BACKEND=OFF && \ + cmake --build . --parallel ${PARALLEL_LEVEL} --target install" && \ + cd ../.. && \ + rm -rf ccache-${CCACHE_VERSION} diff --git a/java/pom.xml b/java/pom.xml index 50b6ca59440..31a79ec9801 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -173,6 +173,7 @@ 0.15.1 4 + @@ -382,6 +383,7 @@ failonerror="true" executable="cmake"> +