From c482d2e5756f7bd4e98dd4ee04e320d6ef74c55c Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 14:56:20 -0500 Subject: [PATCH 1/7] replace `ccache` with `sccache` --- ci/gpu/build.sh | 6 +++++- conda/recipes/libraft_distance/meta.yaml | 10 ++++++---- conda/recipes/libraft_headers/meta.yaml | 10 ++++++---- conda/recipes/libraft_nn/meta.yaml | 10 ++++++---- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index e79c21f0c5..8f7f9c1ef4 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -21,7 +21,7 @@ export CUDA_REL=${CUDA_VERSION%.*} # Set home to the job's workspace export HOME="$WORKSPACE" -# Parse git describei +# Parse git describe cd "$WORKSPACE" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` @@ -29,6 +29,10 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` # ucx-py version export UCX_PY_VERSION='0.25.*' +export CMAKE_CUDA_COMPILER_LAUNCHER="sccache" +export CMAKE_CXX_COMPILER_LAUNCHER="sccache" +export CMAKE_C_COMPILER_LAUNCHER="sccache" + ################################################################################ # SETUP - Check environment ################################################################################ diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index 8c3a381a1f..f7aaf0d261 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -24,13 +24,15 @@ build: - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH - - CCACHE_DIR - - CCACHE_NOHASHDIR - - CCACHE_COMPILERCHECK - CMAKE_GENERATOR - CMAKE_C_COMPILER_LAUNCHER - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER + - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] + - SCCACHE_BUCKET=rapids-sccache + - SCCACHE_REGION=us-west-2 + - SCCACHE_IDLE_TIMEOUT=32768 requirements: build: @@ -54,4 +56,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft-distance library \ No newline at end of file + summary: libraft-distance library diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index c7aa793ae5..a03146a7d5 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -24,13 +24,15 @@ build: - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH - - CCACHE_DIR - - CCACHE_NOHASHDIR - - CCACHE_COMPILERCHECK - CMAKE_GENERATOR - CMAKE_C_COMPILER_LAUNCHER - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER + - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] + - SCCACHE_BUCKET=rapids-sccache + - SCCACHE_REGION=us-west-2 + - SCCACHE_IDLE_TIMEOUT=32768 requirements: build: @@ -53,4 +55,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft-headers library \ No newline at end of file + summary: libraft-headers library diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 710e99902b..53f44f2e97 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -24,13 +24,15 @@ build: - PARALLEL_LEVEL - VERSION_SUFFIX - PROJECT_FLASH - - CCACHE_DIR - - CCACHE_NOHASHDIR - - CCACHE_COMPILERCHECK - CMAKE_GENERATOR - CMAKE_C_COMPILER_LAUNCHER - CMAKE_CXX_COMPILER_LAUNCHER - CMAKE_CUDA_COMPILER_LAUNCHER + - SCCACHE_S3_KEY_PREFIX=libraft-aarch64 # [aarch64] + - SCCACHE_S3_KEY_PREFIX=libraft-linux64 # [linux64] + - SCCACHE_BUCKET=rapids-sccache + - SCCACHE_REGION=us-west-2 + - SCCACHE_IDLE_TIMEOUT=32768 requirements: build: @@ -53,4 +55,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft-nn library \ No newline at end of file + summary: libraft-nn library From d63132b58a935abf25ac1e1f595123365d187c55 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 15:58:29 -0500 Subject: [PATCH 2/7] expose AWS creds to conda builds for sccache --- conda/recipes/libraft_distance/meta.yaml | 2 ++ conda/recipes/libraft_headers/meta.yaml | 2 ++ conda/recipes/libraft_nn/meta.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index f7aaf0d261..d8402f5622 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -33,6 +33,8 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY requirements: build: diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index a03146a7d5..dc013c9135 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -33,6 +33,8 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY requirements: build: diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 53f44f2e97..16a11ef8b3 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -33,6 +33,8 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY requirements: build: From ace6cd9a88f008e6e06de67ff3d94b1b9ae4f4fe Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 15:58:36 -0500 Subject: [PATCH 3/7] show sccache stats after build --- ci/gpu/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 8f7f9c1ef4..ff22be0d20 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -90,12 +90,16 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" +sccache --zero-stats if hasArg --skip-tests; then "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs --nogtest else "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs fi +gpuci_logger "sccache stats" +sccache --show-stats + gpuci_logger "Building docs" "$WORKSPACE/build.sh" docs -v From f8289ceed1611b724c107ae890a0354384d9f359 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 16:32:48 -0500 Subject: [PATCH 4/7] add remaining sccache args to build script --- ci/gpu/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index ff22be0d20..47d7041cb4 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -32,6 +32,10 @@ export UCX_PY_VERSION='0.25.*' export CMAKE_CUDA_COMPILER_LAUNCHER="sccache" export CMAKE_CXX_COMPILER_LAUNCHER="sccache" export CMAKE_C_COMPILER_LAUNCHER="sccache" +export SCCACHE_S3_KEY_PREFIX="libraft-$(uname -m)" +export SCCACHE_BUCKET="rapids-sccache" +export SCCACHE_REGION="us-west-2" +export SCCACHE_IDLE_TIMEOUT="32768" ################################################################################ # SETUP - Check environment From 1ee09d0ebc47473101fb205389eecb8717d84613 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 16:34:22 -0500 Subject: [PATCH 5/7] Revert "expose AWS creds to conda builds for sccache" The `conda` recipes are not used in the CI process, so these credentials do not need to be exposed to the `conda` build environment. This reverts commit d63132b58a935abf25ac1e1f595123365d187c55. --- conda/recipes/libraft_distance/meta.yaml | 2 -- conda/recipes/libraft_headers/meta.yaml | 2 -- conda/recipes/libraft_nn/meta.yaml | 2 -- 3 files changed, 6 deletions(-) diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index d8402f5622..f7aaf0d261 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -33,8 +33,6 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY requirements: build: diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index dc013c9135..a03146a7d5 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -33,8 +33,6 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY requirements: build: diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 16a11ef8b3..53f44f2e97 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -33,8 +33,6 @@ build: - SCCACHE_BUCKET=rapids-sccache - SCCACHE_REGION=us-west-2 - SCCACHE_IDLE_TIMEOUT=32768 - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY requirements: build: From 8f54be317aee7c6dacec8020f479f17ac14b5b3a Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 31 Jan 2022 16:45:48 -0500 Subject: [PATCH 6/7] add -y flag to install cmd --- ci/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 47d7041cb4..8299b91790 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -53,7 +53,7 @@ gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids gpuci_logger "Installing packages needed for RAFT" -gpuci_mamba_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ +gpuci_mamba_retry install -y -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ "cudatoolkit=${CUDA_REL}" \ "libcusolver>=11.2.1" \ "cudf=${MINOR_VERSION}" \ From 2663fb608f7c195a1005ea29e04017954967bdc8 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 1 Feb 2022 14:50:08 -0500 Subject: [PATCH 7/7] rm --zero-stats --- ci/gpu/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 8299b91790..d019655b5e 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -94,7 +94,6 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -sccache --zero-stats if hasArg --skip-tests; then "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs --nogtest else