From 56d2d4d57dca7817f865b790f86df6f7822902ee Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 2 Nov 2023 20:33:13 -0500 Subject: [PATCH 1/5] Build with spdlog 1.12.0 and fmt 10.1.1. --- cpp/template/cmake/thirdparty/fetch_rapids.cmake | 2 ++ fetch_rapids.cmake | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake index 22afb1a3f5..4042c81e0f 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -14,6 +14,8 @@ # Use this variable to update RAPIDS and RAFT versions set(RAPIDS_VERSION "23.12") +set(rapids-cmake-repo kkraus14/rapids-cmake) +set(rapids-cmake-branch spdlog_1.12_fmt_10) if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index c74c09fb26..d9f1a1b1dc 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -11,6 +11,8 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. # ============================================================================= +set(rapids-cmake-repo kkraus14/rapids-cmake) +set(rapids-cmake-branch spdlog_1.12_fmt_10) if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.12/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake From 434c7ca94fdf4552ac4afac4c4c952d466aff96a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 20 Nov 2023 16:23:26 -0600 Subject: [PATCH 2/5] Use upstream rmm artifacts. --- ci/build_cpp.sh | 6 +++++- ci/build_python.sh | 21 ++++++++++++++++----- ci/test_cpp.sh | 2 ++ ci/test_python.sh | 4 ++++ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 178ce723a5..0e62239695 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,10 +9,14 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) + version=$(rapids-generate-version) rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libraft +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ + --channel "${LIBRMM_CHANNEL}" \ + conda/recipes/libraft rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index cf34776542..2b5f7cf76f 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,17 +9,20 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 python) + rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) version=$(rapids-generate-version) git_commit=$(git rev-parse HEAD) -export RAPIDS_PACKAGE_VERSION=${version} +export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION package_dir="python" -for package_name in pylibraft raft-dask; do +for package_name in pylibraft raft-dask; do underscore_package_name=$(echo "${package_name}" | tr "-" "_") sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" done @@ -29,19 +32,25 @@ done rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ conda/recipes/pylibraft rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-dask # Build ann-bench for each cuda and python version rapids-conda-retry mambabuild \ ---no-test \ ---channel "${CPP_CHANNEL}" \ ---channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python @@ -51,6 +60,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench-cpu fi diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 0f8efb171e..02041ee0c0 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -19,6 +19,7 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -26,6 +27,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ libraft-headers libraft libraft-tests rapids-logger "Check GPU usage" diff --git a/ci/test_python.sh b/ci/test_python.sh index cb6b7631e4..d6026a61ac 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -21,6 +21,8 @@ set -u rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) +RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -31,6 +33,8 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${RMM_CHANNEL}" \ libraft libraft-headers pylibraft raft-dask rapids-logger "Check GPU usage" From d5551c16477d732d58be1dd5222a6660dda215b5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 22 Nov 2023 13:09:42 -0600 Subject: [PATCH 3/5] Update pinnings. --- conda/recipes/libraft/conda_build_config.yaml | 4 ++-- conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 25493a34fa..f1229281bb 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -73,7 +73,7 @@ cuda11_cuda_profiler_api_run_version: - ">=11.4.240,<12" spdlog_version: - - ">=1.11.0,<1.12" + - ">=1.12.0,<1.13" fmt_version: - - ">=9.1.0,<10" + - ">=10.1.1,<11" diff --git a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml index fda3e4e53d..93a5532962 100644 --- a/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml +++ b/conda/recipes/raft-ann-bench-cpu/conda_build_config.yaml @@ -20,7 +20,7 @@ nlohmann_json_version: - ">=3.11.2" spdlog_version: - - ">=1.11.0,<1.12" + - ">=1.12.0,<1.13" fmt_version: - - ">=9.1.0,<10" + - ">=10.1.1,<11" From 337a9a8fd4d85e1e9f43ec603ba10f72d4406d1b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 29 Nov 2023 09:01:42 -0600 Subject: [PATCH 4/5] Rerun CI. From 889c62ced27c984324c897b21da980125ce072fe Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 30 Nov 2023 10:55:53 -0600 Subject: [PATCH 5/5] Revert build changes. --- ci/build_cpp.sh | 6 +----- ci/build_python.sh | 13 +------------ ci/test_cpp.sh | 2 -- ci/test_python.sh | 4 ---- cpp/template/cmake/thirdparty/fetch_rapids.cmake | 2 -- fetch_rapids.cmake | 2 -- 6 files changed, 2 insertions(+), 27 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 0e62239695..178ce723a5 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,14 +9,10 @@ export CMAKE_GENERATOR=Ninja rapids-print-env -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) - version=$(rapids-generate-version) rapids-logger "Begin cpp build" -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ - --channel "${LIBRMM_CHANNEL}" \ - conda/recipes/libraft +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libraft rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 2b5f7cf76f..3e67edd5db 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,9 +9,6 @@ export CMAKE_GENERATOR=Ninja rapids-print-env -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 python) - rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) @@ -32,15 +29,11 @@ done rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ conda/recipes/pylibraft rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-dask @@ -48,10 +41,8 @@ rapids-conda-retry mambabuild \ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ -conda/recipes/raft-ann-bench + conda/recipes/raft-ann-bench # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python # version @@ -60,8 +51,6 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench-cpu fi diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 02041ee0c0..0f8efb171e 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -19,7 +19,6 @@ conda activate test set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -27,7 +26,6 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ libraft-headers libraft libraft-tests rapids-logger "Check GPU usage" diff --git a/ci/test_python.sh b/ci/test_python.sh index d6026a61ac..cb6b7631e4 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -21,8 +21,6 @@ set -u rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) -LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 cpp) -RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1374 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -33,8 +31,6 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${RMM_CHANNEL}" \ libraft libraft-headers pylibraft raft-dask rapids-logger "Check GPU usage" diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake index 93972cb43b..15b6c43a6f 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -14,8 +14,6 @@ # Use this variable to update RAPIDS and RAFT versions set(RAPIDS_VERSION "24.02") -set(rapids-cmake-repo kkraus14/rapids-cmake) -set(rapids-cmake-branch spdlog_1.12_fmt_10) if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index b9f45f1999..ca871c5759 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -11,8 +11,6 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. # ============================================================================= -set(rapids-cmake-repo kkraus14/rapids-cmake) -set(rapids-cmake-branch spdlog_1.12_fmt_10) if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.02/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake