Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updating cugraph to use consolidated libraft target #3348

Merged
merged 6 commits into from
Mar 21, 2023
Merged

Updating cugraph to use consolidated libraft target #3348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7f0b1fb
Updating cugraph to use consolidated libraft target
cjnolet Mar 18, 2023
c3c813a
Pinning raft consolidation branch
cjnolet Mar 18, 2023
7d28f46
Updating get_raft pinned branch
cjnolet Mar 20, 2023
5d1f347
temporarily update CI scripts to use new libraft
cjnolet Mar 21, 2023
684d238
Using proper path
cjnolet Mar 21, 2023
8171a47
Using proper command
cjnolet Mar 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions ci/build_cpp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2022, NVIDIA CORPORATION.
# Copyright (c) 2022-2023, NVIDIA CORPORATION.

set -euo pipefail

Expand All @@ -11,6 +11,8 @@ rapids-print-env

rapids-logger "Begin cpp build"

rapids-mamba-retry mambabuild conda/recipes/libcugraph
LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_cpp_cuda11_$(arch).tar.gz)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cjnolet, @sevagh, changes like this are fine for temporarily testing upstream packages, but they should never be merged.

when changes like this are merged, there's too much potential to forget to revert them in the future which will cause unexpected issues in later builds.


rapids-mamba-retry mambabuild --channel "${LIBRAFT_CHANNEL}" conda/recipes/libcugraph

rapids-upload-conda-to-s3 cpp
15 changes: 14 additions & 1 deletion ci/build_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2022, NVIDIA CORPORATION.
# Copyright (c) 2022-2023, NVIDIA CORPORATION.

set -euo pipefail

Expand All @@ -10,6 +10,9 @@ export CMAKE_GENERATOR=Ninja
rapids-print-env

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PY_VER=${RAPIDS_PY_VERSION//./}
LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_cpp_cuda11_$(arch).tar.gz)
RAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_python_cuda11_${PY_VER}_$(arch).tar.gz)

rapids-logger "Begin py build"

Expand All @@ -18,29 +21,39 @@ rapids-logger "Begin py build"
rapids-mamba-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/pylibcugraph

rapids-mamba-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
conda/recipes/cugraph

rapids-mamba-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
conda/recipes/cugraph-service

rapids-mamba-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
conda/recipes/cugraph-pyg

rapids-mamba-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
--channel dglteam \
--channel pytorch \
Expand Down
2 changes: 2 additions & 0 deletions ci/test_cpp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@ conda activate test
set -u

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_cpp_cuda11_$(arch).tar.gz)
RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
mkdir -p "${RAPIDS_TESTS_DIR}"

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
libcugraph libcugraph_etl libcugraph-tests

rapids-logger "Check GPU usage"
Expand Down
5 changes: 5 additions & 0 deletions ci/test_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ set -u
rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
PY_VER=${RAPIDS_PY_VERSION//./}
LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_cpp_cuda11_$(arch).tar.gz)
RAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1333/c60c0cb/raft_conda_python_cuda11_${PY_VER}_$(arch).tar.gz)

RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
Expand All @@ -31,6 +34,8 @@ rapids-print-env
rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph \
pylibcugraph \
cugraph \
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-x86_64.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ dependencies:
- ipython
- libcudf=23.04.*
- libcugraphops=23.04.*
- libraft-distance=23.04.*
- libraft-headers=23.04.*
- libraft=23.04.*
- librmm=23.04.*
- nbsphinx
- nccl>=2.9.9
Expand Down
4 changes: 2 additions & 2 deletions conda/recipes/cugraph/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ requirements:
- libcusolver-dev {{ libcusolver_host_version }}
- libcusparse {{ libcusparse_host_version }}
- libcusparse-dev {{ libcusparse_host_version }}
- libraft-distance ={{ minor_version }}
- libraft ={{ minor_version }}
- libraft-headers ={{ minor_version }}
- pylibraft ={{ minor_version}}
- python x.x
Expand All @@ -75,7 +75,7 @@ requirements:
- dask >=2023.1.1
- distributed >=2023.1.1
- libcugraph ={{ version }}
- libraft-distance ={{ minor_version }}
- libraft ={{ minor_version }}
- libraft-headers ={{ minor_version }}
- pylibcugraph ={{ version }}
- pylibraft ={{ minor_version }}
Expand Down
4 changes: 2 additions & 2 deletions conda/recipes/libcugraph/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ requirements:
- libcusolver-dev {{ libcusolver_host_version }}
- libcusparse {{ libcusparse_host_version }}
- libcusparse-dev {{ libcusparse_host_version }}
- libraft-distance ={{ minor_version }}
- libraft ={{ minor_version }}
- libraft-headers ={{ minor_version }}
- librmm ={{ minor_version }}
- nccl {{ nccl_version }}
Expand All @@ -77,7 +77,7 @@ outputs:
run:
- cudatoolkit {{ cuda_spec }}
- libcugraphops ={{ minor_version }}
- libraft-distance ={{ minor_version }}
- libraft ={{ minor_version }}
- libraft-headers ={{ minor_version }}
- librmm ={{ minor_version }}
- nccl {{ nccl_version }}
Expand Down
2 changes: 1 addition & 1 deletion conda/recipes/pylibcugraph/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ requirements:
- libcusolver-dev {{ libcusolver_host_version }}
- libcusparse {{ libcusparse_host_version }}
- libcusparse-dev {{ libcusparse_host_version }}
- libraft-distance ={{ minor_version }}
- libraft ={{ minor_version }}
- libraft-headers ={{ minor_version }}
- pylibraft ={{ minor_version}}
- python x.x
Expand Down
14 changes: 7 additions & 7 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc (useful for cud
option(BUILD_TESTS "Configure CMake to build tests" ON)
option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON)
option(USE_RAFT_STATIC "Build raft as a static library" OFF)
option(CUGRAPH_COMPILE_RAFT_DIST_LIBS "Compile the raft library instead of using it header-only" ON)
option(CUGRAPH_COMPILE_RAFT_LIB "Compile the raft library instead of using it header-only" ON)
option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF)
option(CUGRAPH_USE_CUGRAPH_OPS_STATIC "Build and statically link the cugraph-ops library" OFF)
option(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL "Exclude cugraph-ops targets from cuGraph's 'all' target" OFF)
Expand Down Expand Up @@ -352,10 +352,10 @@ if (USE_CUGRAPH_OPS)
rmm::rmm
cugraph-ops::cugraph-ops++
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::raft>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::distance>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::compiled>
PRIVATE
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::raft>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::distance>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::compiled>
cuco::cuco
cugraph::cuHornet
NCCL::NCCL
Expand All @@ -365,10 +365,10 @@ else()
PUBLIC
rmm::rmm
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::raft>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::distance>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::compiled>
PRIVATE
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::raft>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::distance>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::compiled>
cuco::cuco
cugraph::cuHornet
NCCL::NCCL
Expand Down Expand Up @@ -468,12 +468,12 @@ target_link_libraries(cugraph_c
CUDA::cusparse${_ctk_static_suffix}
rmm::rmm
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::raft>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::distance>
$<$<NOT:$<BOOL:${CUDA_STATIC_RUNTIME}>>:raft::compiled>
PRIVATE
cuco::cuco
cugraph::cugraph
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::raft>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::distance>
$<$<BOOL:${CUDA_STATIC_RUNTIME}>:raft::compiled>
)

################################################################################
Expand Down
10 changes: 4 additions & 6 deletions cpp/cmake/thirdparty/get_raft.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set(CUGRAPH_BRANCH_VERSION_raft "${CUGRAPH_VERSION_MAJOR}.${CUGRAPH_VERSION_MINO

function(find_and_configure_raft)

set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC COMPILE_RAFT_DIST_LIBS)
set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC COMPILE_RAFT_LIB)
cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN} )

if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${CUGRAPH_BRANCH_VERSION_raft}")
Expand All @@ -39,19 +39,17 @@ function(find_and_configure_raft)
GLOBAL_TARGETS raft::raft
BUILD_EXPORT_SET cugraph-exports
INSTALL_EXPORT_SET cugraph-exports
COMPONENTS distance
COMPONENTS compiled
CPM_ARGS
EXCLUDE_FROM_ALL TRUE
GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git
GIT_TAG ${PKG_PINNED_TAG}
SOURCE_SUBDIR cpp
OPTIONS
"RAFT_COMPILE_LIBRARIES OFF"
"RAFT_COMPILE_DIST_LIBRARY ${PKG_COMPILE_RAFT_DIST_LIBS}"
"RAFT_COMPILE_LIBRARY ${PKG_COMPILE_RAFT_LIB}"
"BUILD_TESTS OFF"
"BUILD_BENCH OFF"
"BUILD_SHARED_LIBS ${BUILD_RAFT_SHARED}"
"RAFT_ENABLE_cuco_DEPENDENCY OFF"
)

if(raft_ADDED)
Expand All @@ -74,5 +72,5 @@ find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft}
# even if it's already installed.
CLONE_ON_PIN ON
USE_RAFT_STATIC ${USE_RAFT_STATIC}
COMPILE_RAFT_DIST_LIBS ${CUGRAPH_COMPILE_RAFT_DIST_LIBS}
COMPILE_RAFT_LIB ${CUGRAPH_COMPILE_RAFT_LIB}
)
2 changes: 1 addition & 1 deletion cpp/src/community/legacy/spectral_clustering.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
#include <cugraph/legacy/graph.hpp>
#include <cugraph/utilities/error.hpp>

#if defined RAFT_DISTANCE_COMPILED
#if defined RAFT_COMPILED
#include <raft/distance/specializations.cuh>
#endif
#include <raft/spectral/modularity_maximization.cuh>
Expand Down
2 changes: 1 addition & 1 deletion dependencies.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ dependencies:
- gtest=1.10.0
- libcugraphops=23.04.*
- libraft-headers=23.04.*
- libraft-distance=23.04.*
- libraft=23.04.*
- librmm=23.04.*
- openmpi # Required for building cpp-mgtests (multi-GPU tests)
specific:
Expand Down