Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test CI with artifacts from rapidsai/rmm#1095 #3983

Closed
wants to merge 7 commits into from
Closed

Test CI with artifacts from rapidsai/rmm#1095 #3983

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
73ebcaf
Test CI with artifacts from rapidsai/rmm#1095
harrism Nov 7, 2023
7c69422
Merge branch 'branch-23.12' into test/test-cugraph-rmm-1095
harrism Nov 14, 2023
9393394
Merge branch 'branch-23.12' into test/test-cugraph-rmm-1095
harrism Nov 15, 2023
aeee873
Update to use gha tool and rerun
harrism Nov 16, 2023
55b69a2
Merge branch 'test/test-cugraph-rmm-1095' of github.com:harrism/cugra…
harrism Nov 16, 2023
b237099
Merge branch 'branch-23.12' into test/test-cugraph-rmm-1095
harrism Nov 16, 2023
867a1a1
Fix scripts
harrism Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion ci/build_cpp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,20 @@ source rapids-env-update

export CMAKE_GENERATOR=Ninja

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)

rapids-print-env

version=$(rapids-generate-version)

rapids-logger "Begin cpp build"

RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph
RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
--channel "${LIBRMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
conda/recipes/libcugraph

rapids-upload-conda-to-s3 cpp
13 changes: 13 additions & 0 deletions ci/build_docs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,22 @@ rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)
RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python)

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph \
pylibcugraph \
cugraph \
Expand Down
45 changes: 44 additions & 1 deletion ci/build_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ rapids-print-env

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)
RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python)

version=$(rapids-generate-version)
git_commit=$(git rev-parse HEAD)
export RAPIDS_PACKAGE_VERSION=${version}
Expand All @@ -19,7 +26,7 @@ echo "${version}" > VERSION
rapids-logger "Begin py build"

package_dir="python"
for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do
for package_name in pylibcugraph cugraph nx-cugraph cugraph-pyg cugraph-dgl; do
underscore_package_name=$(echo "${package_name}" | tr "-" "_")
sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py"
done
Expand All @@ -29,12 +36,24 @@ done
rapids-conda-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/pylibcugraph

rapids-conda-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/cugraph

# NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA
Expand All @@ -45,6 +64,12 @@ rapids-conda-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/nx-cugraph

# NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are
Expand All @@ -59,6 +84,12 @@ rapids-conda-retry mambabuild \
--no-test \
--channel "${CPP_CHANNEL}" \
--channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/cugraph-service

RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
Expand All @@ -72,6 +103,12 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
--channel pyg \
--channel pytorch \
--channel pytorch-nightly \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/cugraph-pyg

# Only CUDA 11 is supported right now due to PyTorch requirement.
Expand All @@ -82,6 +119,12 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
--channel dglteam \
--channel pytorch \
--channel pytorch-nightly \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
conda/recipes/cugraph-dgl
fi

Expand Down
8 changes: 8 additions & 0 deletions ci/test_cpp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,21 @@ conda activate test
set -u

CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)

RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
mkdir -p "${RAPIDS_TESTS_DIR}"

rapids-print-env

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
libcugraph libcugraph_etl libcugraph-tests

rapids-logger "Check GPU usage"
Expand Down
13 changes: 13 additions & 0 deletions ci/test_notebooks.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,22 @@ rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)
RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python)

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph pylibcugraph cugraph

NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
Expand Down
33 changes: 32 additions & 1 deletion ci/test_python.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ rapids-logger "Downloading artifacts from previous jobs"
CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)

LIBRMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 cpp)
RMM_CHANNEL=$(rapids-get-pr-conda-artifact rmm 1095 python)
LIBCUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 cpp)
CUDF_CHANNEL=$(rapids-get-pr-conda-artifact cudf 14365 python)
LIBRAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 cpp)
RAFT_CHANNEL=$(rapids-get-pr-conda-artifact raft 1964 python)

RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
Expand All @@ -31,6 +38,12 @@ rapids-print-env
rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph \
pylibcugraph \
cugraph \
Expand Down Expand Up @@ -152,6 +165,12 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
--channel pytorch-nightly \
--channel dglteam/label/cu118 \
--channel nvidia \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph \
pylibcugraph \
pylibcugraphops \
Expand Down Expand Up @@ -203,16 +222,28 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
--channel pyg \
--channel pytorch \
--channel nvidia \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
'pyg=2.3' \
'pytorch=2.0.0' \
'pytorch-cuda=11.8'

# Install pyg dependencies (which requires pip)
pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.0.0+cu118.html

rapids-mamba-retry install \
--channel "${CPP_CHANNEL}" \
--channel "${PYTHON_CHANNEL}" \
--channel "${LIBRMM_CHANNEL}" \
--channel "${RMM_CHANNEL}" \
--channel "${LIBCUDF_CHANNEL}" \
--channel "${CUDF_CHANNEL}" \
--channel "${LIBRAFT_CHANNEL}" \
--channel "${RAFT_CHANNEL}" \
libcugraph \
pylibcugraph \
pylibcugraphops \
Expand Down
Loading