Consolidate C++ conda recipes and add libraft-tests package

Signed-off-by: Jordan Jacobelli <[email protected]>

ci/cpu/build.sh

@@ -65,13 +65,8 @@ conda list --show-channel-urls
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
-# FIXME: for now, force the building of all packages so they are built on a
-# machine with a single CUDA version, then have the gpu/build.sh script simply
-# install. This should eliminate a mismatch between different CUDA versions on
-# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
 if [ "$BUILD_LIBRAFT" == "1" ]; then
-  BUILD_RAFT=1
-  # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR}
+  # If we are doing CUDA builds, libraft package is located at ${CONDA_BLD_DIR}
   CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
 else
   # If we are doing Python builds only, libraft package is placed here by Project Flash
@@ -85,29 +80,17 @@ gpuci_mamba_retry install -c conda-forge boa
 ###############################################################################
 
 if [ "$BUILD_LIBRAFT" == "1" ]; then
-  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
+  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft
   else
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft
     gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_headers/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_headers/work
-
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_nn/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_nn/work
-
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
-    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_distance/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_distance/work
+    mkdir -p ${CONDA_BLD_DIR}/libraft/work
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi
 else
-  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
+  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
 fi
 
 if [ "$BUILD_RAFT" == '1' ]; then

ci/cpu/upload.sh

@@ -10,8 +10,9 @@ export GPUCI_RETRY_MAX=3
 export GPUCI_RETRY_SLEEP=30
 
 # Set label option.
-# LABEL_OPTION="--label testing"
-LABEL_OPTION="--label main"
+# TODO: Switch back to main label
+LABEL_OPTION="--label testing"
+# LABEL_OPTION="--label main"
 
 # Skip uploads unless BUILD_MODE == "branch"
 if [ ${BUILD_MODE} != "branch" ]; then
@@ -25,43 +26,21 @@ if [ -z "$MY_UPLOAD_KEY" ]; then
   return 0
 fi
 
-################################################################################
-# SETUP - Get conda file output locations
-################################################################################
-
-gpuci_logger "Get conda file output locations"
-
-export LIBRAFT_HEADERS_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_headers --output`
-export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_nn --output`
-export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_distance --output`
-export PYRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output`
-export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output`
-
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
-
-  test -e ${LIBRAFT_HEADERS_FILE}
-  echo "Upload libraft-headers"
-  echo ${LIBRAFT_HEADERS_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
-
-  test -e ${LIBRAFT_NN_FILE}
-  echo "Upload libraft-nn"
-  echo ${LIBRAFT_NN_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
-
-  test -e ${LIBRAFT_DISTANCE_FILE}
-  echo "Upload libraft-distance"
-  echo ${LIBRAFT_DISTANCE_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
+  LIBRAFT_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft --output)
+  echo "Upload libraft-headers, libraft-nn, libraft-distance and libraft-tests"
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${LIBRAFT_FILES}
 fi
 
 if [[ "$BUILD_RAFT" == "1" ]]; then
+  PYRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output)
+  PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output)
   test -e ${PYRAFT_FILE}
   echo "Upload pyraft"
   echo ${PYRAFT_FILE}

ci/gpu/build.sh

@@ -25,18 +25,11 @@ export HOME="$WORKSPACE"
 cd "$WORKSPACE"
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-export SCCACHE_S3_KEY_PREFIX="libraft-$(uname -m)"
-export SCCACHE_BUCKET="rapids-sccache"
-export SCCACHE_REGION="us-west-2"
-export SCCACHE_IDLE_TIMEOUT="32768"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -47,40 +40,14 @@ env
 gpuci_logger "Check GPU usage"
 nvidia-smi
 
-# temporary usage of gpuci_mamba_retry install with packages listed here, looking into
-# using the repos yaml files for this
 gpuci_logger "Activate conda env"
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
-gpuci_logger "Installing packages needed for RAFT"
-gpuci_mamba_retry install -y -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
-      "cudatoolkit=${CUDA_REL}" \
-      "libcusolver>=11.2.1" \
-      "cudf=${MINOR_VERSION}" \
-      "rmm=${MINOR_VERSION}" \
-      "breathe" \
-      "dask-cudf=${MINOR_VERSION}" \
-      "dask-cuda=${MINOR_VERSION}" \
-      "ucx-py=${UCX_PY_VERSION}" \
-      "rapids-build-env=${MINOR_VERSION}.*" \
-      "rapids-notebook-env=${MINOR_VERSION}.*"
-
-if [ "$(arch)" = "x86_64" ]; then
-    gpuci_mamba_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
-        "rapids-doc-env=${MINOR_VERSION}.*"
-fi
-
-# Install the master version of dask, distributed, and dask-ml
-gpuci_logger "Install the master version of dask and distributed"
-set -x
-pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
-pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
-set +x
 
 # Install pre-built conda packages from previous CI step
 gpuci_logger "Install libraft conda packages from CPU job"
-export LIBRAFT_CONDA_PACKAGES="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
-gpuci_mamba_retry install -c "${LIBRAFT_CONDA_PACKAGES}" libraft-headers libraft-distance libraft-nn
+CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
+gpuci_mamba_retry install -c "${CONDA_ARTIFACT_PATH}" libraft-headers libraft-distance libraft-nn libraft-tests
 
 gpuci_logger "Check compiler versions"
 python --version
@@ -96,24 +63,16 @@ conda list --show-channel-urls
 # BUILD - Build RAFT tests
 ################################################################################
 
-gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH"
-
-export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
-
-gpuci_logger "Build C++ and Python targets"
-# These should link against the existing shared libs
-if hasArg --skip-tests; then
-  "$WORKSPACE/build.sh" pyraft pylibraft libraft -v --cmake-args=\"-DFIND_RAFT_CPP=ON\"
-else
-  "$WORKSPACE/build.sh" pyraft pylibraft libraft tests bench -v --cmake-args=\"-DFIND_RAFT_CPP=ON\"
-fi
+gpuci_logger "Build and install Python targets"
+CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+gpuci_mamba_retry install boa
+gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pyraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
+gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
+gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pyraft pylibraft
 
 gpuci_logger "sccache stats"
 sccache --show-stats
 
-gpuci_logger "Building docs"
-"$WORKSPACE/build.sh" docs -v
-
 ################################################################################
 # TEST - Run GoogleTest and py.tests for RAFT
 ################################################################################
@@ -123,22 +82,30 @@ if hasArg --skip-tests; then
     exit 0
 fi
 
+# Install the master version of dask, distributed, and dask-ml
+gpuci_logger "Install the master version of dask and distributed"
+set -x
+pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
+pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
+set +x
+
 gpuci_logger "Check GPU usage"
 nvidia-smi
 
 gpuci_logger "GoogleTest for raft"
-cd "$WORKSPACE/cpp/build"
-GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test_raft
+cd "$WORKSPACE"
+GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" "$CONDA_PREFIX/bin/libraft/gtests/test_raft"
 
 gpuci_logger "Python pytest for pyraft"
-cd "$WORKSPACE/python/raft"
+cd "$WORKSPACE/python/raft/raft/test"
 python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pyraft.xml" -v -s
 
 gpuci_logger "Python pytest for pylibraft"
-cd "$WORKSPACE/python/pylibraft"
+cd "$WORKSPACE/python/pylibraft/pylibraft/test"
 python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pylibraft.xml" -v -s
 
 if [ "$(arch)" = "x86_64" ]; then
   gpuci_logger "Building docs"
+  gpuci_mamba_retry install "rapids-doc-env=${MINOR_VERSION}.*"
   "$WORKSPACE/build.sh" docs -v
 fi

conda/recipes/libraft/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+./build.sh libraft tests bench -v --allgpuarch --compile-dist --compile-nn --no-nvtx

conda/recipes/libraft/conda_build_config.yaml

@@ -0,0 +1,14 @@
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
+nccl_version:
+  - ">=2.9.9"
+
+gtest_version:
+  - "=1.10.0"
+
+libcusolver_version:
+  - ">=11.2.1"
+
+libfaiss_version:
+  - "1.7.0 *_cuda"

conda/recipes/libraft_headers/build.sh → conda/recipes/libraft/install_libraft_distance.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 # Copyright (c) 2022, NVIDIA CORPORATION.
 
-./build.sh libraft --install -v --allgpuarch --no-nvtx
+cmake --install cpp/build --component raft_distance

conda/recipes/libraft/install_libraft_headers.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component raft

conda/recipes/libraft/install_libraft_nn.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component raft_nn

conda/recipes/libraft/install_libraft_tests.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component testing