Consolidate C++ conda recipes and add libraft-tests package (#641)

This PR includes the following changes:
- Adds a `libraft-tests` package to the `libraft-split` recipe
  - This is a prerequisite for removing "Project Flash" from our build/CI scripts
   - The `libraft-tests` package was added as an additional output to the `libraft-split` recipe
- Consolidates remaining C++ recipes into `libraft-split` recipe
   - This gets rid of a lot of duplicate code between the recipes and reduces the number of times we have to call conda build in our CI scripts
- Migrate the "from sources" builds done in GPU tests by building packages using conda. This is done for the following reasons:
  - This is required step to improve the Ops CI/CD setup to a more convenient pipeline
  - This is required to start using conda compilers and `mamba` to build RAPIDS packages
  - This prevent us from manually managing and installing the dependencies in GPU job
  - This ensure the packages can be installed (no conda conflict while installing)
  - This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested

Dependency version specs are now specified in `conda/recipes/libraft/conda_build_config.yaml`. The exception here is the version spec used for cudatoolkit since that comes from an environment variable in the CI process.

Authors:
  - Jordan Jacobelli (https://github.com/Ethyling)

Approvers:
  - Corey J. Nolet (https://github.com/cjnolet)
  - AJ Schmidt (https://github.com/ajschmidt8)

URL: #641

ci/cpu/build.sh

@@ -22,7 +22,9 @@ cd $WORKSPACE
 
 # If nightly build, append current YYMMDD to version
 if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
-  export VERSION_SUFFIX=`date +%y%m%d`
+  export VERSION_SUFFIX=$(date +%y%m%d)
+else
+  export VERSION_SUFFIX=""
 fi
 
 # Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
@@ -65,13 +67,8 @@ conda list --show-channel-urls
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
-# FIXME: for now, force the building of all packages so they are built on a
-# machine with a single CUDA version, then have the gpu/build.sh script simply
-# install. This should eliminate a mismatch between different CUDA versions on
-# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
 if [ "$BUILD_LIBRAFT" == "1" ]; then
-  BUILD_RAFT=1
-  # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR}
+  # If we are doing CUDA builds, libraft package is located at ${CONDA_BLD_DIR}
   CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
 else
   # If we are doing Python builds only, libraft package is placed here by Project Flash
@@ -85,29 +82,17 @@ gpuci_mamba_retry install -c conda-forge boa
 ###############################################################################
 
 if [ "$BUILD_LIBRAFT" == "1" ]; then
-  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
+  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft
   else
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
-    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_headers/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_headers/work
-
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_nn/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_nn/work
-
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft
     gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
-    mkdir -p ${CONDA_BLD_DIR}/libraft_distance/work
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_distance/work
+    mkdir -p ${CONDA_BLD_DIR}/libraft/work
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi
 else
-  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
+  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance, libraft-headers and libraft-tests"
 fi
 
 if [ "$BUILD_RAFT" == '1' ]; then

ci/cpu/upload.sh

@@ -10,7 +10,7 @@ export GPUCI_RETRY_MAX=3
 export GPUCI_RETRY_SLEEP=30
 
 # Set label option.
-# LABEL_OPTION="--label testing"
+#LABEL_OPTION="--label testing"
 LABEL_OPTION="--label main"
 
 # Skip uploads unless BUILD_MODE == "branch"
@@ -25,43 +25,21 @@ if [ -z "$MY_UPLOAD_KEY" ]; then
   return 0
 fi
 
-################################################################################
-# SETUP - Get conda file output locations
-################################################################################
-
-gpuci_logger "Get conda file output locations"
-
-export LIBRAFT_HEADERS_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_headers --output`
-export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_nn --output`
-export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft_distance --output`
-export PYRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output`
-export PYLIBRAFT_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output`
-
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
-
-  test -e ${LIBRAFT_HEADERS_FILE}
-  echo "Upload libraft-headers"
-  echo ${LIBRAFT_HEADERS_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
-
-  test -e ${LIBRAFT_NN_FILE}
-  echo "Upload libraft-nn"
-  echo ${LIBRAFT_NN_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
-
-  test -e ${LIBRAFT_DISTANCE_FILE}
-  echo "Upload libraft-distance"
-  echo ${LIBRAFT_DISTANCE_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
+  LIBRAFT_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/libraft --output)
+  echo "Upload libraft-headers, libraft-nn, libraft-distance and libraft-tests"
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${LIBRAFT_FILES}
 fi
 
 if [[ "$BUILD_RAFT" == "1" ]]; then
+  PYRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pyraft --python=$PYTHON --output)
+  PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output)
   test -e ${PYRAFT_FILE}
   echo "Upload pyraft"
   echo ${PYRAFT_FILE}

ci/gpu/build.sh

@@ -25,18 +25,11 @@ export HOME="$WORKSPACE"
 cd "$WORKSPACE"
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # ucx-py version
 export UCX_PY_VERSION='0.27.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-export SCCACHE_S3_KEY_PREFIX="libraft-$(uname -m)"
-export SCCACHE_BUCKET="rapids-sccache"
-export SCCACHE_REGION="us-west-2"
-export SCCACHE_IDLE_TIMEOUT="32768"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -47,40 +40,14 @@ env
 gpuci_logger "Check GPU usage"
 nvidia-smi
 
-# temporary usage of gpuci_mamba_retry install with packages listed here, looking into
-# using the repos yaml files for this
 gpuci_logger "Activate conda env"
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
-gpuci_logger "Installing packages needed for RAFT"
-gpuci_mamba_retry install -y -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
-      "cudatoolkit=${CUDA_REL}" \
-      "libcusolver>=11.2.1" \
-      "cudf=${MINOR_VERSION}" \
-      "rmm=${MINOR_VERSION}" \
-      "breathe" \
-      "dask-cudf=${MINOR_VERSION}" \
-      "dask-cuda=${MINOR_VERSION}" \
-      "ucx-py=${UCX_PY_VERSION}" \
-      "rapids-build-env=${MINOR_VERSION}.*" \
-      "rapids-notebook-env=${MINOR_VERSION}.*"
-
-if [ "$(arch)" = "x86_64" ]; then
-    gpuci_mamba_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
-        "rapids-doc-env=${MINOR_VERSION}.*"
-fi
-
-# Install the master version of dask, distributed, and dask-ml
-gpuci_logger "Install the master version of dask and distributed"
-set -x
-pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
-pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
-set +x
 
 # Install pre-built conda packages from previous CI step
 gpuci_logger "Install libraft conda packages from CPU job"
-export LIBRAFT_CONDA_PACKAGES="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
-gpuci_mamba_retry install -c "${LIBRAFT_CONDA_PACKAGES}" libraft-headers libraft-distance libraft-nn
+CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/raft/cpu/.conda-bld/" # notice there is no `linux-64` here
+gpuci_mamba_retry install -c "${CONDA_ARTIFACT_PATH}" libraft-headers libraft-distance libraft-nn libraft-tests
 
 gpuci_logger "Check compiler versions"
 python --version
@@ -96,24 +63,16 @@ conda list --show-channel-urls
 # BUILD - Build RAFT tests
 ################################################################################
 
-gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH"
-
-export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
-
-gpuci_logger "Build C++ and Python targets"
-# These should link against the existing shared libs
-if hasArg --skip-tests; then
-  "$WORKSPACE/build.sh" pyraft pylibraft libraft -v
-else
-  "$WORKSPACE/build.sh" pyraft pylibraft libraft tests bench -v
-fi
+gpuci_logger "Build and install Python targets"
+CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+gpuci_mamba_retry install boa
+gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pyraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
+gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}"
+gpuci_mamba_retry install -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pyraft pylibraft
 
 gpuci_logger "sccache stats"
 sccache --show-stats
 
-gpuci_logger "Building docs"
-"$WORKSPACE/build.sh" docs -v
-
 ################################################################################
 # TEST - Run GoogleTest and py.tests for RAFT
 ################################################################################
@@ -123,22 +82,30 @@ if hasArg --skip-tests; then
     exit 0
 fi
 
+# Install the master version of dask, distributed, and dask-ml
+gpuci_logger "Install the master version of dask and distributed"
+set -x
+pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
+pip install "git+https://github.com/dask/[email protected]" --upgrade --no-deps
+set +x
+
 gpuci_logger "Check GPU usage"
 nvidia-smi
 
 gpuci_logger "GoogleTest for raft"
-cd "$WORKSPACE/cpp/build"
-GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test_raft
+cd "$WORKSPACE"
+GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" "$CONDA_PREFIX/bin/libraft/gtests/test_raft"
 
 gpuci_logger "Python pytest for pyraft"
-cd "$WORKSPACE/python/raft"
+cd "$WORKSPACE/python/raft/raft/test"
 python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pyraft.xml" -v -s
 
 gpuci_logger "Python pytest for pylibraft"
-cd "$WORKSPACE/python/pylibraft"
+cd "$WORKSPACE/python/pylibraft/pylibraft/test"
 python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-pylibraft.xml" -v -s
 
 if [ "$(arch)" = "x86_64" ]; then
   gpuci_logger "Building docs"
+  gpuci_mamba_retry install "rapids-doc-env=${MINOR_VERSION}.*"
   "$WORKSPACE/build.sh" docs -v
 fi

conda/recipes/libraft/build_libraft_tests.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+./build.sh tests bench -v --allgpuarch --no-nvtx
+cmake --install cpp/build --component testing

conda/recipes/libraft/conda_build_config.yaml

@@ -0,0 +1,14 @@
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
+nccl_version:
+  - ">=2.9.9"
+
+gtest_version:
+  - "=1.10.0"
+
+libcusolver_version:
+  - ">=11.2.1"
+
+libfaiss_version:
+  - "1.7.0 *_cuda"