From b50344f91a4a2d6b6f2a2cf677b4ae8a750d92eb Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 13:18:39 -0500
Subject: [PATCH 01/40] Adding cpu checks for ci build

---
 ci/cpu/build.sh    | 121 +++++++++++++++++++++++++++++++++++++++++++++
 ci/cpu/prebuild.sh |  21 ++++++++
 ci/cpu/upload.sh   |  69 ++++++++++++++++++++++++++
 3 files changed, 211 insertions(+)
 create mode 100644 ci/cpu/build.sh
 create mode 100644 ci/cpu/prebuild.sh
 create mode 100644 ci/cpu/upload.sh

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
new file mode 100644
index 0000000000..fc8f4d76d6
--- /dev/null
+++ b/ci/cpu/build.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+#########################################
+#  RAFT CPU conda build script for CI   #
+#########################################
+set -e
+
+# Set path and build parallel level
+# openmpi dir is required on CentOS for finding MPI libs from cmake
+if [[ -e /etc/os-release ]] && (grep -qi centos /etc/os-release); then
+    export PATH=/opt/conda/bin:/usr/local/cuda/bin:/usr/lib64/openmpi/bin:$PATH
+else
+    export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+fi
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# If nightly build, append current YYMMDD to version
+if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
+  export VERSION_SUFFIX=`date +%y%m%d`
+fi
+
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
+# Use Ninja to build
+export CMAKE_GENERATOR="Ninja"
+export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
+
+# ucx-py version
+export UCX_PY_VERSION='0.25.*'
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+gpuci_logger "Check environment variables"
+env
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
+# Remove rapidsai-nightly channel if we are building main branch
+if [ "$SOURCE_BRANCH" = "main" ]; then
+  conda config --system --remove channels rapidsai-nightly
+fi
+
+gpuci_logger "Check versions"
+python --version
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+# FIXME: for now, force the building of all packages so they are built on a
+# machine with a single CUDA version, then have the gpu/build.sh script simply
+# install. This should eliminate a mismatch between different CUDA versions on
+# cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
+if [ "$BUILD_LIBRAFT" == '1' ]; then
+  BUILD_PYRAFT=1
+  # If we are doing CUDA + Python builds, libcugraph package is located at ${CONDA_BLD_DIR}
+  CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
+else
+  # If we are doing Python builds only, libcugraph package is placed here by Project Flash
+  CONDA_LOCAL_CHANNEL="ci/artifacts/raft/cpu/.conda-bld/"
+fi
+
+
+###############################################################################
+# BUILD - Conda package builds
+###############################################################################
+
+if [ "$BUILD_LIBRAFT" == '1' ]; then
+  gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-headers
+  else
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-headers
+    mkdir -p ${CONDA_BLD_DIR}/libraft
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
+  fi
+else
+  gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
+fi
+
+if [ "$BUILD_CUGRAPH" == "1" ]; then
+  gpuci_logger "Building conda packages for pyraft"
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
+  else
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
+    mkdir -p ${CONDA_BLD_DIR}/pyraft
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/pyraft/work
+  fi
+else
+  gpuci_logger "SKIPPING build of conda packages for pyraft"
+fi
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Upload conda packages"
+source ci/cpu/upload.sh
\ No newline at end of file
diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh
new file mode 100644
index 0000000000..69e19ffbdd
--- /dev/null
+++ b/ci/cpu/prebuild.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    #If project flash is not activate, always build both
+    export BUILD_PYRAFT=1
+    export BUILD_LIBRAFT=1
+fi
+
+export UPLOAD_LIBRAFT=1
diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
new file mode 100644
index 0000000000..c17ec57b0e
--- /dev/null
+++ b/ci/cpu/upload.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+
+export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
+export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
+export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --python=$PYTHON --output`
+export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+
+if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
+  # libraft-nn
+  test -e ${LIBRAFT_NN_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_NN_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
+
+  # libraft-distance
+  test -e ${LIBRAFT_DISTANCE_FILE}
+  echo "Upload libraft-distance"
+  echo ${LIBRAFT_DISTANCE_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
+
+  # libraft-headers
+  test -e ${LIBRAFT_HEADERS_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_HEADERS_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
+fi
+
+if [[ "$BUILD_PYRAFT" == "1" ]]; then
+  test -e ${PYRAFT_FILE}
+  echo "Upload pyraft"
+  echo ${PYRAFT_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${PYRAFT_FILE} --no-progress
+fi

From e7a1e9836e1648009006296b021b7c95047d0ed9 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 13:21:24 -0500
Subject: [PATCH 02/40] Updating years

---
 ci/cpu/build.sh    | 2 +-
 ci/cpu/prebuild.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index fc8f4d76d6..d804ef2312 100644
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+# Copyright (c) 2022, NVIDIA CORPORATION.
 #########################################
 #  RAFT CPU conda build script for CI   #
 #########################################
diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh
index 69e19ffbdd..e37bcedb52 100644
--- a/ci/cpu/prebuild.sh
+++ b/ci/cpu/prebuild.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+# Copyright (c) 2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

From 6e0628e3dbd602164bffddec408734ffc8d079db Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 13:25:24 -0500
Subject: [PATCH 03/40] Updates

---
 ci/cpu/build.sh    | 6 +++---
 ci/cpu/prebuild.sh | 0
 ci/cpu/upload.sh   | 0
 3 files changed, 3 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 ci/cpu/build.sh
 mode change 100644 => 100755 ci/cpu/prebuild.sh
 mode change 100644 => 100755 ci/cpu/upload.sh

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
old mode 100644
new mode 100755
index d804ef2312..e232cb1942
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -71,10 +71,10 @@ conda config --set ssl_verify False
 # cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
 if [ "$BUILD_LIBRAFT" == '1' ]; then
   BUILD_PYRAFT=1
-  # If we are doing CUDA + Python builds, libcugraph package is located at ${CONDA_BLD_DIR}
+  # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR}
   CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
 else
-  # If we are doing Python builds only, libcugraph package is placed here by Project Flash
+  # If we are doing Python builds only, libraft package is placed here by Project Flash
   CONDA_LOCAL_CHANNEL="ci/artifacts/raft/cpu/.conda-bld/"
 fi
 
@@ -100,7 +100,7 @@ else
   gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
 fi
 
-if [ "$BUILD_CUGRAPH" == "1" ]; then
+if [ "$BUILD_raft" == "1" ]; then
   gpuci_logger "Building conda packages for pyraft"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh
old mode 100644
new mode 100755
diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
old mode 100644
new mode 100755

From 2c59f69454397047b0036d0d1d779a0c3874d97b Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 16:02:49 -0500
Subject: [PATCH 04/40] Fixing libraft_headers

---
 ci/cpu/upload.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index c17ec57b0e..83d1686c0c 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -32,7 +32,7 @@ gpuci_logger "Get conda file output locations"
 
 export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
 export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
-export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --python=$PYTHON --output`
+export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
 export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
 
 ################################################################################

From c52637996fececbf4d193203b788b1058d272560 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 18:41:23 -0500
Subject: [PATCH 05/40] Fixing per review feedback

---
 ci/cpu/build.sh    | 4 ++--
 ci/cpu/prebuild.sh | 2 +-
 ci/cpu/upload.sh   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index e232cb1942..bb8eec77d6 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -70,7 +70,7 @@ conda config --set ssl_verify False
 # install. This should eliminate a mismatch between different CUDA versions on
 # cpu vs. gpu builds that is problematic with CUDA 11.5 Enhanced Compat.
 if [ "$BUILD_LIBRAFT" == '1' ]; then
-  BUILD_PYRAFT=1
+  BUILD_RAFT=1
   # If we are doing CUDA + Python builds, libraft package is located at ${CONDA_BLD_DIR}
   CONDA_LOCAL_CHANNEL="${CONDA_BLD_DIR}"
 else
@@ -100,7 +100,7 @@ else
   gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"
 fi
 
-if [ "$BUILD_raft" == "1" ]; then
+if [ "$BUILD_RAFT" == "1" ]; then
   gpuci_logger "Building conda packages for pyraft"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh
index e37bcedb52..0364394423 100755
--- a/ci/cpu/prebuild.sh
+++ b/ci/cpu/prebuild.sh
@@ -14,7 +14,7 @@
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     #If project flash is not activate, always build both
-    export BUILD_PYRAFT=1
+    export BUILD_RAFT=1
     export BUILD_LIBRAFT=1
 fi
 
diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index 83d1686c0c..ad935ad28a 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -61,7 +61,7 @@ if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
 fi
 
-if [[ "$BUILD_PYRAFT" == "1" ]]; then
+if [[ "$BUILD_RAFT" == "1" ]]; then
   test -e ${PYRAFT_FILE}
   echo "Upload pyraft"
   echo ${PYRAFT_FILE}

From d940faca8924ce3f15b074ef0d400d05353ecae8 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 18:54:07 -0500
Subject: [PATCH 06/40] Add fatbin

---
 cpp/CMakeLists.txt | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 8cd97ec93f..e2be3611b6 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -142,6 +142,16 @@ target_link_libraries(raft INTERFACE
 target_compile_definitions(raft INTERFACE $<$<BOOL:${NVTX}>:NVTX_ENABLED>)
 target_compile_features(raft INTERFACE cxx_std_17 $<BUILD_INTERFACE:cuda_std_17>)
 
+if(RAFT_COMPILE_LIBRARIES)
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld"
+[=[
+SECTIONS
+{
+.nvFatBinSegment : { *(.nvFatBinSegment) }
+.nv_fatbin : { *(.nv_fatbin) }
+}
+]=])
+endif()
 ##############################################################################
 # - raft_distance ------------------------------------------------------------
 add_library(raft_distance INTERFACE)
@@ -180,6 +190,9 @@ if(RAFT_COMPILE_LIBRARIES)
   target_compile_definitions(raft_distance_lib
           INTERFACE "RAFT_DISTANCE_COMPILED")
 
+  # ensure CUDA symbols aren't relocated to the middle of the debug build binaries
+  target_link_options(raft_distance_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld")
+
 endif()
 
 target_link_libraries(raft_distance INTERFACE raft::raft
@@ -214,6 +227,9 @@ if(RAFT_COMPILE_LIBRARIES)
           PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:${RAFT_CXX_FLAGS}>"
           "$<$<COMPILE_LANGUAGE:CUDA>:${RAFT_CUDA_FLAGS}>"
           )
+  # ensure CUDA symbols aren't relocated to the middle of the debug build binaries
+  target_link_options(raft_nn_lib PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld")
+
   target_compile_definitions(raft_nn_lib
           INTERFACE "RAFT_NN_COMPILED")
 

From 3cbf3b10bf1382deb386b38d595cecec8e1c7604 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 19:04:16 -0500
Subject: [PATCH 07/40] fixing ci conda recipe paths

---
 ci/cpu/build.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 85319642ea..e304b6542f 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -86,13 +86,13 @@ fi
 if [ "$BUILD_LIBRAFT" == '1' ]; then
   gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-nn
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-distance
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft-headers
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
   else
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-nn
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-distance
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft-headers
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
     mkdir -p ${CONDA_BLD_DIR}/libraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi

From 133ea650681e6e145e7dab66077e555eec3df806 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 19:13:46 -0500
Subject: [PATCH 08/40] Fixing typo in conda yaml files

---
 conda/recipes/libraft_distance/meta.yaml | 2 +-
 conda/recipes/pyraft/meta.yaml           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml
index 8c3a381a1f..43a239f64e 100644
--- a/conda/recipes/libraft_distance/meta.yaml
+++ b/conda/recipes/libraft_distance/meta.yaml
@@ -36,7 +36,7 @@ requirements:
   build:
     - cmake>=3.20.1
   host:
-    - libraft-headers {{
+    - libraft-headers {{ version }}
     - nccl>=2.9.9
     - cudatoolkit {{ cuda_version }}.*
     - ucx-py {{ ucx_py_version }}
diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml
index b1588d57c0..9103630a1e 100644
--- a/conda/recipes/pyraft/meta.yaml
+++ b/conda/recipes/pyraft/meta.yaml
@@ -29,14 +29,14 @@ requirements:
     - python x.x
     - setuptools
     - cython>=0.29,<0.30
-    - libraft-headers={{ version }}
+    - libraft-headers {{ version }}
     - cudatoolkit {{ cuda_version }}.*
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
   run:
     - python x.x
     - dask-cuda {{ minor_version }}
-    - libraft-headers={{ version }}
+    - libraft-headers {{ version }}
     - cupy>=7.8.0,<10.0.0a0
     - nccl>=2.9.9
     - ucx-py {{ ucx_py_version }}

From 3ea2f05bcf3cf985a651614730d342ed946f4fe6 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 19:14:36 -0500
Subject: [PATCH 09/40] Reordering conda build calls by dependencie

---
 ci/cpu/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index e304b6542f..8d0f3328e6 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -86,13 +86,13 @@ fi
 if [ "$BUILD_LIBRAFT" == '1' ]; then
   gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
   else
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
     mkdir -p ${CONDA_BLD_DIR}/libraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi

From 451f8a8ffc23f881612f3fa8def7c5e5c6fe48c9 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 19:33:06 -0500
Subject: [PATCH 10/40] Installing headers

---
 build.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/build.sh b/build.sh
index a636d100b4..da03641d8a 100755
--- a/build.sh
+++ b/build.sh
@@ -60,7 +60,7 @@ SINGLEGPU=""
 NVTX=OFF
 CLEAN=0
 DISABLE_DEPRECATION_WARNINGS=ON
-CMAKE_TARGET=""
+CMAKE_TARGET="install;"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
@@ -180,16 +180,16 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then
       # Run all c++ targets at once
-      if hasArg --compile-nn || hasArg --compile-dist || hasArg --compile-libs; then
+#      if hasArg --compile-nn || hasArg --compile-dist || hasArg --compile-libs; then
         if ! hasArg --nogtest; then
           CMAKE_TARGET="test_raft;${CMAKE_TARGET}"
         fi
 
         echo "-- Compiling targets: ${CMAKE_TARGET}"
         cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} --target ${CMAKE_TARGET}
-      else
-        cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG}
-      fi
+#      else
+#        cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG}
+#      fi
   fi
 
 fi

From da2ec4e10e84b07dafab36fcd8e33322d5443903 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:02:42 -0500
Subject: [PATCH 11/40] Trying mamba

---
 ci/cpu/build.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 8d0f3328e6..24950e2074 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -86,13 +86,13 @@ fi
 if [ "$BUILD_LIBRAFT" == '1' ]; then
   gpuci_logger "Building conda packages for libraft-nn, libraft-distance, and libraft-headers"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
   else
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
     mkdir -p ${CONDA_BLD_DIR}/libraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi
@@ -103,9 +103,9 @@ fi
 if [ "$BUILD_RAFT" == "1" ]; then
   gpuci_logger "Building conda packages for pyraft"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON
   else
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pyraft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
     mkdir -p ${CONDA_BLD_DIR}/pyraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/pyraft/work
   fi

From 5556e098c877dd5ac4d40de666fbe75075470b8b Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:11:39 -0500
Subject: [PATCH 12/40] trying mambabuild again

---
 ci/cpu/build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 24950e2074..2fed6d9e85 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -78,6 +78,7 @@ else
   CONDA_LOCAL_CHANNEL="ci/artifacts/raft/cpu/.conda-bld/"
 fi
 
+gpuci_mamba_retry install -c conda-forge boa
 
 ###############################################################################
 # BUILD - Conda package builds

From 390c1711223d7857fcc268e5c557faef6115eb4c Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:36:45 -0500
Subject: [PATCH 13/40] Updates

---
 build.sh         | 20 +++++++-------------
 ci/cpu/upload.sh |  8 ++++----
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/build.sh b/build.sh
index da03641d8a..df3d5fc0b9 100755
--- a/build.sh
+++ b/build.sh
@@ -175,23 +175,17 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
           -DNVTX=${NVTX} \
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
-          -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} \
-          ..
+          -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS}
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then
       # Run all c++ targets at once
-#      if hasArg --compile-nn || hasArg --compile-dist || hasArg --compile-libs; then
-        if ! hasArg --nogtest; then
-          CMAKE_TARGET="test_raft;${CMAKE_TARGET}"
-        fi
-
-        echo "-- Compiling targets: ${CMAKE_TARGET}"
-        cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} --target ${CMAKE_TARGET}
-#      else
-#        cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG}
-#      fi
-  fi
+      if ! hasArg --nogtest; then
+        CMAKE_TARGET="test_raft;${CMAKE_TARGET}"
+      fi
 
+      echo "-- Compiling targets: ${CMAKE_TARGET}"
+      cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET} ${VERBOSE_FLAG}
+  fi
 fi
 
 # Build and (optionally) install the cuml Python package
diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index ad935ad28a..9aa61bd1e2 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -30,10 +30,10 @@ fi
 
 gpuci_logger "Get conda file output locations"
 
-export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
-export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
-export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
-export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
+export LIBRAFT_NN_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
+export LIBRAFT_DISTANCE_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
+export LIBRAFT_HEADERS_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
+export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
 
 ################################################################################
 # UPLOAD - Conda packages

From 2ad9f41af3205e620b79bc358450b8a05ff13630 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:37:42 -0500
Subject: [PATCH 14/40] nprc

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index df3d5fc0b9..ca536f60f8 100755
--- a/build.sh
+++ b/build.sh
@@ -66,7 +66,7 @@ CMAKE_TARGET="install;"
 #  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
 #         CONDA_PREFIX, but there is no fallback from there!
 INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}}
-PARALLEL_LEVEL=${PARALLEL_LEVEL:=""}
+PARALLEL_LEVEL=${PARALLEL_LEVEL:=`nproc`}
 BUILD_ABI=${BUILD_ABI:=ON}
 
 # Default to Ninja if generator is not specified

From bf1eb26bd20a621469168f2c54f2520ed6de77c5 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:53:52 -0500
Subject: [PATCH 15/40] remove -x

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index ca536f60f8..04b3a79ab2 100755
--- a/build.sh
+++ b/build.sh
@@ -95,7 +95,7 @@ fi
 if hasArg -v; then
     VERBOSE_FLAG=-v
     CMAKE_LOG_LEVEL="--log-level=VERBOSE"
-    set -x
+#    set -x
 fi
 if hasArg -g; then
     BUILD_TYPE=Debug

From 6e7a4d02b2bded9d1e09db3f47332e036dabde70 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 20:56:31 -0500
Subject: [PATCH 16/40] verbose

---
 build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index 04b3a79ab2..1926b5aec5 100755
--- a/build.sh
+++ b/build.sh
@@ -93,7 +93,7 @@ fi
 
 # Process flags
 if hasArg -v; then
-    VERBOSE_FLAG=-v
+    VERBOSE_FLAG="-v"
     CMAKE_LOG_LEVEL="--log-level=VERBOSE"
 #    set -x
 fi
@@ -184,7 +184,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
       fi
 
       echo "-- Compiling targets: ${CMAKE_TARGET}"
-      cmake --build  ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET} ${VERBOSE_FLAG}
+      cmake --build  "${CPP_RAFT_BUILD_DIR}" -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET} ${VERBOSE_FLAG}
   fi
 fi
 

From 49f44ab4d9ac9f944fe5c24a57fa7ae293762df2 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 21:28:50 -0500
Subject: [PATCH 17/40] Moving verbose flag

---
 build.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index 1926b5aec5..1bcd09a895 100755
--- a/build.sh
+++ b/build.sh
@@ -95,7 +95,6 @@ fi
 if hasArg -v; then
     VERBOSE_FLAG="-v"
     CMAKE_LOG_LEVEL="--log-level=VERBOSE"
-#    set -x
 fi
 if hasArg -g; then
     BUILD_TYPE=Debug
@@ -184,7 +183,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
       fi
 
       echo "-- Compiling targets: ${CMAKE_TARGET}"
-      cmake --build  "${CPP_RAFT_BUILD_DIR}" -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET} ${VERBOSE_FLAG}
+      cmake --build  "${CPP_RAFT_BUILD_DIR}" ${VERBOSE_FLAG} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET}
   fi
 fi
 

From 386ca8d7a1b5334bd070cd77ffb1a4b71f95186d Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 21:29:12 -0500
Subject: [PATCH 18/40] printing verbosityi

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index 1bcd09a895..c9b8af61af 100755
--- a/build.sh
+++ b/build.sh
@@ -182,7 +182,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
         CMAKE_TARGET="test_raft;${CMAKE_TARGET}"
       fi
 
-      echo "-- Compiling targets: ${CMAKE_TARGET}"
+      echo "-- Compiling targets: ${CMAKE_TARGET}, verbose=${VERBOSE_FLAG}"
       cmake --build  "${CPP_RAFT_BUILD_DIR}" ${VERBOSE_FLAG} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET}
   fi
 fi

From a9e9c5a322631e98403436a7f1b150a10a9dc77b Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 21:36:59 -0500
Subject: [PATCH 19/40] using cmake message log level

---
 build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build.sh b/build.sh
index c9b8af61af..e5128c90df 100755
--- a/build.sh
+++ b/build.sh
@@ -174,6 +174,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
           -DNVTX=${NVTX} \
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
+          -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \
           -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS}
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then

From acdf7aaa8ae34244f29d2fe3b3d634e7db86aa7e Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Wed, 2 Feb 2022 22:02:46 -0500
Subject: [PATCH 20/40] Trying verbose again

---
 build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index e5128c90df..fca83f5112 100755
--- a/build.sh
+++ b/build.sh
@@ -94,7 +94,7 @@ fi
 # Process flags
 if hasArg -v; then
     VERBOSE_FLAG="-v"
-    CMAKE_LOG_LEVEL="--log-level=VERBOSE"
+    CMAKE_LOG_LEVEL="VERBOSE"
 fi
 if hasArg -g; then
     BUILD_TYPE=Debug
@@ -166,12 +166,13 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
         echo "Building for *ALL* supported GPU architectures..."
     fi
 
-    cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} ${CMAKE_LOG_LEVEL} \
+    cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \
           -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
           -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \
           -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \
           -DRAFT_ENABLE_NN_DEPENDENCIES=${ENABLE_NN_DEPENDENCIES} \
           -DNVTX=${NVTX} \
+          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
           -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \

From dcc3338d7369330c99862425a33f8d8208cde997 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 05:15:46 -0500
Subject: [PATCH 21/40] adding cuda-python to build

---
 conda/recipes/pyraft/meta.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml
index 9103630a1e..888c9b211e 100644
--- a/conda/recipes/pyraft/meta.yaml
+++ b/conda/recipes/pyraft/meta.yaml
@@ -31,6 +31,7 @@ requirements:
     - cython>=0.29,<0.30
     - libraft-headers {{ version }}
     - cudatoolkit {{ cuda_version }}.*
+    - cuda-python >=11.5,<12.0
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
   run:

From 936aeca1b28f9e6977264f64fd85eaa390d6a0dc Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 09:05:10 -0500
Subject: [PATCH 22/40] Only installing when not compiling any libs

---
 build.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index fca83f5112..8602496110 100755
--- a/build.sh
+++ b/build.sh
@@ -60,7 +60,7 @@ SINGLEGPU=""
 NVTX=OFF
 CLEAN=0
 DISABLE_DEPRECATION_WARNINGS=ON
-CMAKE_TARGET="install;"
+CMAKE_TARGET=""
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
@@ -119,6 +119,10 @@ if hasArg --compile-dist || hasArg --compile-libs; then
     CMAKE_TARGET="raft_distance_lib;${CMAKE_TARGET}"
 fi
 
+if ! hasArg --compile-dist && ! hasArg --compile-nn && ! hasArg --compile-libs; then
+    CMAKE_TARGET="install;"
+fi
+
 if hasArg --buildfaiss; then
     BUILD_STATIC_FAISS=ON
 fi

From 2a2fe1ba73b2ce50f4317eba743061286c5a26ba Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 15:11:37 -0500
Subject: [PATCH 23/40] Adding rmm to pyraft conda dependencies

---
 build.sh                       | 1 -
 conda/recipes/pyraft/meta.yaml | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index 8602496110..dd012dbf15 100755
--- a/build.sh
+++ b/build.sh
@@ -179,7 +179,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
           -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
-          -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \
           -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS}
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then
diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml
index 888c9b211e..89344788fc 100644
--- a/conda/recipes/pyraft/meta.yaml
+++ b/conda/recipes/pyraft/meta.yaml
@@ -29,6 +29,7 @@ requirements:
     - python x.x
     - setuptools
     - cython>=0.29,<0.30
+    - rmm = {{ minor_version }}
     - libraft-headers {{ version }}
     - cudatoolkit {{ cuda_version }}.*
     - cuda-python >=11.5,<12.0
@@ -40,6 +41,7 @@ requirements:
     - libraft-headers {{ version }}
     - cupy>=7.8.0,<10.0.0a0
     - nccl>=2.9.9
+    - rmm = {{ minor_version }}
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
     - dask>=2021.11.1,<=2021.11.2

From d87e34944a1f7d28c9c3c11b1018be41c66e3338 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 15:27:24 -0500
Subject: [PATCH 24/40] Removing the wrong logging

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index dd012dbf15..9af3a7694e 100755
--- a/build.sh
+++ b/build.sh
@@ -176,9 +176,9 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
           -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \
           -DRAFT_ENABLE_NN_DEPENDENCIES=${ENABLE_NN_DEPENDENCIES} \
           -DNVTX=${NVTX} \
-          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
+          -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \
           -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS}
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then

From c946926fabe56fecc83c24ba4e3b0d6bfdbac0da Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 17:31:43 -0500
Subject: [PATCH 25/40] Breaking out jensenshannon specializations

---
 conda/recipes/pyraft/meta.yaml                |  4 +-
 cpp/CMakeLists.txt                            |  4 +-
 ...jensen_shannon_double_double_double_int.cu | 37 ++++++++++++++++++
 ...> jensen_shannon_float_float_float_int.cu} | 14 -------
 ...jensen_shannon_float_float_float_uint32.cu | 39 +++++++++++++++++++
 5 files changed, 81 insertions(+), 17 deletions(-)
 create mode 100644 cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
 rename cpp/src/distance/specializations/detail/{jensen_shannon.cu => jensen_shannon_float_float_float_int.cu} (80%)
 create mode 100644 cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu

diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml
index 89344788fc..4182137f85 100644
--- a/conda/recipes/pyraft/meta.yaml
+++ b/conda/recipes/pyraft/meta.yaml
@@ -29,7 +29,7 @@ requirements:
     - python x.x
     - setuptools
     - cython>=0.29,<0.30
-    - rmm = {{ minor_version }}
+    - rmm {{ minor_version }}
     - libraft-headers {{ version }}
     - cudatoolkit {{ cuda_version }}.*
     - cuda-python >=11.5,<12.0
@@ -41,7 +41,7 @@ requirements:
     - libraft-headers {{ version }}
     - cupy>=7.8.0,<10.0.0a0
     - nccl>=2.9.9
-    - rmm = {{ minor_version }}
+    - rmm {{ minor_version }}
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
     - dask>=2021.11.1,<=2021.11.2
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index e2be3611b6..06724694e4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -171,7 +171,9 @@ if(RAFT_COMPILE_LIBRARIES)
     src/distance/specializations/detail/cosine.cu
     src/distance/specializations/detail/hamming_unexpanded.cu
     src/distance/specializations/detail/hellinger_expanded.cu
-    src/distance/specializations/detail/jensen_shannon.cu
+    src/distance/specializations/detail/jensen_shannon_float_float_float_int.cu
+    src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
+    src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
     src/distance/specializations/detail/kl_divergence.cu
     src/distance/specializations/detail/l1.cu
     src/distance/specializations/detail/l2_expanded.cu
diff --git a/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu b/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
new file mode 100644
index 0000000000..ed93217a43
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+    template void distance<raft::distance::DistanceType::JensenShannon, double, double, double, int>(
+            const double* x,
+            const double* y,
+            double* dist,
+            int m,
+            int n,
+            int k,
+            void* workspace,
+            std::size_t worksize,
+            cudaStream_t stream,
+            bool isRowMajor,
+            double metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/jensen_shannon.cu b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_int.cu
similarity index 80%
rename from cpp/src/distance/specializations/detail/jensen_shannon.cu
rename to cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_int.cu
index 37f1f81fb1..c737e1645e 100644
--- a/cpp/src/distance/specializations/detail/jensen_shannon.cu
+++ b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_int.cu
@@ -45,20 +45,6 @@ template void distance<raft::distance::DistanceType::JensenShannon, double, doub
   bool isRowMajor,
   double metric_arg);
 
-template void
-distance<raft::distance::DistanceType::JensenShannon, float, float, float, std::uint32_t>(
-  const float* x,
-  const float* y,
-  float* dist,
-  std::uint32_t m,
-  std::uint32_t n,
-  std::uint32_t k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
 }  // namespace detail
 }  // namespace distance
 }  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
new file mode 100644
index 0000000000..ea44350d87
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+
+    template void
+    distance<raft::distance::DistanceType::JensenShannon, float, float, float, std::uint32_t>(
+            const float* x,
+            const float* y,
+            float* dist,
+            std::uint32_t m,
+            std::uint32_t n,
+            std::uint32_t k,
+            void* workspace,
+            std::size_t worksize,
+            cudaStream_t stream,
+            bool isRowMajor,
+            float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft

From 6ab5d4eb6d00d2f4348f35c6aaa5ca7b3263e446 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 17:32:51 -0500
Subject: [PATCH 26/40] Fixing style

---
 ...jensen_shannon_double_double_double_int.cu | 24 ++++++++---------
 ...jensen_shannon_float_float_float_uint32.cu | 26 +++++++++----------
 2 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu b/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
index ed93217a43..615af0554c 100644
--- a/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
+++ b/cpp/src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
@@ -19,18 +19,18 @@
 namespace raft {
 namespace distance {
 namespace detail {
-    template void distance<raft::distance::DistanceType::JensenShannon, double, double, double, int>(
-            const double* x,
-            const double* y,
-            double* dist,
-            int m,
-            int n,
-            int k,
-            void* workspace,
-            std::size_t worksize,
-            cudaStream_t stream,
-            bool isRowMajor,
-            double metric_arg);
+template void distance<raft::distance::DistanceType::JensenShannon, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
 
 }  // namespace detail
 }  // namespace distance
diff --git a/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
index ea44350d87..91e5f4b3d1 100644
--- a/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
+++ b/cpp/src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
@@ -20,19 +20,19 @@ namespace raft {
 namespace distance {
 namespace detail {
 
-    template void
-    distance<raft::distance::DistanceType::JensenShannon, float, float, float, std::uint32_t>(
-            const float* x,
-            const float* y,
-            float* dist,
-            std::uint32_t m,
-            std::uint32_t n,
-            std::uint32_t k,
-            void* workspace,
-            std::size_t worksize,
-            cudaStream_t stream,
-            bool isRowMajor,
-            float metric_arg);
+template void
+distance<raft::distance::DistanceType::JensenShannon, float, float, float, std::uint32_t>(
+  const float* x,
+  const float* y,
+  float* dist,
+  std::uint32_t m,
+  std::uint32_t n,
+  std::uint32_t k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
 
 }  // namespace detail
 }  // namespace distance

From 375024cff2907f83f92fff66e7dc788d16c1f9d7 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 20:07:46 -0500
Subject: [PATCH 27/40] No clean in conda recipes

---
 conda/recipes/libraft_distance/build.sh | 2 +-
 conda/recipes/libraft_headers/build.sh  | 2 +-
 conda/recipes/libraft_nn/build.sh       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft_distance/build.sh
index 72b43b7cf0..7523263f01 100644
--- a/conda/recipes/libraft_distance/build.sh
+++ b/conda/recipes/libraft_distance/build.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-./build.sh clean libraft -v --allgpuarch --compile-dist --nogtest
+./build.sh libraft -v --allgpuarch --compile-dist --nogtest
diff --git a/conda/recipes/libraft_headers/build.sh b/conda/recipes/libraft_headers/build.sh
index eec262bb1e..ca6d9b4960 100644
--- a/conda/recipes/libraft_headers/build.sh
+++ b/conda/recipes/libraft_headers/build.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-./build.sh clean libraft -v --allgpuarch --nogtest
+./build.sh libraft -v --allgpuarch --nogtest
diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft_nn/build.sh
index 30dfdbe04f..5c60cd2fa1 100644
--- a/conda/recipes/libraft_nn/build.sh
+++ b/conda/recipes/libraft_nn/build.sh
@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-./build.sh clean libraft -v --allgpuarch --compile-nn --nogtest
+./build.sh libraft -v --allgpuarch --compile-nn --nogtest

From f294c8f4bfff5ef7fd2bf5c6549a19439a30e287 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 20:11:42 -0500
Subject: [PATCH 28/40] Adding dsome logging

---
 build.sh        | 2 ++
 ci/cpu/build.sh | 6 +++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index 9af3a7694e..04f71ee33b 100755
--- a/build.sh
+++ b/build.sh
@@ -170,6 +170,8 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
         echo "Building for *ALL* supported GPU architectures..."
     fi
 
+    mkdir -p ${CPP_RAFT_BUILD_DIR}
+    cd ${CPP_RAFT_BUILD_DIR}
     cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \
           -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
           -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \
diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 2fed6d9e85..05f58a781b 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -92,8 +92,12 @@ if [ "$BUILD_LIBRAFT" == '1' ]; then
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
   else
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
+    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/
+    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
+
     mkdir -p ${CONDA_BLD_DIR}/libraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
   fi

From 8a7368e0d0412629cfe4870d88d53fa2a180fdd2 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 20:27:09 -0500
Subject: [PATCH 29/40] Updating logging

---
 ci/cpu/build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 05f58a781b..8f00d92259 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -92,11 +92,11 @@ if [ "$BUILD_LIBRAFT" == '1' ]; then
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
   else
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
-    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
+    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/
-    gpuci_logger "`ls -la ${CONDA_BUILD_DIR/work}`"
+    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
 
     mkdir -p ${CONDA_BLD_DIR}/libraft
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work

From 147ddf49f34b8f30366ec16eb3a3e9bf18f4d15c Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Thu, 3 Feb 2022 22:28:05 -0500
Subject: [PATCH 30/40] Always installing cmake. Separating out nn from dist
 compile libs

---
 build.sh           | 26 ++++++++++++++++++--------
 cpp/CMakeLists.txt |  6 ++++--
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/build.sh b/build.sh
index 04f71ee33b..6dd52e30a2 100755
--- a/build.sh
+++ b/build.sh
@@ -18,7 +18,7 @@ ARGS=$*
 # script, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libraft pyraft docs -v -g --compile-libs --compile-nn --compile-dist --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss"
+VALIDARGS="clean libraft pyraft docs -v -g --noinstall --compile-libs --compile-nn --compile-dist --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
@@ -36,6 +36,7 @@ HELP="$0 [<target> ...] [<flag> ...]
    --allgpuarch     - build for all supported GPU architectures
    --buildfaiss     - build faiss statically into raft
    --nogtest        - do not build google tests for libraft
+   --noinstall     - do not install cmake targets
    --nvtx           - Enable nvtx for profiling support
    --show_depr_warn - show cmake deprecation warnings
    -h               - print this text
@@ -55,12 +56,15 @@ BUILD_ALL_GPU_ARCH=0
 BUILD_TESTS=YES
 BUILD_STATIC_FAISS=OFF
 COMPILE_LIBRARIES=${BUILD_TESTS}
+COMPILE_NN_LIBRARY=OFF
+COMPILE_DIST_LIBRARY=OFF
 ENABLE_NN_DEPENDENCIES=${BUILD_TESTS}
 SINGLEGPU=""
 NVTX=OFF
 CLEAN=0
 DISABLE_DEPRECATION_WARNINGS=ON
 CMAKE_TARGET=""
+INSTALL_TARGET="install"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check
@@ -76,6 +80,10 @@ function hasArg {
     (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
+if hasArg --noinstall; then
+    INSTALL_TARGET=""
+fi
+
 if hasArg -h || hasArg --help; then
     echo "${HELP}"
     exit 0
@@ -109,20 +117,20 @@ if hasArg --nogtest; then
     ENABLE_NN_DEPENDENCIES=OFF
 fi
 
+if hasArg --compile-libs; then
+  COMPILE_LIBRARIES=ON
+fi
+
 if hasArg --compile-nn || hasArg --compile-libs; then
     ENABLE_NN_DEPENDENCIES=ON
-    COMPILE_LIBRARIES=ON
+    COMPILE_NN_LIBRARY=ON
     CMAKE_TARGET="raft_nn_lib;${CMAKE_TARGET}"
 fi
 if hasArg --compile-dist || hasArg --compile-libs; then
-    COMPILE_LIBRARIES=ON
+    COMPILE_DIST_LIBRARY=ON
     CMAKE_TARGET="raft_distance_lib;${CMAKE_TARGET}"
 fi
 
-if ! hasArg --compile-dist && ! hasArg --compile-nn && ! hasArg --compile-libs; then
-    CMAKE_TARGET="install;"
-fi
-
 if hasArg --buildfaiss; then
     BUILD_STATIC_FAISS=ON
 fi
@@ -181,6 +189,8 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
           -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \
           -DBUILD_TESTS=${BUILD_TESTS} \
           -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \
+          -DRAFT_COMPILE_NN_LIBRARY=${COMPILE_NN_LIBRARY} \
+          -DRAFT_COMPILE_DIST_LIBRARY=${COMPILE_DIST_LIBRARY} \
           -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS}
 
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then
@@ -190,7 +200,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
       fi
 
       echo "-- Compiling targets: ${CMAKE_TARGET}, verbose=${VERBOSE_FLAG}"
-      cmake --build  "${CPP_RAFT_BUILD_DIR}" ${VERBOSE_FLAG} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET}
+      cmake --build  "${CPP_RAFT_BUILD_DIR}" ${VERBOSE_FLAG} -j${PARALLEL_LEVEL} --target ${CMAKE_TARGET} ${INSTALL_TARGET}
   fi
 fi
 
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 06724694e4..4da8e1eb9c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -50,6 +50,8 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF)
 option(NVTX "Enable nvtx markers" OFF)
 
 option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON)
+option(RAFT_COMPILE_NN_LIBRARY "Enable building raft nearest neighbors shared library instantiations" OFF)
+option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" OFF)
 option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" ${RAFT_COMPILE_LIBRARIES})
 include(CMakeDependentOption)
 cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF)
@@ -162,7 +164,7 @@ endif()
 
 set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance)
 
-if(RAFT_COMPILE_LIBRARIES)
+if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_DIST_LIBRARY)
   add_library(raft_distance_lib SHARED
     src/distance/specializations/detail
     src/distance/specializations/detail/canberra.cu
@@ -212,7 +214,7 @@ endif()
 
 set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn)
 
-if(RAFT_COMPILE_LIBRARIES)
+if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_NN_LIBRARY)
   add_library(raft_nn_lib SHARED
     src/nn/specializations/ball_cover.cu
     src/nn/specializations/detail/ball_cover_lowdim.cu

From 199eb5a64ec88f0c122f75ce6c7e3678962831c1 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 06:10:18 -0500
Subject: [PATCH 31/40] fatbin

---
 cpp/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 4da8e1eb9c..f30a288e5c 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -144,7 +144,7 @@ target_link_libraries(raft INTERFACE
 target_compile_definitions(raft INTERFACE $<$<BOOL:${NVTX}>:NVTX_ENABLED>)
 target_compile_features(raft INTERFACE cxx_std_17 $<BUILD_INTERFACE:cuda_std_17>)
 
-if(RAFT_COMPILE_LIBRARIES)
+if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_DIST_LIBRARY OR RAFT_COMPILE_NN_LIBRARY)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld"
 [=[
 SECTIONS

From 499708d8a911b23099c9a14e940e0d8beb7b2028 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 08:39:47 -0500
Subject: [PATCH 32/40] Breaking out separate instantiations for distances to
 speed up compile

---
 cpp/CMakeLists.txt                            | 17 ++++++---
 .../l2_expanded_double_double_double_int.cu   | 36 ++++++++++++++++++
 .../l2_expanded_float_float_float_int.cu      | 37 +++++++++++++++++++
 ...> l2_expanded_float_float_float_uint32.cu} | 26 -------------
 ..._sqrt_expanded_double_double_double_int.cu | 37 +++++++++++++++++++
 .../l2_sqrt_expanded_float_float_float_int.cu | 37 +++++++++++++++++++
 ...sqrt_expanded_float_float_float_uint32.cu} | 25 -------------
 ...qrt_unexpanded_double_double_double_int.cu | 37 +++++++++++++++++++
 ...2_sqrt_unexpanded_float_float_float_int.cu | 37 +++++++++++++++++++
 ...rt_unexpanded_float_float_float_uint32.cu} | 25 -------------
 .../l2_unexpanded_double_double_double_int.cu | 37 +++++++++++++++++++
 .../l2_unexpanded_float_float_float_int.cu    | 37 +++++++++++++++++++
 ...l2_unexpanded_float_float_float_uint32.cu} | 25 -------------
 .../lp_unexpanded_double_double_double_int.cu | 37 +++++++++++++++++++
 .../lp_unexpanded_float_float_float_int.cu    | 37 +++++++++++++++++++
 ...lp_unexpanded_float_float_float_uint32.cu} | 25 -------------
 16 files changed, 381 insertions(+), 131 deletions(-)
 create mode 100644 cpp/src/distance/specializations/detail/l2_expanded_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/l2_expanded_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{l2_expanded.cu => l2_expanded_float_float_float_uint32.cu} (66%)
 create mode 100644 cpp/src/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{l2_sqrt_expanded.cu => l2_sqrt_expanded_float_float_float_uint32.cu} (66%)
 create mode 100644 cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{l2_sqrt_unexpanded.cu => l2_sqrt_unexpanded_float_float_float_uint32.cu} (65%)
 create mode 100644 cpp/src/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{l2_unexpanded.cu => l2_unexpanded_float_float_float_uint32.cu} (66%)
 create mode 100644 cpp/src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{lp_unexpanded.cu => lp_unexpanded_float_float_float_uint32.cu} (66%)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f30a288e5c..ed4d32f7b0 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -178,11 +178,18 @@ if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_DIST_LIBRARY)
     src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
     src/distance/specializations/detail/kl_divergence.cu
     src/distance/specializations/detail/l1.cu
-    src/distance/specializations/detail/l2_expanded.cu
-    src/distance/specializations/detail/l2_sqrt_expanded.cu
-    src/distance/specializations/detail/l2_sqrt_unexpanded.cu
-    src/distance/specializations/detail/l2_unexpanded.cu
-    src/distance/specializations/detail/lp_unexpanded.cu
+    src/distance/specializations/detail/l2_expanded_float_float_float_int.cu
+    src/distance/specializations/detail/l2_expanded_float_float_float_uint32.cu
+    src/distance/specializations/detail/l2_expanded_double_double_double_int.cu
+    src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu
+    src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_uint32.cu
+    src/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu
+    src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu
+    src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_uint32.cu
+    src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu
+    src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu
+    src/distance/specializations/detail/lp_unexpanded_float_float_float_uint32.cu
+    src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu
   )
   set_target_properties(raft_distance_lib PROPERTIES OUTPUT_NAME raft_distance)
 
diff --git a/cpp/src/distance/specializations/detail/l2_expanded_double_double_double_int.cu b/cpp/src/distance/specializations/detail/l2_expanded_double_double_double_int.cu
new file mode 100644
index 0000000000..690fdb304a
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_expanded_double_double_double_int.cu
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2Expanded, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_expanded_float_float_float_int.cu b/cpp/src/distance/specializations/detail/l2_expanded_float_float_float_int.cu
new file mode 100644
index 0000000000..743e885bde
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_expanded_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2Expanded, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_expanded.cu b/cpp/src/distance/specializations/detail/l2_expanded_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/l2_expanded.cu
rename to cpp/src/distance/specializations/detail/l2_expanded_float_float_float_uint32.cu
index 1b122ca331..3e84786db5 100644
--- a/cpp/src/distance/specializations/detail/l2_expanded.cu
+++ b/cpp/src/distance/specializations/detail/l2_expanded_float_float_float_uint32.cu
@@ -19,32 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::L2Expanded, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::L2Expanded, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
-
 template void
 distance<raft::distance::DistanceType::L2Expanded, float, float, float, std::uint32_t>(
   const float* x,
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu
new file mode 100644
index 0000000000..a57d664c7b
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu
new file mode 100644
index 0000000000..836d3b28e4
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_expanded.cu b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/l2_sqrt_expanded.cu
rename to cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_uint32.cu
index f87d08b94b..ff57678a5d 100644
--- a/cpp/src/distance/specializations/detail/l2_sqrt_expanded.cu
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_expanded_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::L2SqrtExpanded, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::L2SqrtExpanded, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void
 distance<raft::distance::DistanceType::L2SqrtExpanded, float, float, float, std::uint32_t>(
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu
new file mode 100644
index 0000000000..b12c70df58
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+
+template void distance<raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu
new file mode 100644
index 0000000000..24d6e6916c
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded.cu b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_uint32.cu
similarity index 65%
rename from cpp/src/distance/specializations/detail/l2_sqrt_unexpanded.cu
rename to cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_uint32.cu
index 7067cc9015..f61c40541c 100644
--- a/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded.cu
+++ b/cpp/src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::L2SqrtUnexpanded, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void
 distance<raft::distance::DistanceType::L2SqrtUnexpanded, float, float, float, std::uint32_t>(
diff --git a/cpp/src/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu b/cpp/src/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu
new file mode 100644
index 0000000000..8c02098c96
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2Unexpanded, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu b/cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu
new file mode 100644
index 0000000000..350cb27874
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L2Unexpanded, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l2_unexpanded.cu b/cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/l2_unexpanded.cu
rename to cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_uint32.cu
index bdd57b13b2..607113a18d 100644
--- a/cpp/src/distance/specializations/detail/l2_unexpanded.cu
+++ b/cpp/src/distance/specializations/detail/l2_unexpanded_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::L2Unexpanded, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::L2Unexpanded, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void
 distance<raft::distance::DistanceType::L2Unexpanded, float, float, float, std::uint32_t>(
diff --git a/cpp/src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu b/cpp/src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu
new file mode 100644
index 0000000000..bd306df055
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+
+template void distance<raft::distance::DistanceType::LpUnexpanded, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu b/cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu
new file mode 100644
index 0000000000..64a0656e27
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::LpUnexpanded, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/lp_unexpanded.cu b/cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/lp_unexpanded.cu
rename to cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_uint32.cu
index a3c15e498d..fcf6f2c65b 100644
--- a/cpp/src/distance/specializations/detail/lp_unexpanded.cu
+++ b/cpp/src/distance/specializations/detail/lp_unexpanded_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::LpUnexpanded, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::LpUnexpanded, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void
 distance<raft::distance::DistanceType::LpUnexpanded, float, float, float, std::uint32_t>(

From 3e301b8864a18d0194a5ae8cadf0efa7e2b527fa Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 09:56:12 -0500
Subject: [PATCH 33/40] Breakig apart more distance source files

---
 cpp/CMakeLists.txt                            |  8 +++-
 .../kl_divergence_double_double_double_int.cu | 37 +++++++++++++++++++
 .../kl_divergence_float_float_float_int.cu    | 37 +++++++++++++++++++
 ...kl_divergence_float_float_float_uint32.cu} | 25 -------------
 .../detail/l1_double_double_double_int.cu     | 37 +++++++++++++++++++
 .../detail/l1_float_float_float_int.cu        | 37 +++++++++++++++++++
 .../{l1.cu => l1_float_float_float_uint32.cu} | 25 -------------
 7 files changed, 154 insertions(+), 52 deletions(-)
 create mode 100644 cpp/src/distance/specializations/detail/kl_divergence_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/kl_divergence_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{kl_divergence.cu => kl_divergence_float_float_float_uint32.cu} (66%)
 create mode 100644 cpp/src/distance/specializations/detail/l1_double_double_double_int.cu
 create mode 100644 cpp/src/distance/specializations/detail/l1_float_float_float_int.cu
 rename cpp/src/distance/specializations/detail/{l1.cu => l1_float_float_float_uint32.cu} (66%)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index ed4d32f7b0..d6c37ffd13 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -176,8 +176,12 @@ if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_DIST_LIBRARY)
     src/distance/specializations/detail/jensen_shannon_float_float_float_int.cu
     src/distance/specializations/detail/jensen_shannon_float_float_float_uint32.cu
     src/distance/specializations/detail/jensen_shannon_double_double_double_int.cu
-    src/distance/specializations/detail/kl_divergence.cu
-    src/distance/specializations/detail/l1.cu
+    src/distance/specializations/detail/kl_divergence_float_float_float_int.cu
+    src/distance/specializations/detail/kl_divergence_float_float_float_uint32.cu
+    src/distance/specializations/detail/kl_divergence_double_double_double_int.cu
+    src/distance/specializations/detail/l1_float_float_float_int.cu
+    src/distance/specializations/detail/l1_float_float_float_uint32.cu
+    src/distance/specializations/detail/l1_double_double_double_int.cu
     src/distance/specializations/detail/l2_expanded_float_float_float_int.cu
     src/distance/specializations/detail/l2_expanded_float_float_float_uint32.cu
     src/distance/specializations/detail/l2_expanded_double_double_double_int.cu
diff --git a/cpp/src/distance/specializations/detail/kl_divergence_double_double_double_int.cu b/cpp/src/distance/specializations/detail/kl_divergence_double_double_double_int.cu
new file mode 100644
index 0000000000..3add90ce4e
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/kl_divergence_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::KLDivergence, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/kl_divergence_float_float_float_int.cu b/cpp/src/distance/specializations/detail/kl_divergence_float_float_float_int.cu
new file mode 100644
index 0000000000..ae81c29aff
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/kl_divergence_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::KLDivergence, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/kl_divergence.cu b/cpp/src/distance/specializations/detail/kl_divergence_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/kl_divergence.cu
rename to cpp/src/distance/specializations/detail/kl_divergence_float_float_float_uint32.cu
index f6412cdefd..d6638004ed 100644
--- a/cpp/src/distance/specializations/detail/kl_divergence.cu
+++ b/cpp/src/distance/specializations/detail/kl_divergence_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::KLDivergence, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::KLDivergence, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void
 distance<raft::distance::DistanceType::KLDivergence, float, float, float, std::uint32_t>(
diff --git a/cpp/src/distance/specializations/detail/l1_double_double_double_int.cu b/cpp/src/distance/specializations/detail/l1_double_double_double_int.cu
new file mode 100644
index 0000000000..49cef9a76f
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l1_double_double_double_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L1, double, double, double, int>(
+  const double* x,
+  const double* y,
+  double* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  double metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l1_float_float_float_int.cu b/cpp/src/distance/specializations/detail/l1_float_float_float_int.cu
new file mode 100644
index 0000000000..afec666d57
--- /dev/null
+++ b/cpp/src/distance/specializations/detail/l1_float_float_float_int.cu
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <raft/distance/detail/distance.cuh>
+
+namespace raft {
+namespace distance {
+namespace detail {
+template void distance<raft::distance::DistanceType::L1, float, float, float, int>(
+  const float* x,
+  const float* y,
+  float* dist,
+  int m,
+  int n,
+  int k,
+  void* workspace,
+  std::size_t worksize,
+  cudaStream_t stream,
+  bool isRowMajor,
+  float metric_arg);
+
+}  // namespace detail
+}  // namespace distance
+}  // namespace raft
diff --git a/cpp/src/distance/specializations/detail/l1.cu b/cpp/src/distance/specializations/detail/l1_float_float_float_uint32.cu
similarity index 66%
rename from cpp/src/distance/specializations/detail/l1.cu
rename to cpp/src/distance/specializations/detail/l1_float_float_float_uint32.cu
index 5df9c9ece6..b12f10a3c3 100644
--- a/cpp/src/distance/specializations/detail/l1.cu
+++ b/cpp/src/distance/specializations/detail/l1_float_float_float_uint32.cu
@@ -19,31 +19,6 @@
 namespace raft {
 namespace distance {
 namespace detail {
-template void distance<raft::distance::DistanceType::L1, float, float, float, int>(
-  const float* x,
-  const float* y,
-  float* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  float metric_arg);
-
-template void distance<raft::distance::DistanceType::L1, double, double, double, int>(
-  const double* x,
-  const double* y,
-  double* dist,
-  int m,
-  int n,
-  int k,
-  void* workspace,
-  std::size_t worksize,
-  cudaStream_t stream,
-  bool isRowMajor,
-  double metric_arg);
 
 template void distance<raft::distance::DistanceType::L1, float, float, float, std::uint32_t>(
   const float* x,

From 773b98580a37cd6482069e8befdcf48e966662b1 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 10:11:31 -0500
Subject: [PATCH 34/40] Making sure files are in correct directoris for s3
 upload

---
 ci/cpu/build.sh | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index 8f00d92259..cf202d8888 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -92,14 +92,19 @@ if [ "$BUILD_LIBRAFT" == '1' ]; then
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance
   else
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_headers
-    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
+    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
+    mkdir -p ${CONDA_BLD_DIR}/libraft_headers/work
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_headers/work
+
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_nn
-    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
-    gpuci_logger "`ls ${CONDA_BUILD_DIR}/work`"
+    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
+    mkdir -p ${CONDA_BLD_DIR}/libraft_nn/work
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_nn/work
 
-    mkdir -p ${CONDA_BLD_DIR}/libraft
-    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft/work
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libraft_distance
+    gpuci_logger "`ls ${CONDA_BLD_DIR}/work`"
+    mkdir -p ${CONDA_BLD_DIR}/libraft_distance/work
+    mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libraft_distance/work
   fi
 else
   gpuci_logger "SKIPPING build of conda packages for libraft-nn, libraft-distance and libraft-headers"

From fce488061a92fcfe71dccc189e9a066bc2a4c1bd Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 12:13:47 -0500
Subject: [PATCH 35/40] Adding missing src files

---
 build.sh           | 2 +-
 cpp/CMakeLists.txt | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index 6dd52e30a2..3c8524f6ce 100755
--- a/build.sh
+++ b/build.sh
@@ -196,7 +196,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then
   if (( ${NUMARGS} == 0 )) || hasArg libraft; then
       # Run all c++ targets at once
       if ! hasArg --nogtest; then
-        CMAKE_TARGET="test_raft;${CMAKE_TARGET}"
+        CMAKE_TARGET="${CMAKE_TARGET};test_raft;"
       fi
 
       echo "-- Compiling targets: ${CMAKE_TARGET}, verbose=${VERBOSE_FLAG}"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index d6c37ffd13..8acd9c0099 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -191,6 +191,9 @@ if(RAFT_COMPILE_LIBRARIES OR RAFT_COMPILE_DIST_LIBRARY)
     src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu
     src/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_uint32.cu
     src/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu
+    src/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu
+    src/distance/specializations/detail/l2_unexpanded_float_float_float_uint32.cu
+    src/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu
     src/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu
     src/distance/specializations/detail/lp_unexpanded_float_float_float_uint32.cu
     src/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu

From 23ea28c8485b4d3f95a69b241fd758ac53385afe Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 14:28:39 -0500
Subject: [PATCH 36/40] Reusing the shared libs in downstream gpu tests

---
 build.sh        | 2 +-
 ci/cpu/build.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index 3c8524f6ce..ca5b97b698 100755
--- a/build.sh
+++ b/build.sh
@@ -55,7 +55,7 @@ VERBOSE_FLAG=""
 BUILD_ALL_GPU_ARCH=0
 BUILD_TESTS=YES
 BUILD_STATIC_FAISS=OFF
-COMPILE_LIBRARIES=${BUILD_TESTS}
+COMPILE_LIBRARIES=OFF
 COMPILE_NN_LIBRARY=OFF
 COMPILE_DIST_LIBRARY=OFF
 ENABLE_NN_DEPENDENCIES=${BUILD_TESTS}
diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
index cf202d8888..64d46a68c7 100755
--- a/ci/cpu/build.sh
+++ b/ci/cpu/build.sh
@@ -12,7 +12,7 @@ if [[ -e /etc/os-release ]] && (grep -qi centos /etc/os-release); then
 else
     export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 fi
-export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-8}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE

From 4db545e67e4de2073b92f7805b6bc15fe870588e Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 14:34:38 -0500
Subject: [PATCH 37/40] Updates

---
 ci/gpu/build.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
index 8b6374c487..be8f74a7d2 100644
--- a/ci/gpu/build.sh
+++ b/ci/gpu/build.sh
@@ -86,10 +86,11 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 
 gpuci_logger "Build C++ and Python targets"
+# These should link against the existing shared libs
 if hasArg --skip-tests; then
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs --nogtest
+  "$WORKSPACE/build.sh" libraft pyraft libraft -v --nogtest
 else
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs
+  "$WORKSPACE/build.sh" libraft pyraft libraft -v
 fi
 
 gpuci_logger "Building docs"

From f434053f68f29898719f2dd2bb2f1bd812c7058b Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 16:56:08 -0500
Subject: [PATCH 38/40] Removing additional libraft in gpu built d

---
 ci/gpu/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
index 13f19d5873..afc6056b42 100644
--- a/ci/gpu/build.sh
+++ b/ci/gpu/build.sh
@@ -96,9 +96,9 @@ export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 gpuci_logger "Build C++ and Python targets"
 # These should link against the existing shared libs
 if hasArg --skip-tests; then
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v --nogtest
+  "$WORKSPACE/build.sh" pyraft libraft -v --nogtest
 else
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v
+  "$WORKSPACE/build.sh" pyraft libraft -v
 fi
 
 gpuci_logger "sccache stats"

From cfd54b1c1c7e907f0fbaeed12cb65a669195b714 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Fri, 4 Feb 2022 19:04:36 -0500
Subject: [PATCH 39/40] Small updates to the conda package upload script

---
 ci/cpu/upload.sh | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index 9aa61bd1e2..a9dafcde33 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -30,9 +30,9 @@ fi
 
 gpuci_logger "Get conda file output locations"
 
+export LIBRAFT_HEADERS_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
 export LIBRAFT_NN_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
 export LIBRAFT_DISTANCE_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
-export LIBRAFT_HEADERS_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
 export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
 
 ################################################################################
@@ -42,6 +42,13 @@ export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyra
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
+
+  # libraft-headers
+  test -e ${LIBRAFT_HEADERS_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_HEADERS_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
+
   # libraft-nn
   test -e ${LIBRAFT_NN_FILE}
   echo "Upload libraft-nn"
@@ -54,11 +61,6 @@ if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
   echo ${LIBRAFT_DISTANCE_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
 
-  # libraft-headers
-  test -e ${LIBRAFT_HEADERS_FILE}
-  echo "Upload libraft-nn"
-  echo ${LIBRAFT_HEADERS_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
 fi
 
 if [[ "$BUILD_RAFT" == "1" ]]; then

From 7d6cb0f417de96389b5f494b6c7d776b56890d68 Mon Sep 17 00:00:00 2001
From: "Corey J. Nolet" <cjnolet@gmail.com>
Date: Mon, 7 Feb 2022 12:50:26 -0500
Subject: [PATCH 40/40] small improvements

---
 ci/cpu/upload.sh | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
index a9dafcde33..fe1d651c31 100755
--- a/ci/cpu/upload.sh
+++ b/ci/cpu/upload.sh
@@ -30,10 +30,10 @@ fi
 
 gpuci_logger "Get conda file output locations"
 
-export LIBRAFT_HEADERS_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
-export LIBRAFT_NN_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
-export LIBRAFT_DISTANCE_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
-export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
+export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
+export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
+export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
+export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
 
 ################################################################################
 # UPLOAD - Conda packages
@@ -43,24 +43,20 @@ gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
 
-  # libraft-headers
   test -e ${LIBRAFT_HEADERS_FILE}
   echo "Upload libraft-nn"
   echo ${LIBRAFT_HEADERS_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
 
-  # libraft-nn
   test -e ${LIBRAFT_NN_FILE}
   echo "Upload libraft-nn"
   echo ${LIBRAFT_NN_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
 
-  # libraft-distance
   test -e ${LIBRAFT_DISTANCE_FILE}
   echo "Upload libraft-distance"
   echo ${LIBRAFT_DISTANCE_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
-
 fi
 
 if [[ "$BUILD_RAFT" == "1" ]]; then