Reusing shared libs in gpu ci builds (#487)

Authors:
  - Corey J. Nolet (https://github.com/cjnolet)

Approvers:
  - Dante Gama Dessavre (https://github.com/dantegd)
  - Mark Sadang (https://github.com/msadang)
  - AJ Schmidt (https://github.com/ajschmidt8)

URL: #487

build.sh

@@ -55,7 +55,7 @@ VERBOSE_FLAG=""
 BUILD_ALL_GPU_ARCH=0
 BUILD_TESTS=YES
 BUILD_STATIC_FAISS=OFF
-COMPILE_LIBRARIES=${BUILD_TESTS}
+COMPILE_LIBRARIES=OFF
 COMPILE_NN_LIBRARY=OFF
 COMPILE_DIST_LIBRARY=OFF
 ENABLE_NN_DEPENDENCIES=${BUILD_TESTS}

ci/cpu/build.sh

@@ -12,7 +12,7 @@ if [[ -e /etc/os-release ]] && (grep -qi centos /etc/os-release); then
 else
     export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 fi
-export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-8}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE

ci/cpu/upload.sh

@@ -30,10 +30,10 @@ fi
 
 gpuci_logger "Get conda file output locations"
 
-export LIBRAFT_NN_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
-export LIBRAFT_DISTANCE_FILE=`conda mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
-export LIBRAFT_HEADERS_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
-export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
+export LIBRAFT_HEADERS_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/libraft_headers --output`
+export LIBRAFT_NN_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_nn --output`
+export LIBRAFT_DISTANCE_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libraft_distance --output`
+export PYRAFT_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pyraft --python=$PYTHON --output`
 
 ################################################################################
 # UPLOAD - Conda packages
@@ -42,23 +42,21 @@ export PYRAFT_FILE=`conda mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/pyra
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then
-  # libraft-nn
+
+  test -e ${LIBRAFT_HEADERS_FILE}
+  echo "Upload libraft-nn"
+  echo ${LIBRAFT_HEADERS_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
+
   test -e ${LIBRAFT_NN_FILE}
   echo "Upload libraft-nn"
   echo ${LIBRAFT_NN_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_NN_FILE} --no-progress
 
-  # libraft-distance
   test -e ${LIBRAFT_DISTANCE_FILE}
   echo "Upload libraft-distance"
   echo ${LIBRAFT_DISTANCE_FILE}
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_DISTANCE_FILE} --no-progress
-
-  # libraft-headers
-  test -e ${LIBRAFT_HEADERS_FILE}
-  echo "Upload libraft-nn"
-  echo ${LIBRAFT_HEADERS_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBRAFT_HEADERS_FILE} --no-progress
 fi
 
 if [[ "$BUILD_RAFT" == "1" ]]; then

ci/gpu/build.sh

@@ -94,10 +94,11 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 
 gpuci_logger "Build C++ and Python targets"
+# These should link against the existing shared libs
 if hasArg --skip-tests; then
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs --nogtest
+  "$WORKSPACE/build.sh" pyraft libraft -v --nogtest
 else
-  "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs
+  "$WORKSPACE/build.sh" pyraft libraft -v
 fi
 
 gpuci_logger "sccache stats"