Use custom conda build directory for ccache integration (#3658)

This changes the conda build directory to be a consistent directory when building packages. The resulting work directory from conda must be moved to a repo specific folder for project flash to work.

Authors:
  - Dillon Cullinan (@dillon-cullinan)

Approvers:
  - AJ Schmidt (@ajschmidt8)
  - Dante Gama Dessavre (@dantegd)

URL: #3658

ci/cpu/build.sh

@@ -19,8 +19,9 @@ export CUDA_REL=${CUDA_VERSION%.*}
 export GPUCI_CONDA_RETRY_MAX=1
 export GPUCI_CONDA_RETRY_SLEEP=30
 
-# Use Ninja to build
+# Use Ninja to build, setup Conda Build Dir
 export CMAKE_GENERATOR="Ninja"
+export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
 
 # Switch to project root; also root of repo checkout
 cd $WORKSPACE
@@ -61,22 +62,24 @@ conda config --set ssl_verify False
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
   if [ "$BUILD_LIBCUML" == '1' -o "$BUILD_CUML" == '1' ]; then
     gpuci_logger "Build conda pkg for libcuml"
-    gpuci_conda_retry build conda/recipes/libcuml
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml
   fi
 else
   if [ "$BUILD_LIBCUML" == '1' ]; then
     gpuci_logger "PROJECT FLASH: Build conda pkg for libcuml"
-    gpuci_conda_retry build conda/recipes/libcuml --dirty --no-remove-work-dir
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --dirty --no-remove-work-dir
+    mkdir -p ${CONDA_BLD_DIR}/libcuml/work
+    cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcuml/work
   fi
 fi
 
 if [ "$BUILD_CUML" == '1' ]; then
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_logger "Build conda pkg for cuml"
-    gpuci_conda_retry build conda/recipes/cuml --python=${PYTHON}
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=${PYTHON}
   else
     gpuci_logger "PROJECT FLASH: Build conda pkg for cuml"
-    gpuci_conda_retry build -c ci/artifacts/cuml/cpu/conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} -c ci/artifacts/cuml/cpu/.conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
   fi
 fi
 

ci/cpu/upload.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-#
-# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 
 set -e
 
@@ -29,8 +28,8 @@ fi
 
 gpuci_logger "Get conda file output locations"
 
-export LIBCUML_FILE=`conda build conda/recipes/libcuml --output`
-export CUML_FILE=`conda build conda/recipes/cuml --python=$PYTHON --output`
+export LIBCUML_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcuml --output`
+export CUML_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cuml --python=$PYTHON --output`
 
 ################################################################################
 # UPLOAD - Conda packages

ci/gpu/build.sh

@@ -16,6 +16,7 @@ function hasArg {
 # Set path and build parallel level
 export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
 export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+export CONDA_ARTIFACT_PATH=${WORKSPACE}/ci/artifacts/cuml/cpu/.conda-bld/
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
@@ -108,7 +109,6 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     ################################################################################
     # TEST - Run GoogleTest and py.tests for libcuml and cuML
     ################################################################################
-
     if hasArg --skip-tests; then
         gpuci_logger "Skipping Tests"
         exit 0
@@ -186,11 +186,11 @@ else
     patchelf --replace-needed `patchelf --print-needed ./test/ml | grep faiss` libfaiss.so ./test/ml
     GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./test/ml
 
-    CONDA_FILE=`find $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ -name "libcuml*.tar.bz2"`
+    CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcuml*.tar.bz2"`
     CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
     CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
     gpuci_logger "Installing $CONDA_FILE"
-    conda install -c $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ "$CONDA_FILE"
+    conda install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
 
     gpuci_logger "Building cuml"
     "$WORKSPACE/build.sh" -v cuml --codecov