Use conda to build python packages during GPU tests (#10648)

This PR convert the `from sources` build we are doing in GPU test job to a `conda build`. This is done for the following reasons:
- This is required step to improve the Ops CI/CD setup to a more convenient pipeline
- This is required to start using `conda` compilers and `mamba` to build RAPIDS packages
- This prevent us from manually managing and installing the dependencies in GPU job
- This ensure the packages can be installed
- This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested.

This may increase the global pipeline time, but the usage of `mamba` should resolve this as `mamba` is faster than `conda` to build packages

Authors:
  - Jordan Jacobelli (https://github.com/Ethyling)

Approvers:
  - AJ Schmidt (https://github.com/ajschmidt8)

URL: #10648

ci/gpu/build.sh

@@ -29,6 +29,7 @@ export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/"
 # Parse git describe
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # Dask & Distributed option to install main(nightly) or `conda-forge` packages.
 export INSTALL_DASK_MAIN=1
@@ -79,30 +80,11 @@ conda info
 conda config --show-sources
 conda list --show-channel-urls
 
-gpuci_logger "Install dependencies"
-gpuci_mamba_retry install -y \
-                  "cudatoolkit=$CUDA_REL" \
-                  "rapids-build-env=$MINOR_VERSION.*" \
-                  "rapids-notebook-env=$MINOR_VERSION.*" \
-                  "dask-cuda=${MINOR_VERSION}" \
-                  "rmm=$MINOR_VERSION.*" \
-                  "ucx-py=${UCX_PY_VERSION}"
-
-# https://docs.rapids.ai/maintainers/depmgmt/
-# gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env
-# gpuci_mamba_retry install -y "your-pkg=1.0.0"
-
-
 gpuci_logger "Check compiler versions"
 python --version
 $CC --version
 $CXX --version
 
-gpuci_logger "Check conda environment"
-conda info
-conda config --show-sources
-conda list --show-channel-urls
-
 function install_dask {
     # Install the conda-forge or nightly version of dask and distributed
     gpuci_logger "Install the conda-forge or nightly version of dask and distributed"
@@ -125,6 +107,19 @@ function install_dask {
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
 
+    gpuci_logger "Install dependencies"
+    gpuci_mamba_retry install -y \
+                  "cudatoolkit=$CUDA_REL" \
+                  "rapids-build-env=$MINOR_VERSION.*" \
+                  "rapids-notebook-env=$MINOR_VERSION.*" \
+                  "dask-cuda=${MINOR_VERSION}" \
+                  "rmm=$MINOR_VERSION.*" \
+                  "ucx-py=${UCX_PY_VERSION}"
+
+    # https://docs.rapids.ai/maintainers/depmgmt/
+    # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env
+    # gpuci_mamba_retry install -y "your-pkg=1.0.0"
+
     install_dask
 
     ################################################################################
@@ -171,8 +166,19 @@ else
     gpuci_logger "Check GPU usage"
     nvidia-smi
 
+    gpuci_logger "Installing libcudf, libcudf_kafka and libcudf-tests"
     gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcudf libcudf_kafka libcudf-tests
 
+    gpuci_logger "Building cudf, dask-cudf, cudf_kafka and custreamz"
+    export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH}
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH}
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON -c ${CONDA_ARTIFACT_PATH}
+    gpuci_conda_retry build --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON -c ${CONDA_ARTIFACT_PATH}
+
+    gpuci_logger "Installing cudf, dask-cudf, cudf_kafka and custreamz"
+    gpuci_mamba_retry install cudf dask-cudf cudf_kafka custreamz -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}"
+
     gpuci_logger "GoogleTests"
     # Run libcudf and libcudf_kafka gtests from libcudf-tests package
     for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do
@@ -209,12 +215,6 @@ else
             # test-results/*.cs.log are processed in gpuci
         fi
     fi
-
-    install_dask
-
-    gpuci_logger "Build python libs from source"
-    "$WORKSPACE/build.sh" cudf dask_cudf cudf_kafka --ptds
-
 fi
 
 # Both regular and Project Flash proceed here