Use conda to build python packages during GPU tests (#4702)

This PR convert the `from sources` build we are doing in GPU test job to a `conda build`. This is done for the following reasons:
- This is required step to improve the Ops CI/CD setup to a more convenient pipeline
- This is required to start using `conda` compilers and `mamba` to build RAPIDS packages
- This prevent us from manually managing and installing the dependencies in GPU job
- This ensure the packages can be installed
- This ensure the tests are running and working against the package content and not the build results. Currently the Python packages are not tested.

This may increase the global pipeline time, but the usage of `mamba` should resolve this as `mamba` is faster than `conda` to build packages

Authors:
  - Jordan Jacobelli (https://github.com/Ethyling)

Approvers:
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Dante Gama Dessavre (https://github.com/dantegd)

URL: #4702

ci/gpu/build.sh

@@ -30,6 +30,7 @@ cd $WORKSPACE
 # Parse git describe
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
@@ -169,6 +170,7 @@ else
     gpuci_logger "Check GPU usage"
     nvidia-smi
 
+    gpuci_logger "Installing libcuml and libcuml-tests"
     gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcuml libcuml-tests
 
     gpuci_logger "Running libcuml test binaries"
@@ -180,20 +182,11 @@ else
         echo "Ran gtest $test_name : return code was: $?, test script exit code is now: $EXITCODE"
     done
 
-    # FIXME: Project FLASH only builds for python version 3.8 which is the one used in
-    # the CUDA 11.0 job, need to change all versions to project flash
-    if [ "$CUDA_REL" == "11.0" ];then
-        gpuci_logger "Using Project FLASH to install cuml python"
-        CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "cuml*.tar.bz2"`
-        CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
-        CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
-        echo "Installing $CONDA_FILE"
-        gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
-
-    else
-        gpuci_logger "Building cuml python in gpu job"
-        "$WORKSPACE/build.sh" -v cuml --codecov
-    fi
+    gpuci_logger "Building and installing cuml"
+    export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+    export VERSION_SUFFIX=""
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} -c ${CONDA_BLD_DIR} cuml
 
     gpuci_logger "Install the main version of dask, distributed, and dask-glm"
     set -x
@@ -206,17 +199,11 @@ else
     set +x
 
     gpuci_logger "Python pytest for cuml"
-    cd $WORKSPACE/python
-
-    # When installing cuml with project flash, we need to delete all folders except
-    # cuml/tests since we are not building cython extensions in place
-    if [ "$PYTHON" == "3.8" ];then
-        find ./cuml -mindepth 1 ! -regex '^./cuml/tests\(/.*\)?' -delete
-    fi
+    cd $WORKSPACE/python/cuml/tests
 
-    pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/tests/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
+    pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=dask --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
 
-    timeout 7200 sh -c "pytest cuml/tests/dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300 --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-dask-coverage.xml --cov-report term"
+    timeout 7200 sh -c "pytest dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300 --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-dask-coverage.xml --cov-report term"
 
     ################################################################################
     # TEST - Run notebook tests

python/cuml/tests/conftest.py

@@ -30,6 +30,71 @@
 from cuml.testing.utils import create_synthetic_dataset
 
 
+# Add the import here for any plugins that should be loaded EVERY TIME
+pytest_plugins = ("cuml.testing.plugins.quick_run_plugin")
+
+
+def pytest_addoption(parser):
+    # Any custom option, that should be available at any time (not just a
+    # plugin), goes here.
+
+    group = parser.getgroup('cuML Custom Options')
+
+    group.addoption(
+        "--run_stress",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'stress'. These are the most intense "
+              "tests that take the longest to run and are designed to stress "
+              "the hardware's compute resources."))
+
+    group.addoption(
+        "--run_quality",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'quality'. These tests are more "
+              "computationally intense than 'unit', but less than 'stress'"))
+
+    group.addoption(
+        "--run_unit",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'unit'. These are the quickest tests "
+              "that are focused on accuracy and correctness."))
+
+
+def pytest_collection_modifyitems(config, items):
+
+    should_run_quality = config.getoption("--run_quality")
+    should_run_stress = config.getoption("--run_stress")
+
+    # Run unit is implied if no --run_XXX is set
+    should_run_unit = config.getoption("--run_unit") or not (
+        should_run_quality or should_run_stress)
+
+    # Mark the tests as "skip" if needed
+    if not should_run_unit:
+        skip_unit = pytest.mark.skip(
+            reason="Stress tests run with --run_unit flag.")
+        for item in items:
+            if "unit" in item.keywords:
+                item.add_marker(skip_unit)
+
+    if not should_run_quality:
+        skip_quality = pytest.mark.skip(
+            reason="Quality tests run with --run_quality flag.")
+        for item in items:
+            if "quality" in item.keywords:
+                item.add_marker(skip_quality)
+
+    if not should_run_stress:
+        skip_stress = pytest.mark.skip(
+            reason="Stress tests run with --run_stress flag.")
+        for item in items:
+            if "stress" in item.keywords:
+                item.add_marker(skip_stress)
+
+
 def pytest_configure(config):
     cp.cuda.set_allocator(None)
     # max_gpu_memory: Capacity of the GPU memory in GB

python/cuml/tests/test_lars.py

@@ -15,6 +15,7 @@
 import cupy as cp
 import numpy as np
 import pytest
+import sys
 
 from cuml.experimental.linear_model import Lars as cuLars
 from cuml.testing.utils import (
@@ -27,7 +28,9 @@
 from sklearn.datasets import load_boston
 from sklearn.linear_model import Lars as skLars
 
-from . test_linear_model import make_regression_dataset
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '.')
+from test_linear_model import make_regression_dataset  # noqa: E402
 
 
 def normalize_data(X, y):

python/cuml/tests/test_multiclass.py

@@ -14,9 +14,14 @@
 #
 import numpy as np
 import pytest
+import sys
+
 from cuml import multiclass as cu_multiclass
 from cuml import LogisticRegression as cuLog
-from . test_linear_model import make_classification_dataset
+
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '.')
+from test_linear_model import make_classification_dataset  # noqa: E402
 
 
 @pytest.mark.parametrize("strategy", ['ovr', 'ovo'])