Use conda to build python packages during GPU tests

ci/gpu/build.sh

@@ -30,6 +30,7 @@ cd $WORKSPACE
 # Parse git describe
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+unset GIT_DESCRIBE_TAG
 
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
@@ -169,6 +170,7 @@ else
     gpuci_logger "Check GPU usage"
     nvidia-smi
 
+    gpuci_logger "Installing libcuml and libcuml-tests"
     gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcuml libcuml-tests
 
     gpuci_logger "Running libcuml test binaries"
@@ -180,20 +182,11 @@ else
         echo "Ran gtest $test_name : return code was: $?, test script exit code is now: $EXITCODE"
     done
 
-    # FIXME: Project FLASH only builds for python version 3.8 which is the one used in
-    # the CUDA 11.0 job, need to change all versions to project flash
-    if [ "$CUDA_REL" == "11.0" ];then
-        gpuci_logger "Using Project FLASH to install cuml python"
-        CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "cuml*.tar.bz2"`
-        CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
-        CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
-        echo "Installing $CONDA_FILE"
-        gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
-
-    else
-        gpuci_logger "Building cuml python in gpu job"
-        "$WORKSPACE/build.sh" -v cuml --codecov
-    fi
+    gpuci_logger "Building and installing cuml"
+    export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
+    export VERSION_SUFFIX=""
+    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} -c ${CONDA_BLD_DIR} cuml
 
     gpuci_logger "Install the main version of dask, distributed, and dask-glm"
     set -x
@@ -206,17 +199,11 @@ else
     set +x
 
     gpuci_logger "Python pytest for cuml"
-    cd $WORKSPACE/python
-
-    # When installing cuml with project flash, we need to delete all folders except
-    # cuml/tests since we are not building cython extensions in place
-    if [ "$PYTHON" == "3.8" ];then
-        find ./cuml -mindepth 1 ! -regex '^./cuml/tests\(/.*\)?' -delete
-    fi
+    cd $WORKSPACE/python/cuml/tests
 
-    pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/tests/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
+    pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=dask --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
 
-    timeout 7200 sh -c "pytest cuml/tests/dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300 --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-dask-coverage.xml --cov-report term"
+    timeout 7200 sh -c "pytest dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300 --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-dask-coverage.xml --cov-report term"
 
     ################################################################################
     # TEST - Run notebook tests

python/cuml/tests/conftest.py

@@ -30,6 +30,71 @@
 from cuml.testing.utils import create_synthetic_dataset
 
 
+# Add the import here for any plugins that should be loaded EVERY TIME
+pytest_plugins = ("cuml.testing.plugins.quick_run_plugin")
+
+
+def pytest_addoption(parser):
+    # Any custom option, that should be available at any time (not just a
+    # plugin), goes here.
+
+    group = parser.getgroup('cuML Custom Options')
+
+    group.addoption(
+        "--run_stress",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'stress'. These are the most intense "
+              "tests that take the longest to run and are designed to stress "
+              "the hardware's compute resources."))
+
+    group.addoption(
+        "--run_quality",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'quality'. These tests are more "
+              "computationally intense than 'unit', but less than 'stress'"))
+
+    group.addoption(
+        "--run_unit",
+        action="store_true",
+        default=False,
+        help=("Runs tests marked with 'unit'. These are the quickest tests "
+              "that are focused on accuracy and correctness."))
+
+
+def pytest_collection_modifyitems(config, items):
+
+    should_run_quality = config.getoption("--run_quality")
+    should_run_stress = config.getoption("--run_stress")
+
+    # Run unit is implied if no --run_XXX is set
+    should_run_unit = config.getoption("--run_unit") or not (
+        should_run_quality or should_run_stress)
+
+    # Mark the tests as "skip" if needed
+    if not should_run_unit:
+        skip_unit = pytest.mark.skip(
+            reason="Stress tests run with --run_unit flag.")
+        for item in items:
+            if "unit" in item.keywords:
+                item.add_marker(skip_unit)
+
+    if not should_run_quality:
+        skip_quality = pytest.mark.skip(
+            reason="Quality tests run with --run_quality flag.")
+        for item in items:
+            if "quality" in item.keywords:
+                item.add_marker(skip_quality)
+
+    if not should_run_stress:
+        skip_stress = pytest.mark.skip(
+            reason="Stress tests run with --run_stress flag.")
+        for item in items:
+            if "stress" in item.keywords:
+                item.add_marker(skip_stress)
+
+
 def pytest_configure(config):
     cp.cuda.set_allocator(None)
     # max_gpu_memory: Capacity of the GPU memory in GB

python/cuml/tests/test_lars.py

@@ -15,6 +15,7 @@
 import cupy as cp
 import numpy as np
 import pytest
+import sys
 
 from cuml.experimental.linear_model import Lars as cuLars
 from cuml.testing.utils import (
@@ -27,7 +28,9 @@
 from sklearn.datasets import load_boston
 from sklearn.linear_model import Lars as skLars
 
-from . test_linear_model import make_regression_dataset
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '.')
+from test_linear_model import make_regression_dataset  # noqa: E402
 
 
 def normalize_data(X, y):

python/cuml/tests/test_multiclass.py

@@ -14,9 +14,14 @@
 #
 import numpy as np
 import pytest
+import sys
+
 from cuml import multiclass as cu_multiclass
 from cuml import LogisticRegression as cuLog
-from . test_linear_model import make_classification_dataset
+
+# As tests directory is not a module, we need to add it to the path
+sys.path.insert(0, '.')
+from test_linear_model import make_classification_dataset  # noqa: E402
 
 
 @pytest.mark.parametrize("strategy", ['ovr', 'ovo'])