Merge pull request #4857 from ajschmidt8/fix-merge-conflicts

Fix forward-merge conflicts

.gitignore

@@ -30,6 +30,8 @@ log
 dask-worker-space/
 tmp/
 .hypothesis
+wheels/
+_skbuild/
 
 ## files pickled in notebook when ran during python docstring generation
 docs/source/*.model

build.sh

@@ -229,7 +229,6 @@ if (( ${CLEAN} == 1 )); then
     cd ${REPODIR}
 fi
 
-# Before
 
 ################################################################################
 # Configure for building all C++ targets
@@ -283,11 +282,16 @@ fi
 
 # Build and (optionally) install the cuml Python package
 if completeBuild || hasArg cuml || hasArg pydocs; then
+    # Append `-DFIND_CUML_CPP=ON` to CUML_EXTRA_CMAKE_ARGS unless a user specified the option.
+    if [[ "${CUML_EXTRA_CMAKE_ARGS}" != *"DFIND_CUML_CPP"* ]]; then
+        CUML_EXTRA_CMAKE_ARGS="${CUML_EXTRA_CMAKE_ARGS} -DFIND_CUML_CPP=ON"
+    fi
+
     cd ${REPODIR}/python
+
+    python setup.py build_ext --inplace -- -DCMAKE_LIBRARY_PATH=${LIBCUML_BUILD_DIR} -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} ${CUML_EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
     if [[ ${INSTALL_TARGET} != "" ]]; then
-        python setup.py build_ext -j${PARALLEL_LEVEL:-1} ${CUML_EXTRA_PYTHON_ARGS} --library-dir=${LIBCUML_BUILD_DIR} install --single-version-externally-managed --record=record.txt
-    else
-        python setup.py build_ext -j${PARALLEL_LEVEL:-1} ${CUML_EXTRA_PYTHON_ARGS} --library-dir=${LIBCUML_BUILD_DIR}
+        python setup.py install --single-version-externally-managed --record=record.txt  -- -DCMAKE_LIBRARY_PATH=${LIBCUML_BUILD_DIR} -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} ${CUML_EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1}
     fi
 
     if hasArg pydocs; then

ci/gpu/build.sh

@@ -124,8 +124,8 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
 
     gpuci_logger "Install the main version of dask and distributed"
     set -x
-    pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
-    pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
+    pip install "git+https://github.com/dask/distributed.git@2022.7.1" --upgrade --no-deps
+    pip install "git+https://github.com/dask/dask.git@2022.7.1" --upgrade --no-deps
     set +x
 
     gpuci_logger "Python pytest for cuml"
@@ -191,14 +191,14 @@ else
     gpuci_logger "Building and installing cuml"
     export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
     export VERSION_SUFFIX=""
-    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
-    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} -c ${CONDA_BLD_DIR} cuml
+    gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+    gpuci_mamba_retry install cuml -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}"
 
     gpuci_logger "Install the main version of dask, distributed, and dask-glm"
     set -x
 
-    pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps
-    pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps
+    pip install "git+https://github.com/dask/distributed.git@2022.7.1" --upgrade --no-deps
+    pip install "git+https://github.com/dask/dask.git@2022.7.1" --upgrade --no-deps
     pip install "git+https://github.com/dask/dask-glm@main" --force-reinstall --no-deps
     pip install sparse
 

ci/release/update-version.sh

@@ -32,7 +32,10 @@ function sed_runner() {
 }
 
 sed_runner 's/'"CUML VERSION .* LANGUAGES"'/'"CUML VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/CMakeLists.txt
-sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' cpp/CMakeLists.txt
+sed_runner 's/'"set(CUML_VERSION .*)"'/'"set(CUML_VERSION ${NEXT_FULL_TAG})"'/g' python/CMakeLists.txt
+# rapids-cmake version
+sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake
+
 
 # RTD update
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py

conda/environments/builddocs_py37.yml

@@ -20,12 +20,11 @@ dependencies:
 # required for building rapids project docs
 - sphinx
 - sphinx_rtd_theme
+- sphinx-markdown-tables
 - sphinxcontrib-websupport
 - nbsphinx
 - numpydoc
 - ipython
 - recommonmark
 - pandoc=<2.0.0
 - pip
-- pip:
-  - sphinx-markdown-tables

conda/environments/cuml_dev_cuda11.0.yml

@@ -10,6 +10,7 @@ dependencies:
 - rapids-build-env=22.10.*
 - rapids-notebook-env=22.10.*
 - rapids-doc-env=22.10.*
+- scikit-build>=0.13.1
 - cudf=22.10.*
 - rmm=22.10.*
 - libcumlprims=22.10.*
@@ -28,16 +29,16 @@ dependencies:
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.24
+- sphinx-markdown-tables
 - treelite=2.4.0
 - statsmodels
 - seaborn
 - hdbscan
 - nltk
 - pip
 - pip:
-    - sphinx_markdown_tables
-    - git+https://github.com/dask/dask.git@main
-    - git+https://github.com/dask/distributed.git@main
+    - git+https://github.com/dask/[email protected]
+    - git+https://github.com/dask/[email protected]
     - git+https://github.com/dask/dask-glm@main
     - sparse
 

conda/environments/cuml_dev_cuda11.2.yml

@@ -10,6 +10,7 @@ dependencies:
 - rapids-build-env=22.10.*
 - rapids-notebook-env=22.10.*
 - rapids-doc-env=22.10.*
+- scikit-build>=0.13.1
 - cudf=22.10.*
 - rmm=22.10.*
 - libcumlprims=22.10.*
@@ -28,16 +29,16 @@ dependencies:
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.24
+- sphinx-markdown-tables
 - treelite=2.4.0
 - statsmodels
 - seaborn
 - hdbscan
 - nltk
 - pip
 - pip:
-    - sphinx_markdown_tables
-    - git+https://github.com/dask/dask.git@main
-    - git+https://github.com/dask/distributed.git@main
+    - git+https://github.com/dask/[email protected]
+    - git+https://github.com/dask/[email protected]
     - git+https://github.com/dask/dask-glm@main
     - sparse
 

conda/environments/cuml_dev_cuda11.4.yml

@@ -10,6 +10,7 @@ dependencies:
 - rapids-build-env=22.10.*
 - rapids-notebook-env=22.10.*
 - rapids-doc-env=22.10.*
+- scikit-build>=0.13.1
 - cudf=22.10.*
 - rmm=22.10.*
 - libcumlprims=22.10.*
@@ -28,16 +29,16 @@ dependencies:
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.24
+- sphinx-markdown-tables
 - treelite=2.4.0
 - statsmodels
 - seaborn
 - hdbscan
 - nltk
 - pip
 - pip:
-    - sphinx_markdown_tables
-    - git+https://github.com/dask/dask.git@main
-    - git+https://github.com/dask/distributed.git@main
+    - git+https://github.com/dask/[email protected]
+    - git+https://github.com/dask/[email protected]
     - git+https://github.com/dask/dask-glm.git@main
     - sparse
 

conda/environments/cuml_dev_cuda11.5.yml

@@ -10,6 +10,7 @@ dependencies:
 - rapids-build-env=22.10.*
 - rapids-notebook-env=22.10.*
 - rapids-doc-env=22.10.*
+- scikit-build>=0.13.1
 - cudf=22.10.*
 - rmm=22.10.*
 - libcumlprims=22.10.*
@@ -28,16 +29,16 @@ dependencies:
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.24
+- sphinx-markdown-tables
 - treelite=2.4.0
 - statsmodels
 - seaborn
 - hdbscan
 - nltk
 - pip
 - pip:
-    - sphinx_markdown_tables
-    - git+https://github.com/dask/dask.git@main
-    - git+https://github.com/dask/distributed.git@main
+    - git+https://github.com/dask/[email protected]
+    - git+https://github.com/dask/[email protected]
     - git+https://github.com/dask/dask-glm@main
     - sparse
 

conda/recipes/cuml/build.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 
 # This assumes the script is executed from the root of the repo directory
-./build.sh cuml
+./build.sh cuml -v

conda/recipes/cuml/conda_build_config.yaml

@@ -7,5 +7,8 @@ cxx_compiler_version:
 cuda_compiler:
   - nvcc
 
+cmake_version:
+  - ">=3.20.1,!=3.23.0"
+
 sysroot_version:
   - "2.17"

conda/recipes/cuml/meta.yaml

@@ -26,14 +26,15 @@ build:
 
 requirements:
   build:
-    - cmake>=3.20.1,!=3.23.0
+    - cmake {{ cmake_version }}
     - {{ compiler('c') }}
     - {{ compiler('cxx') }}
     - {{ compiler('cuda') }} {{ cuda_version }}
     - sysroot_{{ target_platform }} {{ sysroot_version }}
   host:
     - python x.x
     - setuptools
+    - scikit-build>=0.13.1
     - cython>=0.29,<0.30
     - treelite=2.4.0
     - cudf {{ minor_version }}
@@ -56,8 +57,8 @@ requirements:
     - nccl>=2.9.9
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
-    - dask>=2022.05.2
-    - distributed>=2022.05.2
+    - dask==2022.7.1
+    - distributed==2022.7.1
     - joblib >=0.11
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
     - cuda-python >=11.5,<11.7.1

conda/recipes/libcuml/build.sh

@@ -7,4 +7,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then
     export MACOSX_DEPLOYMENT_TARGET=10.11
 fi
 
-./build.sh -n clean libcuml prims -v --allgpuarch
+./build.sh -n libcuml prims -v --allgpuarch

cpp/CMakeLists.txt

@@ -15,9 +15,9 @@
 #=============================================================================
 
 cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR)
-file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.10/RAPIDS.cmake
-    ${CMAKE_BINARY_DIR}/RAPIDS.cmake)
-include(${CMAKE_BINARY_DIR}/RAPIDS.cmake)
+
+include(../fetch_rapids.cmake)
+
 include(rapids-cmake)
 include(rapids-cpm)
 include(rapids-cuda)
@@ -72,36 +72,36 @@ option(CUML_USE_RAFT_STATIC "Build and statically link the RAFT libraries" OFF)
 option(CUML_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" OFF)
 option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF)
 
-message(VERBOSE "CUML: Building libcuml_c shared library. Contains the cuML C API: ${BUILD_CUML_C_LIBRARY}")
-message(VERBOSE "CUML: Building libcuml shared library: ${BUILD_CUML_CPP_LIBRARY}")
-message(VERBOSE "CUML: Building cuML algorithm tests: ${BUILD_CUML_TESTS}")
-message(VERBOSE "CUML: Building cuML multigpu algorithm tests: ${BUILD_CUML_MG_TESTS}")
-message(VERBOSE "CUML: Building ml-prims tests: ${BUILD_PRIMS_TESTS}")
-message(VERBOSE "CUML: Building C++ API usage examples: ${BUILD_CUML_EXAMPLES}")
-message(VERBOSE "CUML: Building cuML C++ benchmark tests: ${BUILD_CUML_BENCH}")
-message(VERBOSE "CUML: Building ml-prims C++ benchmark tests: ${BUILD_CUML_PRIMS_BENCH}")
-message(VERBOSE "CUML: Building the standard NCCL+UCX Communicator: ${BUILD_CUML_STD_COMMS}")
-message(VERBOSE "CUML: Building the MPI+NCCL Communicator (used for testing): ${BUILD_CUML_MPI_COMMS}")
-message(VERBOSE "CUML: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}")
-message(VERBOSE "CUML: Disabling OpenMP: ${DISABLE_OPENMP}")
-message(VERBOSE "CUML: Enabling algorithms that use libcumlprims_mg: ${ENABLE_CUMLPRIMS_MG}")
-message(VERBOSE "CUML: Enabling kernel resource usage info: ${KERNEL_INFO}")
-message(VERBOSE "CUML: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}")
-message(VERBOSE "CUML: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}")
-message(VERBOSE "CUML: Enabling nvtx markers: ${NVTX}")
-message(VERBOSE "CUML: Disabling all mnmg components and comms libraries: ${SINGLEGPU}")
-message(VERBOSE "CUML: Cache build artifacts with ccache: ${USE_CCACHE}")
-message(VERBOSE "CUML: Build and statically link RAFT libraries: ${CUML_USE_RAFT_STATIC}")
-message(VERBOSE "CUML: Build and statically link FAISS library: ${CUML_USE_FAISS_STATIC}")
-message(VERBOSE "CUML: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}")
+message(VERBOSE "CUML_CPP: Building libcuml_c shared library. Contains the cuML C API: ${BUILD_CUML_C_LIBRARY}")
+message(VERBOSE "CUML_CPP: Building libcuml shared library: ${BUILD_CUML_CPP_LIBRARY}")
+message(VERBOSE "CUML_CPP: Building cuML algorithm tests: ${BUILD_CUML_TESTS}")
+message(VERBOSE "CUML_CPP: Building cuML multigpu algorithm tests: ${BUILD_CUML_MG_TESTS}")
+message(VERBOSE "CUML_CPP: Building ml-prims tests: ${BUILD_PRIMS_TESTS}")
+message(VERBOSE "CUML_CPP: Building C++ API usage examples: ${BUILD_CUML_EXAMPLES}")
+message(VERBOSE "CUML_CPP: Building cuML C++ benchmark tests: ${BUILD_CUML_BENCH}")
+message(VERBOSE "CUML_CPP: Building ml-prims C++ benchmark tests: ${BUILD_CUML_PRIMS_BENCH}")
+message(VERBOSE "CUML_CPP: Building the standard NCCL+UCX Communicator: ${BUILD_CUML_STD_COMMS}")
+message(VERBOSE "CUML_CPP: Building the MPI+NCCL Communicator (used for testing): ${BUILD_CUML_MPI_COMMS}")
+message(VERBOSE "CUML_CPP: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}")
+message(VERBOSE "CUML_CPP: Disabling OpenMP: ${DISABLE_OPENMP}")
+message(VERBOSE "CUML_CPP: Enabling algorithms that use libcumlprims_mg: ${ENABLE_CUMLPRIMS_MG}")
+message(VERBOSE "CUML_CPP: Enabling kernel resource usage info: ${KERNEL_INFO}")
+message(VERBOSE "CUML_CPP: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}")
+message(VERBOSE "CUML_CPP: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}")
+message(VERBOSE "CUML_CPP: Enabling nvtx markers: ${NVTX}")
+message(VERBOSE "CUML_CPP: Disabling all mnmg components and comms libraries: ${SINGLEGPU}")
+message(VERBOSE "CUML_CPP: Cache build artifacts with ccache: ${USE_CCACHE}")
+message(VERBOSE "CUML_CPP: Build and statically link RAFT libraries: ${CUML_USE_RAFT_STATIC}")
+message(VERBOSE "CUML_CPP: Build and statically link FAISS library: ${CUML_USE_FAISS_STATIC}")
+message(VERBOSE "CUML_CPP: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}")
 
 set(CUML_ALGORITHMS "ALL" CACHE STRING "Experimental: Choose which algorithms are built into libcuml++.so. Can specify individual algorithms or groups in a semicolon-separated list.")
-message(VERBOSE "CUML: Building libcuml++ with algoriths: '${CUML_ALGORITHMS}'.")
+message(VERBOSE "CUML_CPP: Building libcuml++ with algoriths: '${CUML_ALGORITHMS}'.")
 
 # Set RMM logging level
 set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.")
 set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF")
-message(VERBOSE "CUML: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.")
+message(VERBOSE "CUML_CPP: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.")
 
 ##############################################################################
 # - Target names -------------------------------------------------------------
@@ -121,7 +121,7 @@ set(PRIMS_BENCH_TARGET "prims_benchmark")
 if(DETECT_CONDA_ENV)
   rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH )
   if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX})
-      message(STATUS "CUML: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}")
+      message(STATUS "CUML_CPP: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}")
       set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}")
   endif()
 endif()
@@ -132,7 +132,7 @@ endif()
 if (NOT DISABLE_OPENMP)
   find_package(OpenMP)
   if(OpenMP_FOUND)
-    message(STATUS "CUML: OpenMP found in ${OPENMP_INCLUDE_DIRS}")
+    message(STATUS "CUML_CPP: OpenMP found in ${OPENMP_INCLUDE_DIRS}")
   endif()
 endif()
 
@@ -170,8 +170,8 @@ endif()
 
 # SingleGPU build disables cumlprims_mg and comms components
 if(SINGLEGPU)
-  message(STATUS "Detected SINGLEGPU build option")
-  message(STATUS "Disabling Multi-GPU components and comms libraries")
+  message(STATUS "CUML_CPP: Detected SINGLEGPU build option")
+  message(STATUS "CUML_CPP: Disabling Multi-GPU components and comms libraries")
   set(BUILD_CUML_MG_TESTS OFF)
   set(BUILD_CUML_STD_COMMS OFF)
   set(BUILD_CUML_MPI_COMMS OFF)
@@ -180,7 +180,7 @@ if(SINGLEGPU)
 endif()
 
 if(BUILD_CUML_MG_TESTS AND NOT SINGLEGPU)
-  message(STATUS "Detected BUILD_CUML_MG_TESTS set to ON. Enabling BUILD_CUML_MPI_COMMS")
+  message(STATUS "CUML_CPP: Detected BUILD_CUML_MG_TESTS set to ON. Enabling BUILD_CUML_MPI_COMMS")
   set(BUILD_CUML_MPI_COMMS ON)
 endif()
 

cpp/Doxyfile.in

@@ -1082,7 +1082,7 @@ HTML_FILE_EXTENSION    = .html
 # of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_HEADER            =
+HTML_HEADER            = @CMAKE_CURRENT_SOURCE_DIR@/header.html
 
 # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
 # generated HTML page. If the tag is left blank doxygen will generate a standard