Merge pull request #15 from rapidsai/branch-0.8

0.8 into local 0.8

.gitmodules

@@ -1,19 +1,13 @@
-[submodule "thirdparty/cuml/faiss"]
-	path = thirdparty/cuml/faiss
+[submodule "thirdparty/faiss"]
+	path = thirdparty/faiss
 	url = https://github.com/facebookresearch/faiss
     ignore = dirty
-[submodule "external/googletest"]
-    path = thirdparty/cuml/googletest
-    url = https://github.com/google/googletest
-[submodule "ml-prims/external/cutlass"]
-    path = thirdparty/ml-prims/cutlass
+[submodule "thirdparty/cutlass"]
+	path = thirdparty/cutlass
     url = https://github.com/NVIDIA/cutlass
-[submodule "ml-prims/external/cub"]
-    path = thirdparty/ml-prims/cub
+[submodule "thirdparty/cub"]
+	path = thirdparty/cub
     url = https://github.com/NVlabs/cub
-[submodule "ml-prims/external/googletest"]
-    path = thirdparty/ml-prims/googletest
+[submodule "thirdparty/googletest"]
+	path = thirdparty/googletest
     url = https://github.com/google/googletest
-[submodule "thirdparty/h2o4gpu"]
-	path = thirdparty/h2o4gpu
-	url = https://github.com/h2oai/h2o4gpu

BUILD.md

@@ -4,15 +4,17 @@
 
 To install cuML from source, ensure the dependencies are met:
 
-1. [cuDF](https://github.com/rapidsai/cudf) (>=0.7)
+1. [cuDF](https://github.com/rapidsai/cudf) (>=0.8)
 2. zlib
 3. cmake (>= 3.12.4)
 4. CUDA (>= 9.2)
 5. Cython (>= 0.29)
 6. gcc (>=5.4.0)
 7. BLAS - Any BLAS compatible with cmake's [FindBLAS](https://cmake.org/cmake/help/v3.12/module/FindBLAS.html). Note that the blas has to be installed to the same folder system as cmake, for example if using conda installed cmake, the blas implementation should also be installed in the conda environment.
 
-### Installing from Source:
+## Installing from Source:
+
+### Typical Process
 
 Once dependencies are present, follow the steps below:
 
@@ -21,9 +23,9 @@ Once dependencies are present, follow the steps below:
 $ git clone --recurse-submodules https://github.com/rapidsai/cuml.git
 ```
 
-2. Build and install `libcuml` (the C++/CUDA library containing the cuML algorithms), starting from the repository root folder:
+2. Build and install `libcuml++` (C++/CUDA library containing the cuML algorithms), starting from the repository root folder:
 ```bash
-$ cd cuML
+$ cd cpp
 $ mkdir build
 $ cd build
 $ export CUDA_BIN_PATH=$CUDA_HOME # (optional env variable if cuda binary is not in the PATH. Default CUDA_HOME=/path/to/cuda/)
@@ -47,7 +49,7 @@ The configuration script will print the BLAS found on the search path. If the ve
 If using conda and a conda installed cmake, the `openblas` conda package is recommended and can be explicitly specified for `blas` and `lapack`:
 
 ```bash
-cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DBLAS_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.so -DLAPACK_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.so
+cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DBLAS_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.so
 ```
 
 Additionally, to reduce compile times, you can specify a GPU compute capability to compile for, for example for Volta GPUs:
@@ -56,8 +58,9 @@ Additionally, to reduce compile times, you can specify a GPU compute capability
 $ cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DGPU_ARCHS="70"
 ```
 
+There are many options to configure the build process, see the [customizing build section](#custom-build-options).
 
-3. Build `libcuml`:
+3. Build `libcuml++`:
 
 ```bash
 $ make -j
@@ -67,12 +70,16 @@ $ make install
 To run tests (optional):
 
 ```bash
-$ ./ml_test
+$ ./test/ml # Single GPU algorithm tests
+$ ./test/ml_mg # Multi GPU algorithm tests
+$ ./test/prims # ML Primitive function tests
 ```
 
 If you want a list of the available tests:
 ```bash
-$ ./ml_test --gtest_list_tests
+$ ./test/ml --gtest_list_tests # Single GPU algorithm tests
+$ ./test/ml_mg --gtest_list_tests # Multi GPU algorithm tests
+$ ./test/prims --gtest_list_tests # ML Primitive function tests
 ```
 
 4. Build the `cuml` python package:
@@ -85,12 +92,12 @@ $ python setup.py build_ext --inplace
 To run Python tests (optional):
 
 ```bash
-$ py.test -v
+$ pytest -v
 ```
 
 If you want a list of the available tests:
 ```bash
-$ py.test cuML/test --collect-only
+$ pytest cuML/test --collect-only
 ```
 
 5. Finally, install the Python package to your Python path:
@@ -99,20 +106,20 @@ $ py.test cuML/test --collect-only
 $ python setup.py install
 ```
 
-6. You can also build and run tests for the machine learning primitive header only library located in the `ml-prims` folder. From the repository root:
-
-```bash
-$ cd ml-prims
-$ mkdir build
-$ cd build
-$ cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DGPU_ARCHS="70" # specifying GPU_ARCH is optional, but significantly reduces compile time
-$ make -j
-```
-
-To run the ml-prim tests:
-
-```bash
-$./test/mlcommon_test
-```
+### Custom Build Options
 
+cuML's cmake has the following configurable flags available:
 
+| Flag | Possible Values | Default Value | Behavior |
+| --- | --- | --- | --- |
+| BLAS_LIBRARIES | path/to/blas_lib | "" | Optional variable allowing to manually specify location of BLAS library. |
+| BUILD_CUML_CPP_LIBRARY | [ON, OFF]  | ON  | Enable/disable building libcuml++ shared library. Setting this variable to `OFF` sets the variables BUILD_CUML_TESTS, BUILD_CUML_MG_TESTS and BUILD_CUML_EXAMPLES to `OFF` |
+| BUILD_CUML_TESTS | [ON, OFF]  | ON  |  Enable/disable building cuML algorithm test executable `ml_test`.  |
+| BUILD_CUML_MG_TESTS | [ON, OFF]  | ON  |  Enable/disable building cuML algorithm test executable `ml_mg_test`. |
+| BUILD_PRIMS_TESTS | [ON, OFF]  | ON  | Enable/disable building cuML algorithm test executable `prims_test`.  |
+| BUILD_CUML_EXAMPLES | [ON, OFF]  | ON  | Enable/disable building cuML C++ API usage examples.  |
+| CMAKE_CXX11_ABI | [ON, OFF]  | ON  | Enable/disable the GLIBCXX11 ABI  |
+| DISABLE_OPENMP | [ON, OFF]  | OFF  | Set to `ON` to disable OpenMP  |
+| GPU_ARCHS |  List of GPU architectures, semicolon-separated | 60;70;75  | List of GPU architectures that all artifacts are compiled for.  |
+| KERNEL_INFO | [ON, OFF]  | OFF  | Enable/disable kernel resource usage info in nvcc. |
+| LINE_INFO | [ON, OFF]  | OFF  | Enable/disable lineinfo in nvcc.  |

CHANGELOG.md

@@ -1,19 +1,27 @@
 # cuML 0.8.0 (Date TBD)
 
 ## New Features
-
+- PR #515: Added Random Projection feature
+- PR #504: Contingency matrix ml-prim
 
 ## Improvements
 
+- PR #597: C++ cuML and ml-prims folder refactor
 - PR #590: QN Recover from numeric errors
 - PR #482: Introduce cumlHandle for pca and tsvd
 - PR #573: Remove use of unnecessary cuDF column and series copies
+- PR #601: Cython PEP8 cleanup and CI integration
+- PR #596: Introduce cumlHandle for ols and ridge
+- PR #579: Introduce cumlHandle for cd and sgd, and propagate C++ errors in cython level for cd and sgd
+- PR #604: Adding cumlHandle to kNN, spectral methods, and UMAP
+- PR #622: Updated to use 0.8 dependencies
 
 ## Bug Fixes
 - PR #584: Added missing virtual destructor to deviceAllocator and hostAllocator
+- PR #620: C++: Removed old unit-test files in ml-prims
 
 
-# cuML 0.7.0 (Date TBD)
+# cuML 0.7.0 (10 May 2019)
 
 ## New Features
 
@@ -25,12 +33,12 @@
 - PR #435: Expose cumlhandle in cython + developer guide
 - PR #455: Remove default-stream arguement across ml-prims and cuML
 - PR #375: cuml cpp shared library renamed to libcuml++.so
-- PR #444: Add supervised training to UMAP
 - PR #460: Random Forest & Decision Trees (Single-GPU, Classification)
 - PR #491: Add doxygen build target for ml-prims
 - PR #505: Add R-Squared Score to python interface
 - PR #507: Add coordinate descent for lasso and elastic-net
 - PR #511: Add a minmax ml-prim
+- PR #516: Added Trustworthiness score feature
 - PR #520: Add local build script to mimic gpuCI
 - PR #503: Add column-wise matrix sort primitive
 - PR #525: Add docs build script to cuML
@@ -60,6 +68,7 @@
 - PR #540: Use latest release version in update-version CI script
 - PR #552: Re-enable assert in kmeans tests with xfail as needed
 - PR #581: Add shared memory fast col major to row major function back with bound checks
+- PR #592: More efficient matrix copy/reverse methods
 
 ## Bug Fixes
 
@@ -99,6 +108,14 @@
 - PR #577: Use find libz in prims cmake
 - PR #594: fixed cuda-memcheck mean_center test failures
 
+
+# cuML 0.6.1 (09 Apr 2019)
+
+## Bug Fixes
+
+- PR #462 Runtime library path fix for cuML pip package
+
+
 # cuML 0.6.0 (22 Mar 2019)
 
 ## New Features
@@ -145,6 +162,8 @@
 - PR #315: Documentation updating and enhancements
 - PR #330: Added ignored argument to pca.fit_transform to map to sklearn's implemenation
 - PR #342: Change default ABI to ON
+- PR #572: Pulling DBSCAN components into reusable primitives
+
 
 ## Bug Fixes
 

ci/checks/style.sh

@@ -12,7 +12,7 @@ PATH=/conda/bin:$PATH
 source activate gdf
 
 # Run flake8 and get results/return code
-FLAKE=`flake8 --exclude=cuML,ml-prims,__init__.py,versioneer.py`
+FLAKE=`flake8 --exclude=cuML,ml-prims,__init__.py,versioneer.py && flake8 --config=python/.flake8.cython`
 RETVAL=$?
 
 # Output results if failure otherwise show pass

ci/cpu/build.sh

@@ -15,9 +15,9 @@ export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
 export PARALLEL_LEVEL=4
 
 # Set versions of packages needed to be grabbed
-export CUDF_VERSION=0.7.*
-export NVSTRINGS_VERSION=0.7.*
-export RMM_VERSION=0.7.*
+export CUDF_VERSION=0.8.*
+export NVSTRINGS_VERSION=0.8.*
+export RMM_VERSION=0.8.*
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE

ci/docs/build.sh

@@ -14,8 +14,8 @@ function logger() {
 export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
 export PARALLEL_LEVEL=4
 export CUDA_REL=${CUDA_VERSION%.*}
-export CUDF_VERSION=0.7.*
-export RMM_VERSION=0.7.*
+export CUDF_VERSION=0.8.*
+export RMM_VERSION=0.8.*
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE

ci/gpu/build.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018, NVIDIA CORPORATION.
+# Copyright (c) 2018-2019, NVIDIA CORPORATION.
 #########################################
 # cuML GPU build and test script for CI #
 #########################################
@@ -16,9 +16,9 @@ export PARALLEL_LEVEL=4
 export CUDA_REL=${CUDA_VERSION%.*}
 
 # Set versions of packages needed to be grabbed
-export CUDF_VERSION=0.7.*
-export NVSTRINGS_VERSION=0.7.*
-export RMM_VERSION=0.7.*
+export CUDF_VERSION=0.8.*
+export NVSTRINGS_VERSION=0.8.*
+export RMM_VERSION=0.8.*
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
@@ -64,21 +64,21 @@ fi
 ################################################################################
 
 logger "Build libcuml..."
-mkdir -p $WORKSPACE/cuML/build
-cd $WORKSPACE/cuML/build
+mkdir -p $WORKSPACE/cpp/build
+cd $WORKSPACE/cpp/build
 logger "Run cmake libcuml..."
-cmake -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=ON -DBLAS_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.a -DLAPACK_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.a $GPU_ARCH ..
+cmake -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=ON -DBLAS_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.a $GPU_ARCH -DBUILD_PRIMS_TESTS=OFF ..
 
 logger "Clean up make..."
 make clean
 
-logger "Make libcuml..."
-make -j${PARALLEL_LEVEL}
+logger "Make libcuml++ and algorithm tests..."
+make -j${PARALLEL_LEVEL} cuml++ ml ml_mg
 
-logger "Install libcuml..."
+logger "Install libcuml++..."
 make -j${PARALLEL_LEVEL} install
 
-logger "Build cuML..."
+logger "Build cuml python package..."
 cd $WORKSPACE/python
 python setup.py build_ext --inplace
 
@@ -91,28 +91,33 @@ logger "Check GPU usage..."
 nvidia-smi
 
 logger "GoogleTest for libcuml..."
-cd $WORKSPACE/cuML/build
-GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./ml_test
+cd $WORKSPACE/cpp/build
+GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./test/ml
 
-logger "Python py.test for cuML..."
+# Disabled while CI/the test become compatible
+# logger "GoogleTest for libcuml mg..."
+# cd $WORKSPACE/cpp/build
+# GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp_mg/" ./test/ml_mg
+
+logger "Python pytest for cuml..."
 cd $WORKSPACE/python
-py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuml.xml -v
+pytest --cache-clear --junitxml=${WORKSPACE}/junit-cuml.xml -v
 
 
 ################################################################################
 # TEST - Build and run ml-prim tests
 ################################################################################
 
 logger "Build ml-prims tests..."
-mkdir -p $WORKSPACE/ml-prims/build
-cd $WORKSPACE/ml-prims/build
-cmake $GPU_ARCH ..
+mkdir -p $WORKSPACE/cpp/build_prims
+cd $WORKSPACE/cpp/build_prims
+cmake -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=ON -DBLAS_LIBRARIES=$CONDA_PREFIX/lib/libopenblas.a $GPU_ARCH -DBUILD_CUML_CPP_LIBRARY=OFF ..
 
 logger "Clean up make..."
 make clean
 logger "Make ml-prims test..."
-make -j${PARALLEL_LEVEL}
+make -j${PARALLEL_LEVEL} prims
 
 logger "Run ml-prims test..."
-cd $WORKSPACE/ml-prims/build
-GTEST_OUTPUT="xml:${WORKSPACE}/test-results/ml-prims/" ./test/mlcommon_test
+cd $WORKSPACE/cpp/build_prims
+GTEST_OUTPUT="xml:${WORKSPACE}/test-results/prims/" ./test/prims

conda/recipes/cuml-cuda10/meta.yaml

@@ -28,14 +28,14 @@ requirements:
     - setuptools
     - cython>=0.29,<0.30
     - cmake>=3.12.4
-    - cudf 0.7*
+    - cudf 0.8*
     - libcuml={{ version }}
     - libcumlMG
     - cudatoolkit {{ cuda_version }}.*
   run:
     - python x.x
     - setuptools
-    - cudf 0.7*
+    - cudf 0.8*
     - libcuml={{ version }}
     - libcumlMG
     - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}

conda/recipes/cuml/meta.yaml

@@ -28,13 +28,13 @@ requirements:
     - setuptools
     - cython>=0.29,<0.30
     - cmake>=3.12.4
-    - cudf 0.7*
+    - cudf 0.8*
     - libcuml={{ version }}
     - cudatoolkit {{ cuda_version }}.*
   run:
     - python x.x
     - setuptools
-    - cudf 0.7*
+    - cudf 0.8*
     - libcuml={{ version }}
     - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
 

conda/recipes/libcuml/build.sh

@@ -12,11 +12,11 @@ printenv
 # Cleanup local git
 git clean -xdf
 # Change directory for build process
-cd cuML
+cd cpp
 # Use CMake-based build procedure
 mkdir build
 cd build
 # configure
 cmake $CMAKE_COMMON_VARIABLES ..
 # build
-make -j${PARALLEL_LEVEL} VERBOSE=1 install
+make -j${PARALLEL_LEVEL} cuml++ VERBOSE=1 install