Support sparse input for SVC and SVR (#5273)

This PR adds support for sparse input to SVR and SVC. 'fit' as well as 'predict' can be called with sparse data compatible/convertible to SparseCumlArray. Support vectors in the model might also be stored as sparse data and can be retrieved as such.
This PR requires rapidsai/raft#1296 to provide sparse kernel computations.
Corresponding issue: #2197

Authors:
  - Malte Förster (https://github.com/mfoerste4)
  - Tamas Bela Feher (https://github.com/tfeher)

Approvers:
  - Tamas Bela Feher (https://github.com/tfeher)
  - Corey J. Nolet (https://github.com/cjnolet)

URL: #5273

cpp/bench/sg/svc.cu

@@ -102,7 +102,7 @@ std::vector<SvcParams<D>> getInputs()
 
   // SvmParameter{C, cache_size, max_iter, nochange_steps, tol, verbosity})
   p.svm_param = ML::SVM::SvmParameter{1, 200, 100, 100, 1e-3, CUML_LEVEL_INFO, 0, ML::SVM::C_SVC};
-  p.model     = ML::SVM::SvmModel<D>{0, 0, 0, nullptr, nullptr, nullptr, 0, nullptr};
+  p.model     = ML::SVM::SvmModel<D>{0, 0, 0, nullptr, {}, nullptr, 0, nullptr};
 
   std::vector<Triplets> rowcols = {{50000, 2, 2}, {2048, 100000, 2}, {50000, 1000, 2}};
 

cpp/include/cuml/svm/svc.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -61,6 +61,42 @@ void svcFit(const raft::handle_t& handle,
             SvmModel<math_t>& model,
             const math_t* sample_weight);
 
+/**
+ * @brief Fit a support vector classifier to the training data.
+ *
+ * Each row of the input data stores a feature vector.
+ * We use the SMO method to fit the SVM.
+ *
+ * The output device buffers in model shall be unallocated on entry.
+ *
+ * @tparam math_t floating point type
+ * @param [in] handle the cuML handle
+ * @param [in] indptr device pointer for CSR row positions. Size [n_rows + 1].
+ * @param [in] indices device pointer for CSR column indices. Size [nnz].
+ * @param [in] data device pointer for the CSR data. Size [nnz].
+ * @param [in] n_rows number of rows
+ * @param [in] n_cols number of columns
+ * @param [in] nnz number of stored entries.
+ * @param [in] labels device pointer for the labels. Size [n_rows].
+ * @param [in] param parameters for training
+ * @param [in] kernel_params parameters for the kernel function
+ * @param [out] model parameters of the trained model
+ * @param [in] sample_weight optional sample weights, size [n_rows]
+ */
+template <typename math_t>
+void svcFitSparse(const raft::handle_t& handle,
+                  int* indptr,
+                  int* indices,
+                  math_t* data,
+                  int n_rows,
+                  int n_cols,
+                  int nnz,
+                  math_t* labels,
+                  const SvmParameter& param,
+                  raft::distance::kernels::KernelParams& kernel_params,
+                  SvmModel<math_t>& model,
+                  const math_t* sample_weight);
+
 /**
  * @brief Predict classes or decision function value for samples in input.
  *
@@ -74,7 +110,7 @@ void svcFit(const raft::handle_t& handle,
  * \f]
  *
  * pred(x_i) = label[sign(f(x_i))], if predict_class==true, or
- * pred(x_i) = f(x_i),       if predict_class==falsee.
+ * pred(x_i) = f(x_i),       if predict_class==false.
  *
  * @tparam math_t floating point type
  * @param handle the cuML handle
@@ -101,6 +137,51 @@ void svcPredict(const raft::handle_t& handle,
                 math_t buffer_size,
                 bool predict_class);
 
+/**
+ * @brief Predict classes or decision function value for samples in input.
+ *
+ * We evaluate the decision function f(x_i). Depending on the parameter
+ * predict_class, we either return f(x_i) or the label corresponding to
+ * sign(f(x_i)).
+ *
+ * The predictions are calculated according to the following formulas:
+ * \f[
+ *    f(x_i) = \sum_{j=1}^n_support K(x_i, x_j) * dual_coefs[j] + b)
+ * \f]
+ *
+ * pred(x_i) = label[sign(f(x_i))], if predict_class==true, or
+ * pred(x_i) = f(x_i),       if predict_class==falsee.
+ *
+ * @tparam math_t floating point type
+ * @param handle the cuML handle
+ * @param [in] indptr device pointer for CSR row positions. Size [n_rows + 1].
+ * @param [in] indices device pointer for CSR column indices. Size [nnz].
+ * @param [in] data device pointer for the CSR data. Size [nnz].
+ * @param [in] n_rows number of rows
+ * @param [in] n_cols number of columns
+ * @param [in] nnz number of stored entries.
+ * @param [in] kernel_params parameters for the kernel function
+ * @param [in] model SVM model parameters
+ * @param [out] preds device pointer to store the predicted class labels.
+ *    Size [n_rows]. Should be allocated on entry.
+ * @param [in] buffer_size size of temporary buffer in MiB
+ * @param [in] predict_class whether to predict class label (true), or just
+ *     return the decision function value (false)
+ */
+template <typename math_t>
+void svcPredictSparse(const raft::handle_t& handle,
+                      int* indptr,
+                      int* indices,
+                      math_t* data,
+                      int n_rows,
+                      int n_cols,
+                      int nnz,
+                      raft::distance::kernels::KernelParams& kernel_params,
+                      const SvmModel<math_t>& model,
+                      math_t* preds,
+                      math_t buffer_size,
+                      bool predict_class);
+
 /**
  * Deallocate device buffers in the SvmModel struct.
  *

cpp/include/cuml/svm/svm_model.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,15 @@
 namespace ML {
 namespace SVM {
 
+// Contains array(s) for matrix storage
+template <typename math_t>
+struct SupportStorage {
+  int nnz      = -1;
+  int* indptr  = nullptr;
+  int* indices = nullptr;
+  math_t* data = nullptr;
+};
+
 /**
  * Parameters that describe a trained SVM model.
  * All pointers are device pointers.
@@ -32,8 +41,9 @@ struct SvmModel {
   //! Size [n_support].
   math_t* dual_coefs;
 
-  //! Support vectors in column major format. Size [n_support x n_cols].
-  math_t* x_support;
+  //! Support vector storage - can contain either CSR or dense
+  SupportStorage<math_t> support_matrix;
+
   //! Indices (from the training set) of the support vectors, size [n_support].
   int* support_idx;
 

cpp/include/cuml/svm/svr.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2021, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -57,6 +57,41 @@ void svrFit(const raft::handle_t& handle,
             SvmModel<math_t>& model,
             const math_t* sample_weight = nullptr);
 
+/**
+ * @brief Fit a support vector regressor to the training data.
+ *
+ * Each row of the input data stores a feature vector.
+ *
+ * The output buffers in model shall be unallocated on entry.
+ *
+ * @tparam math_t floating point type
+ * @param [in] handle the cuML handle
+ * @param [in] indptr device pointer for CSR row positions. Size [n_rows + 1].
+ * @param [in] indices device pointer for CSR column indices. Size [nnz].
+ * @param [in] data device pointer for the CSR data. Size [nnz].
+ * @param [in] n_rows number of rows
+ * @param [in] n_cols number of columns
+ * @param [in] nnz number of stored entries.
+ * @param [in] y device pointer for target values. Size [n_rows].
+ * @param [in] param parameters for training
+ * @param [in] kernel_params parameters for the kernel function
+ * @param [out] model parameters of the trained model
+ * @param [in] sample_weight optional sample weights, size [n_rows]
+ */
+template <typename math_t>
+void svrFitSparse(const raft::handle_t& handle,
+                  int* indptr,
+                  int* indices,
+                  math_t* data,
+                  int n_rows,
+                  int n_cols,
+                  int nnz,
+                  math_t* y,
+                  const SvmParameter& param,
+                  raft::distance::kernels::KernelParams& kernel_params,
+                  SvmModel<math_t>& model,
+                  const math_t* sample_weight = nullptr);
+
 // For prediction we use svcPredict
 
 };  // end namespace SVM