From c4c406883735ac4156c6b32005618b96ebeefda3 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 16 Mar 2021 16:18:14 -0400 Subject: [PATCH] [REVIEW] Removing sparse prims and fused l2 nn prim from cuml (#3578) * Removing sparse prims since they've been moved to raft * Updating copyrights * Updating raft hash * Setting libcumprims to 0.18 for now * Using fused l2 nn from raft * Fixing style * Updating copyright * Using raft hash to make CI build * Moving cumlprims conda recipe back to minor_version * Updating style * Updating raft hash to point to my branch until raft pr is merged * Removing tests that are no longer needed * Updating raft hash to branch-0.19 * Updating raft hash * Updating nccl version * Updating includes * Removing files from bad merge --- cpp/bench/prims/fused_l2_nn.cu | 13 +- cpp/cmake/Dependencies.cmake | 5 +- cpp/src/dbscan/adjgraph/algo.cuh | 2 +- cpp/src/dbscan/runner.cuh | 2 +- cpp/src/kmeans/common.cuh | 19 +- cpp/src/knn/knn_sparse.cu | 2 +- cpp/src/spectral/spectral.cu | 6 +- cpp/src/tsne/distances.cuh | 7 +- cpp/src/umap/fuzzy_simpl_set/naive.cuh | 8 +- cpp/src/umap/fuzzy_simpl_set/runner.cuh | 2 +- cpp/src/umap/init_embed/runner.cuh | 2 +- cpp/src/umap/init_embed/spectral_algo.cuh | 2 +- cpp/src/umap/knn_graph/algo.cuh | 2 +- cpp/src/umap/runner.cuh | 12 +- cpp/src/umap/simpl_set_embed/algo.cuh | 6 +- cpp/src/umap/simpl_set_embed/runner.cuh | 2 +- cpp/src/umap/supervised.cuh | 14 +- .../distance/epsilon_neighborhood.cuh | 9 +- cpp/src_prims/distance/fused_l2_nn.cuh | 414 ------------ cpp/src_prims/linalg/contractions.cuh | 303 --------- cpp/src_prims/selection/knn.cuh | 2 +- cpp/src_prims/sparse/convert/coo.cuh | 74 -- cpp/src_prims/sparse/convert/csr.cuh | 189 ------ cpp/src_prims/sparse/convert/dense.cuh | 110 --- cpp/src_prims/sparse/coo.cuh | 259 ------- cpp/src_prims/sparse/csr.cuh | 264 -------- .../sparse/distance/bin_distance.cuh | 207 ------ cpp/src_prims/sparse/distance/common.h | 59 -- cpp/src_prims/sparse/distance/coo_spmv.cuh | 360 ---------- cpp/src_prims/sparse/distance/csr_spmv.cuh | 498 -------------- cpp/src_prims/sparse/distance/distance.cuh | 127 ---- cpp/src_prims/sparse/distance/ip_distance.cuh | 335 --------- cpp/src_prims/sparse/distance/l2_distance.cuh | 300 --------- cpp/src_prims/sparse/distance/lp_distance.cuh | 221 ------ cpp/src_prims/sparse/distance/operators.cuh | 88 --- cpp/src_prims/sparse/linalg/add.cuh | 226 ------- cpp/src_prims/sparse/linalg/degree.cuh | 184 ----- cpp/src_prims/sparse/linalg/norm.cuh | 169 ----- cpp/src_prims/sparse/linalg/spectral.cuh | 103 --- cpp/src_prims/sparse/linalg/symmetrize.cuh | 309 --------- cpp/src_prims/sparse/linalg/transpose.h | 87 --- cpp/src_prims/sparse/op/filter.cuh | 201 ------ cpp/src_prims/sparse/op/row_op.cuh | 76 --- cpp/src_prims/sparse/op/slice.h | 99 --- cpp/src_prims/sparse/op/sort.h | 105 --- cpp/src_prims/sparse/selection/knn.cuh | 450 ------------- cpp/src_prims/sparse/selection/selection.cuh | 159 ----- cpp/src_prims/sparse/utils.h | 115 ---- cpp/test/CMakeLists.txt | 21 +- cpp/test/prims/csr.cu | 208 ------ cpp/test/prims/fused_l2_nn.cu | 331 --------- cpp/test/prims/sparse/add.cu | 174 ----- cpp/test/prims/sparse/convert_coo.cu | 98 --- cpp/test/prims/sparse/convert_csr.cu | 180 ----- cpp/test/prims/sparse/csr_row_slice.cu | 184 ----- cpp/test/prims/sparse/csr_to_dense.cu | 140 ---- cpp/test/prims/sparse/csr_transpose.cu | 174 ----- cpp/test/prims/sparse/degree.cu | 110 --- cpp/test/prims/sparse/dist_coo_spmv.cu | 636 ------------------ cpp/test/prims/sparse/dist_csr_spmv.cu | 620 ----------------- cpp/test/prims/sparse/filter.cu | 122 ---- cpp/test/prims/sparse/knn.cu | 178 ----- cpp/test/prims/sparse/norm.cu | 127 ---- cpp/test/prims/sparse/row_op.cu | 111 --- cpp/test/prims/sparse/selection.cu | 158 ----- cpp/test/prims/sparse/sort.cu | 103 --- cpp/test/prims/sparse/symmetrize.cu | 111 --- 67 files changed, 63 insertions(+), 9931 deletions(-) delete mode 100644 cpp/src_prims/distance/fused_l2_nn.cuh delete mode 100644 cpp/src_prims/linalg/contractions.cuh delete mode 100644 cpp/src_prims/sparse/convert/coo.cuh delete mode 100644 cpp/src_prims/sparse/convert/csr.cuh delete mode 100644 cpp/src_prims/sparse/convert/dense.cuh delete mode 100644 cpp/src_prims/sparse/coo.cuh delete mode 100644 cpp/src_prims/sparse/csr.cuh delete mode 100644 cpp/src_prims/sparse/distance/bin_distance.cuh delete mode 100644 cpp/src_prims/sparse/distance/common.h delete mode 100644 cpp/src_prims/sparse/distance/coo_spmv.cuh delete mode 100644 cpp/src_prims/sparse/distance/csr_spmv.cuh delete mode 100644 cpp/src_prims/sparse/distance/distance.cuh delete mode 100644 cpp/src_prims/sparse/distance/ip_distance.cuh delete mode 100644 cpp/src_prims/sparse/distance/l2_distance.cuh delete mode 100644 cpp/src_prims/sparse/distance/lp_distance.cuh delete mode 100644 cpp/src_prims/sparse/distance/operators.cuh delete mode 100644 cpp/src_prims/sparse/linalg/add.cuh delete mode 100644 cpp/src_prims/sparse/linalg/degree.cuh delete mode 100644 cpp/src_prims/sparse/linalg/norm.cuh delete mode 100644 cpp/src_prims/sparse/linalg/spectral.cuh delete mode 100644 cpp/src_prims/sparse/linalg/symmetrize.cuh delete mode 100644 cpp/src_prims/sparse/linalg/transpose.h delete mode 100644 cpp/src_prims/sparse/op/filter.cuh delete mode 100644 cpp/src_prims/sparse/op/row_op.cuh delete mode 100644 cpp/src_prims/sparse/op/slice.h delete mode 100644 cpp/src_prims/sparse/op/sort.h delete mode 100644 cpp/src_prims/sparse/selection/knn.cuh delete mode 100644 cpp/src_prims/sparse/selection/selection.cuh delete mode 100644 cpp/src_prims/sparse/utils.h delete mode 100644 cpp/test/prims/csr.cu delete mode 100644 cpp/test/prims/fused_l2_nn.cu delete mode 100644 cpp/test/prims/sparse/add.cu delete mode 100644 cpp/test/prims/sparse/convert_coo.cu delete mode 100644 cpp/test/prims/sparse/convert_csr.cu delete mode 100644 cpp/test/prims/sparse/csr_row_slice.cu delete mode 100644 cpp/test/prims/sparse/csr_to_dense.cu delete mode 100644 cpp/test/prims/sparse/csr_transpose.cu delete mode 100644 cpp/test/prims/sparse/degree.cu delete mode 100644 cpp/test/prims/sparse/dist_coo_spmv.cu delete mode 100644 cpp/test/prims/sparse/dist_csr_spmv.cu delete mode 100644 cpp/test/prims/sparse/filter.cu delete mode 100644 cpp/test/prims/sparse/knn.cu delete mode 100644 cpp/test/prims/sparse/norm.cu delete mode 100644 cpp/test/prims/sparse/row_op.cu delete mode 100644 cpp/test/prims/sparse/selection.cu delete mode 100644 cpp/test/prims/sparse/sort.cu delete mode 100644 cpp/test/prims/sparse/symmetrize.cu diff --git a/cpp/bench/prims/fused_l2_nn.cu b/cpp/bench/prims/fused_l2_nn.cu index 60a2a6cfaa..59db5a682e 100644 --- a/cpp/bench/prims/fused_l2_nn.cu +++ b/cpp/bench/prims/fused_l2_nn.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2021, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,8 @@ */ #include -#include #include +#include #include #include #include "../common/ml_benchmark.hpp" @@ -52,7 +52,7 @@ struct FusedL2NN : public Fixture { raft::linalg::rowNorm(yn, y, params.k, params.n, raft::linalg::L2Norm, true, stream); auto blks = raft::ceildiv(params.m, 256); - MLCommon::Distance::initKernel, int> + raft::distance::initKernel, int> <<>>(out, params.m, std::numeric_limits::max(), op); } @@ -69,9 +69,9 @@ struct FusedL2NN : public Fixture { void runBenchmark(::benchmark::State& state) override { loopOnState(state, [this]() { // it is enough to only benchmark the L2-squared metric - MLCommon::Distance::fusedL2NN, int>( + raft::distance::fusedL2NN, int>( out, x, y, xn, yn, params.m, params.n, params.k, (void*)workspace, op, - false, false, stream); + pairRedOp, false, false, stream); }); } @@ -80,7 +80,8 @@ struct FusedL2NN : public Fixture { T *x, *y, *xn, *yn; cub::KeyValuePair* out; int* workspace; - MLCommon::Distance::MinAndDistanceReduceOp op; + raft::distance::KVPMinReduce pairRedOp; + raft::distance::MinAndDistanceReduceOp op; }; // struct FusedL2NN static std::vector getInputs() { diff --git a/cpp/cmake/Dependencies.cmake b/cpp/cmake/Dependencies.cmake index f28a47b341..f0b7a35690 100644 --- a/cpp/cmake/Dependencies.cmake +++ b/cpp/cmake/Dependencies.cmake @@ -39,7 +39,7 @@ else(DEFINED ENV{RAFT_PATH}) ExternalProject_Add(raft GIT_REPOSITORY https://github.com/rapidsai/raft.git - GIT_TAG 4a79adcb0c0e87964dcdc9b9122f242b5235b702 + GIT_TAG 6455e05b3889db2b495cf3189b33c2b07bfbebf2 PREFIX ${RAFT_DIR} CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -92,7 +92,8 @@ message(STATUS "RMM: RMM_INCLUDE_DIRS set to ${RMM_INCLUDE_DIRS}") # - NCCL --------------------------------------------------------------------- if(BUILD_CUML_MPI_COMMS OR BUILD_CUML_STD_COMMS) - find_package(NCCL REQUIRED) + # At least NCCL 2.8 required for p2p methods on comms + find_package(NCCL 2.8 REQUIRED) endif(BUILD_CUML_MPI_COMMS OR BUILD_CUML_STD_COMMS) ############################################################################## diff --git a/cpp/src/dbscan/adjgraph/algo.cuh b/cpp/src/dbscan/adjgraph/algo.cuh index 80474863b2..138a595404 100644 --- a/cpp/src/dbscan/adjgraph/algo.cuh +++ b/cpp/src/dbscan/adjgraph/algo.cuh @@ -23,7 +23,7 @@ #include "../common.cuh" #include "pack.h" -#include +#include using namespace thrust; diff --git a/cpp/src/dbscan/runner.cuh b/cpp/src/dbscan/runner.cuh index dc5fc694bc..1cd2387e89 100644 --- a/cpp/src/dbscan/runner.cuh +++ b/cpp/src/dbscan/runner.cuh @@ -21,7 +21,7 @@ #include #include