From e8f45ef47dba157f667f54aa196393dfda23570e Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 10 Mar 2023 13:35:04 -0500 Subject: [PATCH 1/6] Fixing select_k specializations --- cpp/CMakeLists.txt | 4 +- cpp/bench/matrix/select_k.cu | 42 ++++++++++-------- cpp/include/raft/matrix/specializations.cuh | 19 ++++++++ .../specializations/detail/select_k.cuh | 4 +- ..._uint64_t.cu => select_k_float_int64_t.cu} | 4 +- .../detail/select_k_float_uint32_t.cu | 2 +- ...t_uint64_t.cu => select_k_half_int64_t.cu} | 4 +- .../detail/select_k_half_uint32_t.cu | 2 +- cpp/test/matrix/select_k.cu | 44 ++++++++++--------- 9 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 cpp/include/raft/matrix/specializations.cuh rename cpp/src/distance/matrix/specializations/detail/{select_k_half_uint64_t.cu => select_k_float_int64_t.cu} (93%) rename cpp/src/distance/matrix/specializations/detail/{select_k_float_uint64_t.cu => select_k_half_int64_t.cu} (93%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cf4d0ed55e..83390ea881 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -361,9 +361,9 @@ if(RAFT_COMPILE_DIST_LIBRARY) src/distance/distance/specializations/fused_l2_nn_float_int.cu src/distance/distance/specializations/fused_l2_nn_float_int64.cu src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu - src/distance/matrix/specializations/detail/select_k_float_uint64_t.cu + src/distance/matrix/specializations/detail/select_k_float_int64_t.cu src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu - src/distance/matrix/specializations/detail/select_k_half_uint64_t.cu + src/distance/matrix/specializations/detail/select_k_half_int64_t.cu src/distance/neighbors/ivfpq_build.cu src/distance/neighbors/ivfpq_deserialize.cu src/distance/neighbors/ivfpq_serialize.cu diff --git a/cpp/bench/matrix/select_k.cu b/cpp/bench/matrix/select_k.cu index 2c8b8bb67b..3b6f031c77 100644 --- a/cpp/bench/matrix/select_k.cu +++ b/cpp/bench/matrix/select_k.cu @@ -23,6 +23,10 @@ #include #include +#if defined RAFT_DISTANCE_COMPILED +#include +#endif + #include #include #include @@ -105,24 +109,24 @@ const std::vector kInputs{ RAFT_BENCH_REGISTER(SelectK, #KeyT "/" #IdxT "/" #A, kInputs); \ } -SELECTION_REGISTER(float, int, kPublicApi); // NOLINT -SELECTION_REGISTER(float, int, kRadix8bits); // NOLINT -SELECTION_REGISTER(float, int, kRadix11bits); // NOLINT -SELECTION_REGISTER(float, int, kWarpAuto); // NOLINT -SELECTION_REGISTER(float, int, kWarpImmediate); // NOLINT -SELECTION_REGISTER(float, int, kWarpFiltered); // NOLINT -SELECTION_REGISTER(float, int, kWarpDistributed); // NOLINT -SELECTION_REGISTER(float, int, kWarpDistributedShm); // NOLINT - -SELECTION_REGISTER(double, int, kRadix8bits); // NOLINT -SELECTION_REGISTER(double, int, kRadix11bits); // NOLINT -SELECTION_REGISTER(double, int, kWarpAuto); // NOLINT - -SELECTION_REGISTER(double, size_t, kRadix8bits); // NOLINT -SELECTION_REGISTER(double, size_t, kRadix11bits); // NOLINT -SELECTION_REGISTER(double, size_t, kWarpImmediate); // NOLINT -SELECTION_REGISTER(double, size_t, kWarpFiltered); // NOLINT -SELECTION_REGISTER(double, size_t, kWarpDistributed); // NOLINT -SELECTION_REGISTER(double, size_t, kWarpDistributedShm); // NOLINT +SELECTION_REGISTER(float, uint32_t, kPublicApi); // NOLINT +SELECTION_REGISTER(float, uint32_t, kRadix8bits); // NOLINT +SELECTION_REGISTER(float, uint32_t, kRadix11bits); // NOLINT +SELECTION_REGISTER(float, uint32_t, kWarpAuto); // NOLINT +SELECTION_REGISTER(float, uint32_t, kWarpImmediate); // NOLINT +SELECTION_REGISTER(float, uint32_t, kWarpFiltered); // NOLINT +SELECTION_REGISTER(float, uint32_t, kWarpDistributed); // NOLINT +SELECTION_REGISTER(float, uint32_t, kWarpDistributedShm); // NOLINT + +SELECTION_REGISTER(double, uint32_t, kRadix8bits); // NOLINT +SELECTION_REGISTER(double, uint32_t, kRadix11bits); // NOLINT +SELECTION_REGISTER(double, uint32_t, kWarpAuto); // NOLINT + +SELECTION_REGISTER(double, int64_t, kRadix8bits); // NOLINT +SELECTION_REGISTER(double, int64_t, kRadix11bits); // NOLINT +SELECTION_REGISTER(double, int64_t, kWarpImmediate); // NOLINT +SELECTION_REGISTER(double, int64_t, kWarpFiltered); // NOLINT +SELECTION_REGISTER(double, int64_t, kWarpDistributed); // NOLINT +SELECTION_REGISTER(double, int64_t, kWarpDistributedShm); // NOLINT } // namespace raft::matrix diff --git a/cpp/include/raft/matrix/specializations.cuh b/cpp/include/raft/matrix/specializations.cuh new file mode 100644 index 0000000000..07bdeab507 --- /dev/null +++ b/cpp/include/raft/matrix/specializations.cuh @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include diff --git a/cpp/include/raft/matrix/specializations/detail/select_k.cuh b/cpp/include/raft/matrix/specializations/detail/select_k.cuh index 3326162812..3cb1a2d8dc 100644 --- a/cpp/include/raft/matrix/specializations/detail/select_k.cuh +++ b/cpp/include/raft/matrix/specializations/detail/select_k.cuh @@ -35,8 +35,8 @@ namespace raft::matrix::detail { rmm::mr::device_memory_resource*); // Commonly used types -RAFT_INST(float, uint64_t); -RAFT_INST(half, uint64_t); +RAFT_INST(float, int64_t); +RAFT_INST(half, int64_t); // These instances are used in the ivf_pq::search parameterized by the internal_distance_dtype RAFT_INST(float, uint32_t); diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint64_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu similarity index 93% rename from cpp/src/distance/matrix/specializations/detail/select_k_half_uint64_t.cu rename to cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu index b08dc8641c..7a6c7af96d 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint64_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::matrix::detail { @@ -30,6 +30,6 @@ namespace raft::matrix::detail { rmm::cuda_stream_view, \ rmm::mr::device_memory_resource*); -RAFT_INST(half, uint64_t); +RAFT_INST(float, int64_t); } // namespace raft::matrix::detail diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu index 30b7bbbd3e..15e943df24 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::matrix::detail { diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint64_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu similarity index 93% rename from cpp/src/distance/matrix/specializations/detail/select_k_float_uint64_t.cu rename to cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu index c449bf03e4..4d735c5b89 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint64_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::matrix::detail { @@ -30,6 +30,6 @@ namespace raft::matrix::detail { rmm::cuda_stream_view, \ rmm::mr::device_memory_resource*); -RAFT_INST(float, uint64_t); +RAFT_INST(half, int64_t); } // namespace raft::matrix::detail diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu index 129fc44c81..43b9ce1a6b 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::matrix::detail { diff --git a/cpp/test/matrix/select_k.cu b/cpp/test/matrix/select_k.cu index 344e5b5748..5ef5fbcf34 100644 --- a/cpp/test/matrix/select_k.cu +++ b/cpp/test/matrix/select_k.cu @@ -18,6 +18,10 @@ #include +#if defined RAFT_DISTANCE_COMPILED +#include +#endif + #include #include #include @@ -34,7 +38,7 @@ namespace raft::matrix { template -auto gen_simple_ids(int batch_size, int len) -> std::vector +auto gen_simple_ids(uint32_t batch_size, uint32_t len) -> std::vector { std::vector out(batch_size * len); auto s = rmm::cuda_stream_default; @@ -127,7 +131,7 @@ struct io_computed { update_host(out_dists_.data(), out_dists_d.data(), out_dists_.size(), stream); update_host(out_ids_.data(), out_ids_d.data(), out_ids_.size(), stream); - interruptible::synchronize(stream); + uint32_terruptible::synchronize(stream); auto p = topk_sort_permutation(out_dists_, out_ids_, spec.k, spec.select_min); apply_permutation(out_dists_, p); @@ -147,7 +151,7 @@ struct io_computed { auto topk_sort_permutation(const std::vector& vec, const std::vector& inds, - int k, + uint32_t k, bool select_min) -> std::vector { std::vector p(vec.size()); @@ -228,8 +232,8 @@ struct SelectK // NOLINT auto& in_dists = ref.get_in_dists(); auto compare_ids = [&in_ids, &in_dists](const IdxT& i, const IdxT& j) { if (i == j) return true; - auto ix_i = size_t(std::find(in_ids.begin(), in_ids.end(), i) - in_ids.begin()); - auto ix_j = size_t(std::find(in_ids.begin(), in_ids.end(), j) - in_ids.begin()); + auto ix_i = uint64_t(std::find(in_ids.begin(), in_ids.end(), i) - in_ids.begin()); + auto ix_j = uint64_t(std::find(in_ids.begin(), in_ids.end(), j) - in_ids.begin()); if (ix_i >= in_ids.size() || ix_j >= in_ids.size()) return false; auto dist_i = in_dists[ix_i]; auto dist_j = in_dists[ix_j]; @@ -262,45 +266,45 @@ struct params_simple { }; auto inputs_simple_f = testing::Values( - params_simple::input_t( + params_simple::input_t( {5, 5, 5, true, true}, {5.0, 4.0, 3.0, 2.0, 1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 2.0, 3.0, 5.0, 1.0, 4.0, 5.0, 3.0, 2.0, 4.0, 1.0, 1.0, 3.0, 2.0, 5.0, 4.0}, {1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0}, {4, 3, 2, 1, 0, 0, 1, 2, 3, 4, 3, 0, 1, 4, 2, 4, 2, 1, 3, 0, 0, 2, 1, 4, 3}), - params_simple::input_t( + params_simple::input_t( {5, 5, 3, true, true}, {5.0, 4.0, 3.0, 2.0, 1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 2.0, 3.0, 5.0, 1.0, 4.0, 5.0, 3.0, 2.0, 4.0, 1.0, 1.0, 3.0, 2.0, 5.0, 4.0}, {1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, {4, 3, 2, 0, 1, 2, 3, 0, 1, 4, 2, 1, 0, 2, 1}), - params_simple::input_t( + params_simple::input_t( {5, 5, 5, true, false}, {5.0, 4.0, 3.0, 2.0, 1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 2.0, 3.0, 5.0, 1.0, 4.0, 5.0, 3.0, 2.0, 4.0, 1.0, 1.0, 3.0, 2.0, 5.0, 4.0}, {1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0, 1.0, 2.0, 3.0, 4.0, 5.0}, {4, 3, 2, 1, 0, 0, 1, 2, 3, 4, 3, 0, 1, 4, 2, 4, 2, 1, 3, 0, 0, 2, 1, 4, 3}), - params_simple::input_t( + params_simple::input_t( {5, 5, 3, true, false}, {5.0, 4.0, 3.0, 2.0, 1.0, 1.0, 2.0, 3.0, 4.0, 5.0, 2.0, 3.0, 5.0, 1.0, 4.0, 5.0, 3.0, 2.0, 4.0, 1.0, 1.0, 3.0, 2.0, 5.0, 4.0}, {1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0}, {4, 3, 2, 0, 1, 2, 3, 0, 1, 4, 2, 1, 0, 2, 1}), - params_simple::input_t( + params_simple::input_t( {5, 7, 3, true, true}, {5.0, 4.0, 3.0, 2.0, 1.3, 7.5, 19.0, 9.0, 2.0, 3.0, 3.0, 5.0, 6.0, 4.0, 2.0, 3.0, 5.0, 1.0, 4.0, 1.0, 1.0, 5.0, 7.0, 2.5, 4.0, 7.0, 8.0, 8.0, 1.0, 3.0, 2.0, 5.0, 4.0, 1.1, 1.2}, {1.3, 2.0, 3.0, 2.0, 3.0, 3.0, 1.0, 1.0, 1.0, 2.5, 4.0, 5.0, 1.0, 1.1, 1.2}, {4, 3, 2, 1, 2, 3, 3, 5, 6, 2, 3, 0, 0, 5, 6}), - params_simple::input_t( + params_simple::input_t( {1, 7, 3, true, true}, {2.0, 3.0, 5.0, 1.0, 4.0, 1.0, 1.0}, {1.0, 1.0, 1.0}, {3, 5, 6}), - params_simple::input_t( + params_simple::input_t( {1, 7, 3, false, false}, {2.0, 3.0, 5.0, 1.0, 4.0, 1.0, 1.0}, {5.0, 4.0, 3.0}, {2, 4, 1}), - params_simple::input_t( + params_simple::input_t( {1, 7, 3, false, true}, {2.0, 3.0, 5.0, 9.0, 4.0, 9.0, 9.0}, {9.0, 9.0, 9.0}, {3, 5, 6}), - params_simple::input_t( + params_simple::input_t( {1, 130, 5, false, true}, {19, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, @@ -309,7 +313,7 @@ auto inputs_simple_f = testing::Values( 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 4, 4, 2, 3, 2, 3, 2, 3, 2, 3, 2, 20}, {20, 19, 18, 17, 16}, {129, 0, 117, 116, 115}), - params_simple::input_t( + params_simple::input_t( {1, 130, 15, false, true}, {19, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, @@ -319,7 +323,7 @@ auto inputs_simple_f = testing::Values( {20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6}, {129, 0, 117, 116, 115, 114, 113, 112, 111, 110, 109, 108, 107, 106, 105})); -using SimpleFloatInt = SelectK; +using SimpleFloatInt = SelectK; TEST_P(SimpleFloatInt, Run) { run(); } // NOLINT INSTANTIATE_TEST_CASE_P( // NOLINT SelectK, @@ -414,7 +418,7 @@ auto inputs_random_largek = testing::Values(select::params{100, 100000, 1000, tr select::params{100, 100000, 1237, true}); using ReferencedRandomFloatInt = - SelectK::params_random>; + SelectK::params_random>; TEST_P(ReferencedRandomFloatInt, Run) { run(); } // NOLINT INSTANTIATE_TEST_CASE_P( // NOLINT SelectK, @@ -428,7 +432,7 @@ INSTANTIATE_TEST_CASE_P( // NOLINT select::Algo::kWarpDistributedShm))); using ReferencedRandomDoubleSizeT = - SelectK::params_random>; + SelectK::params_random>; TEST_P(ReferencedRandomDoubleSizeT, Run) { run(); } // NOLINT INSTANTIATE_TEST_CASE_P( // NOLINT SelectK, @@ -442,7 +446,7 @@ INSTANTIATE_TEST_CASE_P( // NOLINT select::Algo::kWarpDistributedShm))); using ReferencedRandomDoubleInt = - SelectK::params_random>; + SelectK::params_random>; TEST_P(ReferencedRandomDoubleInt, LargeSize) { run(); } // NOLINT INSTANTIATE_TEST_CASE_P( // NOLINT SelectK, @@ -450,7 +454,7 @@ INSTANTIATE_TEST_CASE_P( // NOLINT testing::Combine(inputs_random_largesize, testing::Values(select::Algo::kWarpAuto))); using ReferencedRandomFloatSizeT = - SelectK::params_random>; + SelectK::params_random>; TEST_P(ReferencedRandomFloatSizeT, LargeK) { run(); } // NOLINT INSTANTIATE_TEST_CASE_P(SelectK, // NOLINT ReferencedRandomFloatSizeT, From 54b15297cbe3949f44a529421a31ef6e3317cd0b Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 10 Mar 2023 13:41:42 -0500 Subject: [PATCH 2/6] Removing specializations headers from specializations source files --- .../detail/compute_similarity_float_float_fast.cu | 2 +- .../detail/compute_similarity_float_float_no_basediff.cu | 2 +- .../detail/compute_similarity_float_float_no_smem_lut.cu | 2 +- .../detail/compute_similarity_float_fp8s_fast.cu | 2 +- .../detail/compute_similarity_float_fp8s_no_basediff.cu | 2 +- .../detail/compute_similarity_float_fp8s_no_smem_lut.cu | 2 +- .../detail/compute_similarity_float_fp8u_fast.cu | 2 +- .../detail/compute_similarity_float_fp8u_no_basediff.cu | 2 +- .../detail/compute_similarity_float_fp8u_no_smem_lut.cu | 2 +- .../detail/compute_similarity_float_half_fast.cu | 2 +- .../detail/compute_similarity_float_half_no_basediff.cu | 2 +- .../detail/compute_similarity_float_half_no_smem_lut.cu | 2 +- .../specializations/detail/compute_similarity_half_fp8s_fast.cu | 2 +- .../detail/compute_similarity_half_fp8s_no_basediff.cu | 2 +- .../detail/compute_similarity_half_fp8s_no_smem_lut.cu | 2 +- .../specializations/detail/compute_similarity_half_fp8u_fast.cu | 2 +- .../detail/compute_similarity_half_fp8u_no_basediff.cu | 2 +- .../detail/compute_similarity_half_fp8u_no_smem_lut.cu | 2 +- .../specializations/detail/compute_similarity_half_half_fast.cu | 2 +- .../detail/compute_similarity_half_half_no_basediff.cu | 2 +- .../detail/compute_similarity_half_half_no_smem_lut.cu | 2 +- .../detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu | 2 +- .../neighbors/specializations/ivfpq_build_float_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_build_int8_t_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_extend_float_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_search_float_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_search_int8_t_int64_t.cu | 2 +- .../neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu index 579eee8378..a12a5b0b7d 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu index af42b2370b..a91318ca34 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu index 7d3bd75b4f..1348d52ceb 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu index 4caf2b55f6..65ed409e9a 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu index be24c22c68..a7c8b3b7e0 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu index 7c19a6ebb2..b9af5c00a5 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu index 5d82a9ebff..74364a5480 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu index 4bfea60ce9..75f03c2df4 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu index 2cd5323c8a..0c5404d5ea 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu index 4cb1cebc41..9eee24815b 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu index f98fbbcf05..e066459169 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu index 9722e046b3..e0cec88955 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu index be3116307f..9a1ae65cb2 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu index 4aab40ced9..6d4662e8c4 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu index e6f8f940ce..be8e53ca62 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu index 64f8f64ee8..b01b5c9139 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu index 063596c48d..dcc4486a2f 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu index 64fab5349c..190b6eb6a3 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu index 2b6d78013e..3cdc75b66b 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu index 246a55b89e..fb40962a24 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu index 8d3e478a96..088c94d4ad 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu index 9722e046b3..e0cec88955 100644 --- a/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu index b1ebf65c6a..50ed302595 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu index 8c6adcf7d5..99860b3200 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu index 19b6ea3705..61e078e466 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu index b416589520..84571f0a01 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu index d4907b5c60..ded43bf9c2 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu index 64a206b54d..643f5d45a5 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu index f530961c0d..615316ab00 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu index 249979763a..86e045315e 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu index b63ac43a6b..543f7a28b6 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu @@ -14,7 +14,7 @@ * limitations under the License. */ -#include +#include namespace raft::neighbors::ivf_pq { From 137a719f63b6269d08ff37287533a63a721e5d26 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 10 Mar 2023 17:17:14 -0500 Subject: [PATCH 3/6] MOre fixes --- cpp/internal/raft_internal/matrix/select_k.cuh | 4 ++-- cpp/src/distance/neighbors/ivfpq_deserialize.cu | 1 + cpp/src/distance/neighbors/ivfpq_serialize.cu | 1 + cpp/test/matrix/select_k.cu | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/internal/raft_internal/matrix/select_k.cuh b/cpp/internal/raft_internal/matrix/select_k.cuh index 398c6c8d1b..59cbff9dfb 100644 --- a/cpp/internal/raft_internal/matrix/select_k.cuh +++ b/cpp/internal/raft_internal/matrix/select_k.cuh @@ -20,8 +20,8 @@ #include #include -#if defined RAFT_DISTANCE_COMPILED -#include +#ifdef RAFT_DISTANCE_COMPILED +#include #endif #include diff --git a/cpp/src/distance/neighbors/ivfpq_deserialize.cu b/cpp/src/distance/neighbors/ivfpq_deserialize.cu index e6b9a2176e..7663223ae0 100644 --- a/cpp/src/distance/neighbors/ivfpq_deserialize.cu +++ b/cpp/src/distance/neighbors/ivfpq_deserialize.cu @@ -15,6 +15,7 @@ */ #include +#include #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_serialize.cu b/cpp/src/distance/neighbors/ivfpq_serialize.cu index 711240ec04..60fdf49fcf 100644 --- a/cpp/src/distance/neighbors/ivfpq_serialize.cu +++ b/cpp/src/distance/neighbors/ivfpq_serialize.cu @@ -15,6 +15,7 @@ */ #include +#include #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/test/matrix/select_k.cu b/cpp/test/matrix/select_k.cu index 5ef5fbcf34..a9fc4c8f40 100644 --- a/cpp/test/matrix/select_k.cu +++ b/cpp/test/matrix/select_k.cu @@ -18,7 +18,7 @@ #include -#if defined RAFT_DISTANCE_COMPILED +#ifdef RAFT_DISTANCE_COMPILED #include #endif @@ -131,7 +131,7 @@ struct io_computed { update_host(out_dists_.data(), out_dists_d.data(), out_dists_.size(), stream); update_host(out_ids_.data(), out_ids_d.data(), out_ids_.size(), stream); - uint32_terruptible::synchronize(stream); + interruptible::synchronize(stream); auto p = topk_sort_permutation(out_dists_, out_ids_, spec.k, spec.select_min); apply_permutation(out_dists_, p); From 22f7e2c611d80e202efef1d59e8428045c1e98ca Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 10 Mar 2023 20:35:00 -0500 Subject: [PATCH 4/6] Lots of cleanup. Adding some docs around using the pre-compiled template specializations. --- build.sh | 2 + cpp/bench/CMakeLists.txt | 2 +- cpp/bench/cluster/kmeans.cu | 4 +- cpp/bench/cluster/kmeans_balanced.cu | 4 +- cpp/bench/distance/masked_nn.cu | 4 +- cpp/bench/neighbors/knn.cuh | 6 +- cpp/bench/neighbors/refine_float_int64_t.cu | 1 - cpp/include/raft/cluster/specializations.cuh | 4 +- .../raft/neighbors/specializations.cuh | 11 ++-- .../neighbors/specializations/ball_cover.cuh | 1 + .../raft/neighbors/specializations/ivf_pq.cuh | 4 +- .../distance/cluster/cluster_cost_double.cu | 2 +- .../distance/cluster/cluster_cost_float.cu | 2 +- cpp/src/distance/cluster/kmeans_fit_double.cu | 2 +- cpp/src/distance/cluster/kmeans_fit_float.cu | 2 +- .../cluster/kmeans_init_plus_plus_double.cu | 2 +- .../cluster/kmeans_init_plus_plus_float.cu | 2 +- cpp/src/distance/cluster/update_centroids.cuh | 2 +- .../cluster/update_centroids_double.cu | 2 +- .../cluster/update_centroids_float.cu | 2 +- .../canberra_double_double_double_int.cu | 1 + .../detail/canberra_float_float_float_int.cu | 1 + .../correlation_double_double_double_int.cu | 1 + .../correlation_float_float_float_int.cu | 1 + .../detail/cosine_double_double_double_int.cu | 1 + .../detail/cosine_float_float_float_int.cu | 1 + ...ing_unexpanded_double_double_double_int.cu | 1 + ...amming_unexpanded_float_float_float_int.cu | 1 + ...inger_expanded_double_double_double_int.cu | 1 + ...ellinger_expanded_float_float_float_int.cu | 1 + .../inner_product_double_double_double_int.cu | 1 + .../inner_product_float_float_float_int.cu | 1 + ...jensen_shannon_double_double_double_int.cu | 1 + .../jensen_shannon_float_float_float_int.cu | 1 + .../detail/kernels/gram_matrix_base_double.cu | 3 +- .../detail/kernels/gram_matrix_base_float.cu | 3 +- .../kernels/polynomial_kernel_double_int.cu | 3 +- .../kernels/polynomial_kernel_float_int.cu | 3 +- .../detail/kernels/rbf_kernel_double.cu | 3 +- .../detail/kernels/rbf_kernel_float.cu | 3 +- .../detail/kernels/tanh_kernel_double.cu | 3 +- .../detail/kernels/tanh_kernel_float.cu | 3 +- .../kl_divergence_double_double_double_int.cu | 1 + .../kl_divergence_float_float_float_int.cu | 1 + .../detail/l1_double_double_double_int.cu | 1 + .../detail/l1_float_float_float_int.cu | 1 + .../l2_expanded_double_double_double_int.cu | 1 + .../l2_expanded_float_float_float_int.cu | 1 + ..._sqrt_expanded_double_double_double_int.cu | 1 + .../l2_sqrt_expanded_float_float_float_int.cu | 1 + ...qrt_unexpanded_double_double_double_int.cu | 1 + ...2_sqrt_unexpanded_float_float_float_int.cu | 1 + .../l2_unexpanded_double_double_double_int.cu | 1 + .../l2_unexpanded_float_float_float_int.cu | 1 + .../detail/l_inf_double_double_double_int.cu | 1 + .../detail/l_inf_float_float_float_int.cu | 1 + .../lp_unexpanded_double_double_double_int.cu | 1 + .../lp_unexpanded_float_float_float_int.cu | 1 + .../russel_rao_double_double_double_int.cu | 1 + .../russel_rao_float_float_float_int.cu | 1 + .../specializations/fused_l2_nn_double_int.cu | 3 +- .../fused_l2_nn_double_int64.cu | 3 +- .../specializations/fused_l2_nn_float_int.cu | 3 +- .../fused_l2_nn_float_int64.cu | 3 +- .../detail/select_k_float_int64_t.cu | 1 + .../detail/select_k_float_uint32_t.cu | 1 + .../detail/select_k_half_int64_t.cu | 1 + .../detail/select_k_half_uint32_t.cu | 1 + cpp/src/distance/neighbors/ivfpq_build.cu | 2 +- .../distance/neighbors/ivfpq_deserialize.cu | 3 +- .../neighbors/ivfpq_search_float_int64_t.cu | 3 +- .../neighbors/ivfpq_search_int8_t_int64_t.cu | 3 +- .../neighbors/ivfpq_search_uint8_t_int64_t.cu | 3 +- cpp/src/distance/neighbors/ivfpq_serialize.cu | 3 +- .../neighbors/refine_d_int64_t_float.cu | 2 +- .../neighbors/refine_d_int64_t_int8_t.cu | 2 +- .../neighbors/refine_d_int64_t_uint8_t.cu | 2 +- .../neighbors/refine_h_int64_t_float.cu | 2 +- .../neighbors/refine_h_int64_t_int8_t.cu | 2 +- .../neighbors/refine_h_int64_t_uint8_t.cu | 2 +- .../compute_similarity_float_float_fast.cu | 4 +- ...pute_similarity_float_float_no_basediff.cu | 1 + ...pute_similarity_float_float_no_smem_lut.cu | 1 + .../compute_similarity_float_fp8s_fast.cu | 1 + ...mpute_similarity_float_fp8s_no_basediff.cu | 1 + ...mpute_similarity_float_fp8s_no_smem_lut.cu | 1 + .../compute_similarity_float_fp8u_fast.cu | 1 + ...mpute_similarity_float_fp8u_no_basediff.cu | 1 + ...mpute_similarity_float_fp8u_no_smem_lut.cu | 1 + .../compute_similarity_float_half_fast.cu | 1 + ...mpute_similarity_float_half_no_basediff.cu | 1 + ...mpute_similarity_float_half_no_smem_lut.cu | 1 + .../compute_similarity_half_fp8s_fast.cu | 1 + ...ompute_similarity_half_fp8s_no_basediff.cu | 1 + ...ompute_similarity_half_fp8s_no_smem_lut.cu | 1 + .../compute_similarity_half_fp8u_fast.cu | 1 + ...ompute_similarity_half_fp8u_no_basediff.cu | 1 + ...ompute_similarity_half_fp8u_no_smem_lut.cu | 1 + .../compute_similarity_half_half_fast.cu | 1 + ...ompute_similarity_half_half_no_basediff.cu | 1 + ...ompute_similarity_half_half_no_smem_lut.cu | 1 + ...mpute_similarity_float_half_no_smem_lut.cu | 1 + .../ivfpq_build_float_int64_t.cu | 1 + .../ivfpq_build_int8_t_int64_t.cu | 1 + .../ivfpq_build_uint8_t_int64_t.cu | 1 + .../ivfpq_extend_float_int64_t.cu | 1 + .../ivfpq_extend_int8_t_int64_t.cu | 1 + .../ivfpq_extend_uint8_t_int64_t.cu | 1 + .../ivfpq_search_float_int64_t.cu | 1 + .../ivfpq_search_int8_t_int64_t.cu | 1 + .../ivfpq_search_uint8_t_int64_t.cu | 1 + .../specializations/refine_d_int64_t_float.cu | 1 + .../refine_d_int64_t_int8_t.cu | 1 + .../refine_d_int64_t_uint8_t.cu | 1 + .../specializations/refine_h_int64_t_float.cu | 1 + .../refine_h_int64_t_int8_t.cu | 1 + .../refine_h_int64_t_uint8_t.cu | 1 + .../ball_cover_all_knn_query.cu | 5 +- .../specializations/ball_cover_build_index.cu | 5 +- .../specializations/ball_cover_knn_query.cu | 5 +- .../brute_force_knn_long_float_int.cu | 3 + .../brute_force_knn_long_float_uint.cu | 3 + .../brute_force_knn_uint32_t_float_int.cu | 3 + .../brute_force_knn_uint32_t_float_uint.cu | 3 + .../detail/ball_cover_lowdim_pass_one_2d.cu | 4 +- .../detail/ball_cover_lowdim_pass_one_3d.cu | 4 +- .../detail/ball_cover_lowdim_pass_two_2d.cu | 4 +- .../detail/ball_cover_lowdim_pass_two_3d.cu | 4 +- .../fused_l2_knn_int_float_false.cu | 5 +- .../fused_l2_knn_int_float_true.cu | 5 +- .../fused_l2_knn_long_float_false.cu | 5 +- .../fused_l2_knn_long_float_true.cu | 5 +- cpp/test/CMakeLists.txt | 2 + cpp/test/cluster/kmeans.cu | 2 +- cpp/test/cluster/kmeans_balanced.cu | 4 +- cpp/test/cluster/kmeans_find_k.cu | 2 +- cpp/test/distance/masked_nn.cu | 4 ++ cpp/test/neighbors/ann_ivf_flat.cuh | 5 +- .../ann_ivf_flat/test_float_int64_t.cu | 3 +- .../ann_ivf_flat/test_int8_t_int64_t.cu | 3 +- .../ann_ivf_flat/test_uint8_t_int64_t.cu | 3 +- cpp/test/neighbors/ann_ivf_pq.cuh | 4 +- cpp/test/neighbors/ball_cover.cu | 2 +- cpp/test/neighbors/epsilon_neighborhood.cu | 4 ++ cpp/test/neighbors/fused_l2_knn.cu | 2 +- cpp/test/neighbors/knn.cu | 2 +- cpp/test/neighbors/refine.cu | 2 +- cpp/test/neighbors/selection.cu | 3 - docs/source/index.rst | 1 + docs/source/using_libraft.md | 59 +++++++++++++++++++ 150 files changed, 277 insertions(+), 95 deletions(-) create mode 100644 docs/source/using_libraft.md diff --git a/build.sh b/build.sh index 8dc9ce3881..575f6bdaa1 100755 --- a/build.sh +++ b/build.sh @@ -308,6 +308,7 @@ if hasArg tests || (( ${NUMARGS} == 0 )); then $CMAKE_TARGET == *"DISTANCE_TEST"* || \ $CMAKE_TARGET == *"SPARSE_DIST_TEST" || \ $CMAKE_TARGET == *"SPARSE_NEIGHBORS_TEST"* || \ + $CMAKE_TARGET == *"MATRIX_TEST"* || \ $CMAKE_TARGET == *"NEIGHBORS_TEST" || \ $CMAKE_TARGET == *"STATS_TEST"* ]]; then echo "-- Enabling distance lib for gtests" @@ -329,6 +330,7 @@ if hasArg bench || (( ${NUMARGS} == 0 )); then # Force compile distance library when needed benchmark targets are specified if [[ $CMAKE_TARGET == *"CLUSTER_BENCH"* || \ + $CMAKE_TARGET == *"MATRIX_BENCH"* || \ $CMAKE_TARGET == *"NEIGHBORS_BENCH"* ]]; then echo "-- Enabling distance lib for benchmarks" COMPILE_DIST_LIBRARY=ON diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt index 68020e4ed3..e2324de654 100644 --- a/cpp/bench/CMakeLists.txt +++ b/cpp/bench/CMakeLists.txt @@ -106,7 +106,7 @@ if(BUILD_BENCH) ConfigureBench( NAME MATRIX_BENCH PATH bench/matrix/argmin.cu bench/matrix/gather.cu bench/matrix/select_k.cu - bench/main.cpp + bench/main.cpp OPTIONAL DIST ) ConfigureBench( diff --git a/cpp/bench/cluster/kmeans.cu b/cpp/bench/cluster/kmeans.cu index 76b16cfe56..7079e33f25 100644 --- a/cpp/bench/cluster/kmeans.cu +++ b/cpp/bench/cluster/kmeans.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include +#include #endif namespace raft::bench::cluster { diff --git a/cpp/bench/cluster/kmeans_balanced.cu b/cpp/bench/cluster/kmeans_balanced.cu index 9c53e86d8c..1cfdfbe49a 100644 --- a/cpp/bench/cluster/kmeans_balanced.cu +++ b/cpp/bench/cluster/kmeans_balanced.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include +#include #endif namespace raft::bench::cluster { diff --git a/cpp/bench/distance/masked_nn.cu b/cpp/bench/distance/masked_nn.cu index 30ad3f7168..1fecb455c3 100644 --- a/cpp/bench/distance/masked_nn.cu +++ b/cpp/bench/distance/masked_nn.cu @@ -30,8 +30,8 @@ #include #include -#if defined RAFT_NN_COMPILED -#include +#ifdef RAFT_DISTANCE_COMPILED +#include #endif namespace raft::bench::distance::masked_nn { diff --git a/cpp/bench/neighbors/knn.cuh b/cpp/bench/neighbors/knn.cuh index ed3c6db909..0a59da3970 100644 --- a/cpp/bench/neighbors/knn.cuh +++ b/cpp/bench/neighbors/knn.cuh @@ -25,8 +25,10 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include -#include +#include + +// TODO: Legacy. Remove when FAISS is removed +#include #endif #if defined RAFT_NN_COMPILED diff --git a/cpp/bench/neighbors/refine_float_int64_t.cu b/cpp/bench/neighbors/refine_float_int64_t.cu index 1ff59b5d4d..40ab2bc0ca 100644 --- a/cpp/bench/neighbors/refine_float_int64_t.cu +++ b/cpp/bench/neighbors/refine_float_int64_t.cu @@ -18,7 +18,6 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include #include #endif diff --git a/cpp/include/raft/cluster/specializations.cuh b/cpp/include/raft/cluster/specializations.cuh index 3bb5a26ace..9b68d7adc9 100644 --- a/cpp/include/raft/cluster/specializations.cuh +++ b/cpp/include/raft/cluster/specializations.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,6 @@ #pragma once #include -#include +#include #endif \ No newline at end of file diff --git a/cpp/include/raft/neighbors/specializations.cuh b/cpp/include/raft/neighbors/specializations.cuh index 8652c21e4f..f9bdda4e49 100644 --- a/cpp/include/raft/neighbors/specializations.cuh +++ b/cpp/include/raft/neighbors/specializations.cuh @@ -14,14 +14,11 @@ * limitations under the License. */ -#ifndef __KNN_SPECIALIZATIONS_H -#define __KNN_SPECIALIZATIONS_H - #pragma once -#include -#include #include -#include +#include -#endif +#include +#include +#include \ No newline at end of file diff --git a/cpp/include/raft/neighbors/specializations/ball_cover.cuh b/cpp/include/raft/neighbors/specializations/ball_cover.cuh index d44cb7064f..d6a6b2e296 100644 --- a/cpp/include/raft/neighbors/specializations/ball_cover.cuh +++ b/cpp/include/raft/neighbors/specializations/ball_cover.cuh @@ -18,6 +18,7 @@ #include #include +#include #include diff --git a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh b/cpp/include/raft/neighbors/specializations/ivf_pq.cuh index 142cc7806f..a651be90db 100644 --- a/cpp/include/raft/neighbors/specializations/ivf_pq.cuh +++ b/cpp/include/raft/neighbors/specializations/ivf_pq.cuh @@ -16,7 +16,9 @@ #pragma once -#include +#include +#include +#include #include #include diff --git a/cpp/src/distance/cluster/cluster_cost_double.cu b/cpp/src/distance/cluster/cluster_cost_double.cu index 6feb3076dd..2244ba4ed3 100644 --- a/cpp/src/distance/cluster/cluster_cost_double.cu +++ b/cpp/src/distance/cluster/cluster_cost_double.cu @@ -15,9 +15,9 @@ */ #include "cluster_cost.cuh" +#include #include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/cluster_cost_float.cu b/cpp/src/distance/cluster/cluster_cost_float.cu index 24af5efb25..4164265b55 100644 --- a/cpp/src/distance/cluster/cluster_cost_float.cu +++ b/cpp/src/distance/cluster/cluster_cost_float.cu @@ -15,9 +15,9 @@ */ #include "cluster_cost.cuh" +#include #include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/kmeans_fit_double.cu b/cpp/src/distance/cluster/kmeans_fit_double.cu index cbc9fa45cb..12f4fba318 100644 --- a/cpp/src/distance/cluster/kmeans_fit_double.cu +++ b/cpp/src/distance/cluster/kmeans_fit_double.cu @@ -15,8 +15,8 @@ */ #include +#include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/kmeans_fit_float.cu b/cpp/src/distance/cluster/kmeans_fit_float.cu index 6dcbd73b8d..48505dcc3e 100644 --- a/cpp/src/distance/cluster/kmeans_fit_float.cu +++ b/cpp/src/distance/cluster/kmeans_fit_float.cu @@ -15,8 +15,8 @@ */ #include +#include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/kmeans_init_plus_plus_double.cu b/cpp/src/distance/cluster/kmeans_init_plus_plus_double.cu index 53132e13e7..5bb0835595 100644 --- a/cpp/src/distance/cluster/kmeans_init_plus_plus_double.cu +++ b/cpp/src/distance/cluster/kmeans_init_plus_plus_double.cu @@ -15,8 +15,8 @@ */ #include +#include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/kmeans_init_plus_plus_float.cu b/cpp/src/distance/cluster/kmeans_init_plus_plus_float.cu index 814b0b41b8..f211afd06e 100644 --- a/cpp/src/distance/cluster/kmeans_init_plus_plus_float.cu +++ b/cpp/src/distance/cluster/kmeans_init_plus_plus_float.cu @@ -15,8 +15,8 @@ */ #include +#include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/update_centroids.cuh b/cpp/src/distance/cluster/update_centroids.cuh index e0d3bd8487..7c13252384 100644 --- a/cpp/src/distance/cluster/update_centroids.cuh +++ b/cpp/src/distance/cluster/update_centroids.cuh @@ -15,9 +15,9 @@ */ #include +#include #include #include -#include #include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/update_centroids_double.cu b/cpp/src/distance/cluster/update_centroids_double.cu index cb63de2ca5..0f38c7dd53 100644 --- a/cpp/src/distance/cluster/update_centroids_double.cu +++ b/cpp/src/distance/cluster/update_centroids_double.cu @@ -15,9 +15,9 @@ */ #include "update_centroids.cuh" +#include #include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/cluster/update_centroids_float.cu b/cpp/src/distance/cluster/update_centroids_float.cu index 7ce74b584c..8f0e79b438 100644 --- a/cpp/src/distance/cluster/update_centroids_float.cu +++ b/cpp/src/distance/cluster/update_centroids_float.cu @@ -15,9 +15,9 @@ */ #include "update_centroids.cuh" +#include #include #include -#include namespace raft::runtime::cluster::kmeans { diff --git a/cpp/src/distance/distance/specializations/detail/canberra_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/canberra_double_double_double_int.cu index d3a629f5a0..4e9e608792 100644 --- a/cpp/src/distance/distance/specializations/detail/canberra_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/canberra_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/canberra_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/canberra_float_float_float_int.cu index 71899c00a7..6dfc385e55 100644 --- a/cpp/src/distance/distance/specializations/detail/canberra_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/canberra_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/correlation_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/correlation_double_double_double_int.cu index 4c626a0c73..2df77a4b5d 100644 --- a/cpp/src/distance/distance/specializations/detail/correlation_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/correlation_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/correlation_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/correlation_float_float_float_int.cu index 5f23a5042e..76ed00afa6 100644 --- a/cpp/src/distance/distance/specializations/detail/correlation_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/correlation_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/cosine_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/cosine_double_double_double_int.cu index b7f2eb569d..3e0bcb92ed 100644 --- a/cpp/src/distance/distance/specializations/detail/cosine_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/cosine_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/cosine_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/cosine_float_float_float_int.cu index 650c889a0d..23131ce2c7 100644 --- a/cpp/src/distance/distance/specializations/detail/cosine_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/cosine_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_double_double_double_int.cu index 282449f48b..b618fd024c 100644 --- a/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_float_float_float_int.cu index 2bed6c2401..18e7aad9e9 100644 --- a/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/hamming_unexpanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/hellinger_expanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/hellinger_expanded_double_double_double_int.cu index b14425eec3..08ab20cfe5 100644 --- a/cpp/src/distance/distance/specializations/detail/hellinger_expanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/hellinger_expanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/hellinger_expanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/hellinger_expanded_float_float_float_int.cu index b738b3983b..79eed075fb 100644 --- a/cpp/src/distance/distance/specializations/detail/hellinger_expanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/hellinger_expanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/inner_product_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/inner_product_double_double_double_int.cu index 41bd6c2c14..3db0a3572e 100644 --- a/cpp/src/distance/distance/specializations/detail/inner_product_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/inner_product_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/inner_product_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/inner_product_float_float_float_int.cu index f147c51b52..2b06ca4dc2 100644 --- a/cpp/src/distance/distance/specializations/detail/inner_product_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/inner_product_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/jensen_shannon_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/jensen_shannon_double_double_double_int.cu index c490033dc3..ed84ee6dc4 100644 --- a/cpp/src/distance/distance/specializations/detail/jensen_shannon_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/jensen_shannon_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/jensen_shannon_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/jensen_shannon_float_float_float_int.cu index ae1d860b3e..a241af767c 100644 --- a/cpp/src/distance/distance/specializations/detail/jensen_shannon_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/jensen_shannon_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_double.cu b/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_double.cu index c893e9a358..7c80eb29d0 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_double.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::GramMatrixBase; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_float.cu b/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_float.cu index 3265f828e6..d777e73dc9 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_float.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/gram_matrix_base_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::GramMatrixBase; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu b/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu index 0edf45a6f1..28306d0c21 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_double_int.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::PolynomialKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu b/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu index a719175e6b..6609de69ac 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/polynomial_kernel_float_int.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::PolynomialKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_double.cu b/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_double.cu index 6577e1b6c7..7ea4b60e09 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_double.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::RBFKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_float.cu b/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_float.cu index 1d2582cf81..423613dcd1 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_float.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/rbf_kernel_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::RBFKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_double.cu b/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_double.cu index 13d5159504..ab818db73b 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_double.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_double.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::TanhKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_float.cu b/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_float.cu index ee62de7d34..f7825e577a 100644 --- a/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_float.cu +++ b/cpp/src/distance/distance/specializations/detail/kernels/tanh_kernel_float.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,6 @@ */ #include +#include template class raft::distance::kernels::detail::TanhKernel; \ No newline at end of file diff --git a/cpp/src/distance/distance/specializations/detail/kl_divergence_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/kl_divergence_double_double_double_int.cu index ed2a29bccf..c4c944d123 100644 --- a/cpp/src/distance/distance/specializations/detail/kl_divergence_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/kl_divergence_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/kl_divergence_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/kl_divergence_float_float_float_int.cu index 511e0fb1a6..aa1db5a837 100644 --- a/cpp/src/distance/distance/specializations/detail/kl_divergence_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/kl_divergence_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l1_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l1_double_double_double_int.cu index 7a1c884223..391a1c2aa4 100644 --- a/cpp/src/distance/distance/specializations/detail/l1_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l1_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l1_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l1_float_float_float_int.cu index f307ade06f..7b45e52ca1 100644 --- a/cpp/src/distance/distance/specializations/detail/l1_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l1_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_expanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l2_expanded_double_double_double_int.cu index 18ddf10b14..8c5f746fa2 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_expanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_expanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_expanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l2_expanded_float_float_float_int.cu index 28e7b73243..c266125f98 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_expanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_expanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu index 404dc5f8b4..399b120527 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu index b4f8757d4a..66de212b8e 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_sqrt_expanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu index 10cafac4a7..562d93b2de 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu index 6a5f43bc46..386bbafc5f 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_sqrt_unexpanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu index 174a893d36..7733c3af48 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_unexpanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu index 0cc43b099f..4ea18d31de 100644 --- a/cpp/src/distance/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l2_unexpanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l_inf_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/l_inf_double_double_double_int.cu index c071ddc902..74414f8fd6 100644 --- a/cpp/src/distance/distance/specializations/detail/l_inf_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l_inf_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/l_inf_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/l_inf_float_float_float_int.cu index d42d769306..e418fc455f 100644 --- a/cpp/src/distance/distance/specializations/detail/l_inf_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/l_inf_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu index 66e765b1c0..402cb51b7e 100644 --- a/cpp/src/distance/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/lp_unexpanded_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu index 6fbd52ac89..7efe2b3349 100644 --- a/cpp/src/distance/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/lp_unexpanded_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/russel_rao_double_double_double_int.cu b/cpp/src/distance/distance/specializations/detail/russel_rao_double_double_double_int.cu index 77e1fc70d0..b1e6f5e1f4 100644 --- a/cpp/src/distance/distance/specializations/detail/russel_rao_double_double_double_int.cu +++ b/cpp/src/distance/distance/specializations/detail/russel_rao_double_double_double_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/detail/russel_rao_float_float_float_int.cu b/cpp/src/distance/distance/specializations/detail/russel_rao_float_float_float_int.cu index 6f898310c7..1e12bcd705 100644 --- a/cpp/src/distance/distance/specializations/detail/russel_rao_float_float_float_int.cu +++ b/cpp/src/distance/distance/specializations/detail/russel_rao_float_float_float_int.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/fused_l2_nn_double_int.cu b/cpp/src/distance/distance/specializations/fused_l2_nn_double_int.cu index 4448ee0cc2..b49132b042 100644 --- a/cpp/src/distance/distance/specializations/fused_l2_nn_double_int.cu +++ b/cpp/src/distance/distance/specializations/fused_l2_nn_double_int.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/fused_l2_nn_double_int64.cu b/cpp/src/distance/distance/specializations/fused_l2_nn_double_int64.cu index 54478a1656..b1e3a900a9 100644 --- a/cpp/src/distance/distance/specializations/fused_l2_nn_double_int64.cu +++ b/cpp/src/distance/distance/specializations/fused_l2_nn_double_int64.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/fused_l2_nn_float_int.cu b/cpp/src/distance/distance/specializations/fused_l2_nn_float_int.cu index e25c9fad91..44b4953d8c 100644 --- a/cpp/src/distance/distance/specializations/fused_l2_nn_float_int.cu +++ b/cpp/src/distance/distance/specializations/fused_l2_nn_float_int.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/distance/specializations/fused_l2_nn_float_int64.cu b/cpp/src/distance/distance/specializations/fused_l2_nn_float_int64.cu index b7abd91304..9ca2b639a9 100644 --- a/cpp/src/distance/distance/specializations/fused_l2_nn_float_int64.cu +++ b/cpp/src/distance/distance/specializations/fused_l2_nn_float_int64.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ #include #include +#include namespace raft { namespace distance { diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu index 7a6c7af96d..370ab1ba50 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_float_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::matrix::detail { diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu index 15e943df24..c6733c2a46 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_float_uint32_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::matrix::detail { diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu index 4d735c5b89..38e28ac54d 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_half_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::matrix::detail { diff --git a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu b/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu index 43b9ce1a6b..108bd30b49 100644 --- a/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu +++ b/cpp/src/distance/matrix/specializations/detail/select_k_half_uint32_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::matrix::detail { diff --git a/cpp/src/distance/neighbors/ivfpq_build.cu b/cpp/src/distance/neighbors/ivfpq_build.cu index bfc893dbd3..4a4d16cbac 100644 --- a/cpp/src/distance/neighbors/ivfpq_build.cu +++ b/cpp/src/distance/neighbors/ivfpq_build.cu @@ -15,7 +15,7 @@ */ #include -#include +#include #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_deserialize.cu b/cpp/src/distance/neighbors/ivfpq_deserialize.cu index 7663223ae0..8d54e3cc55 100644 --- a/cpp/src/distance/neighbors/ivfpq_deserialize.cu +++ b/cpp/src/distance/neighbors/ivfpq_deserialize.cu @@ -15,7 +15,8 @@ */ #include -#include +#include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_search_float_int64_t.cu b/cpp/src/distance/neighbors/ivfpq_search_float_int64_t.cu index c38e27f196..47dbc48e44 100644 --- a/cpp/src/distance/neighbors/ivfpq_search_float_int64_t.cu +++ b/cpp/src/distance/neighbors/ivfpq_search_float_int64_t.cu @@ -15,7 +15,8 @@ */ #include -#include +#include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_search_int8_t_int64_t.cu b/cpp/src/distance/neighbors/ivfpq_search_int8_t_int64_t.cu index 5df3f15eb6..45218b215c 100644 --- a/cpp/src/distance/neighbors/ivfpq_search_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/ivfpq_search_int8_t_int64_t.cu @@ -15,7 +15,8 @@ */ #include -#include +#include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/ivfpq_search_uint8_t_int64_t.cu index 0293d05246..b7f028002f 100644 --- a/cpp/src/distance/neighbors/ivfpq_search_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/ivfpq_search_uint8_t_int64_t.cu @@ -15,7 +15,8 @@ */ #include -#include +#include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/ivfpq_serialize.cu b/cpp/src/distance/neighbors/ivfpq_serialize.cu index 60fdf49fcf..e251f1442f 100644 --- a/cpp/src/distance/neighbors/ivfpq_serialize.cu +++ b/cpp/src/distance/neighbors/ivfpq_serialize.cu @@ -15,7 +15,8 @@ */ #include -#include +#include + #include namespace raft::runtime::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/refine_d_int64_t_float.cu b/cpp/src/distance/neighbors/refine_d_int64_t_float.cu index 101c756987..8ad8f9e8f1 100644 --- a/cpp/src/distance/neighbors/refine_d_int64_t_float.cu +++ b/cpp/src/distance/neighbors/refine_d_int64_t_float.cu @@ -15,7 +15,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/refine_d_int64_t_int8_t.cu b/cpp/src/distance/neighbors/refine_d_int64_t_int8_t.cu index 57cbc8a454..817369ed6a 100644 --- a/cpp/src/distance/neighbors/refine_d_int64_t_int8_t.cu +++ b/cpp/src/distance/neighbors/refine_d_int64_t_int8_t.cu @@ -15,7 +15,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/refine_d_int64_t_uint8_t.cu b/cpp/src/distance/neighbors/refine_d_int64_t_uint8_t.cu index 1f9d93cd35..fb426b2c02 100644 --- a/cpp/src/distance/neighbors/refine_d_int64_t_uint8_t.cu +++ b/cpp/src/distance/neighbors/refine_d_int64_t_uint8_t.cu @@ -15,7 +15,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/refine_h_int64_t_float.cu b/cpp/src/distance/neighbors/refine_h_int64_t_float.cu index 9d6f34312a..1f950dc3b6 100644 --- a/cpp/src/distance/neighbors/refine_h_int64_t_float.cu +++ b/cpp/src/distance/neighbors/refine_h_int64_t_float.cu @@ -16,7 +16,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/refine_h_int64_t_int8_t.cu b/cpp/src/distance/neighbors/refine_h_int64_t_int8_t.cu index 8757b15956..da99df3618 100644 --- a/cpp/src/distance/neighbors/refine_h_int64_t_int8_t.cu +++ b/cpp/src/distance/neighbors/refine_h_int64_t_int8_t.cu @@ -15,7 +15,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/refine_h_int64_t_uint8_t.cu b/cpp/src/distance/neighbors/refine_h_int64_t_uint8_t.cu index 3d6c8aa201..990754b033 100644 --- a/cpp/src/distance/neighbors/refine_h_int64_t_uint8_t.cu +++ b/cpp/src/distance/neighbors/refine_h_int64_t_uint8_t.cu @@ -15,7 +15,7 @@ */ #include -#include +#include namespace raft::runtime::neighbors { diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu index a12a5b0b7d..33c4e7ffc0 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_fast.cu @@ -14,9 +14,9 @@ * limitations under the License. */ -#include - #include +#include +#include namespace raft::neighbors::ivf_pq::detail { diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu index a91318ca34..f543369de5 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu index 1348d52ceb..1a0322a722 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_float_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu index 65ed409e9a..c7b5c9ffe9 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu index a7c8b3b7e0..efb2a477a7 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu index b9af5c00a5..b9051eb011 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8s_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu index 74364a5480..c6b1bad123 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu index 75f03c2df4..d6033345da 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu index 0c5404d5ea..1add18cb4a 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_fp8u_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu index 9eee24815b..6020d7035b 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu index e066459169..62be67e1a9 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu index e0cec88955..145312f334 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_float_half_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu index 9a1ae65cb2..c9365e1bb4 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu index 6d4662e8c4..d5c6934da2 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu index be8e53ca62..bac8c8706b 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8s_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu index b01b5c9139..2809005dd0 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu index dcc4486a2f..015ef21a15 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu index 190b6eb6a3..0ac96c8440 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_fp8u_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu index 3cdc75b66b..f3501d11c0 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_fast.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu index fb40962a24..7d10020480 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_basediff.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu index 088c94d4ad..91ec2eca3e 100644 --- a/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/compute_similarity_half_half_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu b/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu index e0cec88955..145312f334 100644 --- a/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu +++ b/cpp/src/distance/neighbors/specializations/detail/ivfpq_compute_similarity_float_half_no_smem_lut.cu @@ -15,6 +15,7 @@ */ #include +#include #include diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu index 50ed302595..d559291b93 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_float_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu index 99860b3200..c8b31e1fff 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_int8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu index 61e078e466..5fc62969f0 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_build_uint8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu index 84571f0a01..ccd773832d 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_float_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu index ded43bf9c2..ea182da5f0 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_int8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu index 643f5d45a5..7c5ef6af8f 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_extend_uint8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu index 615316ab00..fd8031f269 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_float_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu index 86e045315e..43c5953c25 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_int8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu b/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu index 543f7a28b6..2e365cc164 100644 --- a/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu +++ b/cpp/src/distance/neighbors/specializations/ivfpq_search_uint8_t_int64_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors::ivf_pq { diff --git a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_float.cu b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_float.cu index a40f428291..0b0125459d 100644 --- a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_float.cu +++ b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_float.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { diff --git a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_int8_t.cu b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_int8_t.cu index 26d3a7b455..d6c817b971 100644 --- a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_int8_t.cu +++ b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_int8_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { diff --git a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_uint8_t.cu b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_uint8_t.cu index 9080eb2297..3e0ca627a6 100644 --- a/cpp/src/distance/neighbors/specializations/refine_d_int64_t_uint8_t.cu +++ b/cpp/src/distance/neighbors/specializations/refine_d_int64_t_uint8_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { diff --git a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_float.cu b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_float.cu index 03e387f543..66a6bace53 100644 --- a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_float.cu +++ b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_float.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { diff --git a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_int8_t.cu b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_int8_t.cu index 045532e6a1..22824b3a8e 100644 --- a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_int8_t.cu +++ b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_int8_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { template void refine( diff --git a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_uint8_t.cu b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_uint8_t.cu index e33dae8ce8..58dcfc87c9 100644 --- a/cpp/src/distance/neighbors/specializations/refine_h_int64_t_uint8_t.cu +++ b/cpp/src/distance/neighbors/specializations/refine_h_int64_t_uint8_t.cu @@ -15,6 +15,7 @@ */ #include +#include namespace raft::neighbors { diff --git a/cpp/src/nn/specializations/ball_cover_all_knn_query.cu b/cpp/src/nn/specializations/ball_cover_all_knn_query.cu index da5cd8de4f..d9cb836bfc 100644 --- a/cpp/src/nn/specializations/ball_cover_all_knn_query.cu +++ b/cpp/src/nn/specializations/ball_cover_all_knn_query.cu @@ -22,9 +22,8 @@ #include #endif -#include -#include -#include +// TODO: Change this to proper specializations after FAISS is removed +#include #include diff --git a/cpp/src/nn/specializations/ball_cover_build_index.cu b/cpp/src/nn/specializations/ball_cover_build_index.cu index 70fcbec356..76c5a2bd5b 100644 --- a/cpp/src/nn/specializations/ball_cover_build_index.cu +++ b/cpp/src/nn/specializations/ball_cover_build_index.cu @@ -22,9 +22,8 @@ #include #endif -#include -#include -#include +// TODO: Change this to proper specializations after FAISS is removed +#include #include diff --git a/cpp/src/nn/specializations/ball_cover_knn_query.cu b/cpp/src/nn/specializations/ball_cover_knn_query.cu index d5ca1cbc1c..c01da452b2 100644 --- a/cpp/src/nn/specializations/ball_cover_knn_query.cu +++ b/cpp/src/nn/specializations/ball_cover_knn_query.cu @@ -22,9 +22,8 @@ #include #endif -#include -#include -#include +// TODO: Change this to proper specializations after FAISS is removed +#include #include diff --git a/cpp/src/nn/specializations/brute_force_knn_long_float_int.cu b/cpp/src/nn/specializations/brute_force_knn_long_float_int.cu index b08bcfbc79..a74ce0c0c5 100644 --- a/cpp/src/nn/specializations/brute_force_knn_long_float_int.cu +++ b/cpp/src/nn/specializations/brute_force_knn_long_float_int.cu @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/brute_force_knn_long_float_uint.cu b/cpp/src/nn/specializations/brute_force_knn_long_float_uint.cu index 78cb92bb38..576c74d9e0 100644 --- a/cpp/src/nn/specializations/brute_force_knn_long_float_uint.cu +++ b/cpp/src/nn/specializations/brute_force_knn_long_float_uint.cu @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_int.cu b/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_int.cu index 0082a30796..cba2436a4d 100644 --- a/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_int.cu +++ b/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_int.cu @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_uint.cu b/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_uint.cu index b2a1af2cf0..17c687ebce 100644 --- a/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_uint.cu +++ b/cpp/src/nn/specializations/brute_force_knn_uint32_t_float_uint.cu @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu index cff83ad3cf..9a71ce4f9a 100644 --- a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu +++ b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_2d.cu @@ -15,9 +15,11 @@ */ #include -#include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu index 1a1c17b29f..b1b3439e8f 100644 --- a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu +++ b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_one_3d.cu @@ -15,9 +15,11 @@ */ #include -#include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu index 7e784cb4d8..9f512dcda1 100644 --- a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu +++ b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_2d.cu @@ -15,9 +15,11 @@ */ #include -#include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu index e650c7ed37..0eeb448d1e 100644 --- a/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu +++ b/cpp/src/nn/specializations/detail/ball_cover_lowdim_pass_two_3d.cu @@ -15,9 +15,11 @@ */ #include -#include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/fused_l2_knn_int_float_false.cu b/cpp/src/nn/specializations/fused_l2_knn_int_float_false.cu index 7d1747cfc3..41cf409416 100644 --- a/cpp/src/nn/specializations/fused_l2_knn_int_float_false.cu +++ b/cpp/src/nn/specializations/fused_l2_knn_int_float_false.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/fused_l2_knn_int_float_true.cu b/cpp/src/nn/specializations/fused_l2_knn_int_float_true.cu index d6748a0e4a..7d183d7220 100644 --- a/cpp/src/nn/specializations/fused_l2_knn_int_float_true.cu +++ b/cpp/src/nn/specializations/fused_l2_knn_int_float_true.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/fused_l2_knn_long_float_false.cu b/cpp/src/nn/specializations/fused_l2_knn_long_float_false.cu index b96bb8987e..fa273986dc 100644 --- a/cpp/src/nn/specializations/fused_l2_knn_long_float_false.cu +++ b/cpp/src/nn/specializations/fused_l2_knn_long_float_false.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/src/nn/specializations/fused_l2_knn_long_float_true.cu b/cpp/src/nn/specializations/fused_l2_knn_long_float_true.cu index 379c24bf36..5313a87786 100644 --- a/cpp/src/nn/specializations/fused_l2_knn_long_float_true.cu +++ b/cpp/src/nn/specializations/fused_l2_knn_long_float_true.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,9 @@ #include #include +// TODO: Change this to proper specializations after FAISS is removed +#include + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 7cd0feba61..52fea785cd 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -198,6 +198,8 @@ if(BUILD_TESTS) test/matrix/slice.cu test/matrix/triangular.cu test/sparse/spectral_matrix.cu + OPTIONAL + DIST ) ConfigureTest( diff --git a/cpp/test/cluster/kmeans.cu b/cpp/test/cluster/kmeans.cu index 685bd1f965..3e2153dcde 100644 --- a/cpp/test/cluster/kmeans.cu +++ b/cpp/test/cluster/kmeans.cu @@ -30,7 +30,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include +#include #endif namespace raft { diff --git a/cpp/test/cluster/kmeans_balanced.cu b/cpp/test/cluster/kmeans_balanced.cu index 028819563e..ae06572061 100644 --- a/cpp/test/cluster/kmeans_balanced.cu +++ b/cpp/test/cluster/kmeans_balanced.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include +#include #endif /* This test takes advantage of the fact that make_blobs generates balanced clusters. diff --git a/cpp/test/cluster/kmeans_find_k.cu b/cpp/test/cluster/kmeans_find_k.cu index 5ac4ebd293..e80cbaa93b 100644 --- a/cpp/test/cluster/kmeans_find_k.cu +++ b/cpp/test/cluster/kmeans_find_k.cu @@ -25,7 +25,7 @@ #include #include -#if defined RAFT_DISTANCE_COMPILED && defined RAFT_NN_COMPILED +#if defined RAFT_DISTANCE_COMPILED #include #endif diff --git a/cpp/test/distance/masked_nn.cu b/cpp/test/distance/masked_nn.cu index 66d5a77dbf..6f7d8bf44a 100644 --- a/cpp/test/distance/masked_nn.cu +++ b/cpp/test/distance/masked_nn.cu @@ -28,6 +28,10 @@ #include #include +#ifdef RAFT_DISTANCE_COMPILED +#include +#endif + namespace raft::distance::masked_nn { // The adjacency pattern determines what distances get computed. diff --git a/cpp/test/neighbors/ann_ivf_flat.cuh b/cpp/test/neighbors/ann_ivf_flat.cuh index 26b8301cb1..510061e283 100644 --- a/cpp/test/neighbors/ann_ivf_flat.cuh +++ b/cpp/test/neighbors/ann_ivf_flat.cuh @@ -37,10 +37,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include -#endif -#if defined RAFT_DISTANCE_COMPILED && defined RAFT_NN_COMPILED -#include +#include #endif #include diff --git a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu index 975cf0387c..cee0d03c99 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_float_int64_t.cu @@ -18,8 +18,7 @@ #include "../ann_ivf_flat.cuh" -#if defined RAFT_DISTANCE_COMPILED && defined RAFT_NN_COMPILED -#include +#if defined RAFT_DISTANCE_COMPILED #include #endif diff --git a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu index 249e6fe487..95876f9165 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_int8_t_int64_t.cu @@ -18,8 +18,7 @@ #include "../ann_ivf_flat.cuh" -#if defined RAFT_DISTANCE_COMPILED && defined RAFT_NN_COMPILED -#include +#if defined RAFT_DISTANCE_COMPILED #include #endif diff --git a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu index 84397cf5c4..ebee20c2b6 100644 --- a/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu +++ b/cpp/test/neighbors/ann_ivf_flat/test_uint8_t_int64_t.cu @@ -18,8 +18,7 @@ #include "../ann_ivf_flat.cuh" -#if defined RAFT_DISTANCE_COMPILED && defined RAFT_NN_COMPILED -#include +#if defined RAFT_DISTANCE_COMPILED #include #endif diff --git a/cpp/test/neighbors/ann_ivf_pq.cuh b/cpp/test/neighbors/ann_ivf_pq.cuh index e2a938aef8..37b3e91434 100644 --- a/cpp/test/neighbors/ann_ivf_pq.cuh +++ b/cpp/test/neighbors/ann_ivf_pq.cuh @@ -24,8 +24,8 @@ #include #include #include -#if defined RAFT_DISTANCE_COMPILED -#include +#ifdef RAFT_DISTANCE_COMPILED +#include #else #pragma message("NN specializations are not enabled; expect very long building times.") #endif diff --git a/cpp/test/neighbors/ball_cover.cu b/cpp/test/neighbors/ball_cover.cu index a97df7df75..6e387a3bb1 100644 --- a/cpp/test/neighbors/ball_cover.cu +++ b/cpp/test/neighbors/ball_cover.cu @@ -23,7 +23,7 @@ #include #include #if defined RAFT_NN_COMPILED -#include +#include #endif #include diff --git a/cpp/test/neighbors/epsilon_neighborhood.cu b/cpp/test/neighbors/epsilon_neighborhood.cu index c78a15dd2d..977e8f3ce8 100644 --- a/cpp/test/neighbors/epsilon_neighborhood.cu +++ b/cpp/test/neighbors/epsilon_neighborhood.cu @@ -23,6 +23,10 @@ #include #include +#ifdef RAFT_DISTANCE_COMPILED +#include +#endif + namespace raft { namespace spatial { namespace knn { diff --git a/cpp/test/neighbors/fused_l2_knn.cu b/cpp/test/neighbors/fused_l2_knn.cu index 6ab0671229..5f0ae152de 100644 --- a/cpp/test/neighbors/fused_l2_knn.cu +++ b/cpp/test/neighbors/fused_l2_knn.cu @@ -23,7 +23,7 @@ #include #if defined RAFT_NN_COMPILED -#include +#include #endif #include diff --git a/cpp/test/neighbors/knn.cu b/cpp/test/neighbors/knn.cu index 6814d47dcb..fd7a1a03aa 100644 --- a/cpp/test/neighbors/knn.cu +++ b/cpp/test/neighbors/knn.cu @@ -21,7 +21,7 @@ #include #include #if defined RAFT_NN_COMPILED -#include +#include #endif #include diff --git a/cpp/test/neighbors/refine.cu b/cpp/test/neighbors/refine.cu index f0e85e0428..8866c404a9 100644 --- a/cpp/test/neighbors/refine.cu +++ b/cpp/test/neighbors/refine.cu @@ -32,7 +32,7 @@ #include #if defined RAFT_DISTANCE_COMPILED -#include +#include #endif #include diff --git a/cpp/test/neighbors/selection.cu b/cpp/test/neighbors/selection.cu index 141e14d727..26e37e433f 100644 --- a/cpp/test/neighbors/selection.cu +++ b/cpp/test/neighbors/selection.cu @@ -25,9 +25,6 @@ #include #include #if defined RAFT_DISTANCE_COMPILED -#include -#endif -#if defined RAFT_NN_COMPILED #include #endif diff --git a/docs/source/index.rst b/docs/source/index.rst index 9890bd932f..2418c6a767 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,6 +46,7 @@ While not exhaustive, the following general categories help summarize the accele pylibraft_api.rst raft_dask_api.rst using_comms.rst + using_libraft.md contributing.md diff --git a/docs/source/using_libraft.md b/docs/source/using_libraft.md new file mode 100644 index 0000000000..f4f966f2c8 --- /dev/null +++ b/docs/source/using_libraft.md @@ -0,0 +1,59 @@ +# Using The Pre-Compiled Binary + +At its core, RAFT is a header-only template library, which makes it very powerful in that APIs can be called with various different combinations of data types and only the templates which are actually used will be compiled into your binaries. This increased flexibility comes with a drawback that all the APIs need to be declared inline and thus calls which are made frequently in your code could be compiled again each source file for which they are invoked. + +For most functions, this overhead is pretty minimal and not noticeable but some of RAFT's APIs consist of very complex hierarchies of function calls that ultimately end up dispatching to device code that's executed on the GPU. The compile times for these APIs may still be bearable when compiling for only a single compute architecture but could end up becoming extremely slow to compile for all of the supported architectures at once. + +There are three ways to solve this problem and speed up compile times: +1. Continue to use RAFT as a header-only library and create a CUDA source file in your project to explicitly instantiate the templates which are slow to compile. This can be tedious and will still require compiling the slow code at least once, but it's the most flexible option if you are using types that aren't already compiled into `libraft` +2. If you are able to use one of the template types that are already being compiled into `libraft`, you can use the pre-compiled template specializations, which I will describe in more detail in the following section. +3. If you would like to use RAFT but either cannot or would prefer not to compile any CUDA code yourself, you can simply add `libraft` to your link libraries and use the growing set of runtime APIs. + +## Using Template Specializations + +As mentioned above, the pre-compiled template instantiations can save a lot of time if you are able to use the type combinations for the templates which are already specialized in the `libraft` binary. This will, of course, mean that you will need to add `libraft` to your link libraries. + +At the top level of each namespace containing pre-compiled template specializations is a header file called `specializations.cuh`. This header file includes `extern template` directives for all the specializations which are compiled into libraft. As an example, including `raft/neighbors/specializations.cuh` in one of your source files will effectively tell the compiler to skip over any of the template specializations that are already compiled into the `libraft` binary. + +### How do I verify template specializations didn't compile into my binary? + +Which specializations were chosen to instantiations were based on compile time analysis and reuse. This means you can't assume that all specializations are for the public API itself. Take the following example in `raft/neighbors/specializations/detail/ivf_pq_compute_similarity.cuh`: + +```c++ +namespace raft::neighbors::ivf_pq::detail { + +namespace { +using fp8s_t = fp_8bit<5, true>; +using fp8u_t = fp_8bit<5, false>; +} // namespace + +#define RAFT_INST(OutT, LutT) \ + extern template auto get_compute_similarity_kernel(uint32_t, uint32_t) \ + ->compute_similarity_kernel_t; \ + extern template auto get_compute_similarity_kernel(uint32_t, uint32_t) \ + ->compute_similarity_kernel_t; \ + extern template auto get_compute_similarity_kernel(uint32_t, uint32_t) \ + ->compute_similarity_kernel_t; + +#define RAFT_INST_ALL_OUT_T(LutT) \ + RAFT_INST(float, LutT) \ + RAFT_INST(half, LutT) + +RAFT_INST_ALL_OUT_T(float) +RAFT_INST_ALL_OUT_T(half) +RAFT_INST_ALL_OUT_T(fp8s_t) +RAFT_INST_ALL_OUT_T(fp8u_t) + +#undef RAFT_INST +#undef RAFT_INST_ALL_OUT_T + +} // namespace raft::neighbors::ivf_pq::detail +``` + +We can see here that the function `raft::neighbors::ivf_pq::detail::get_compute_similarity_kernel` is being instantiated for the cartesian product of `OutT={float, half, fp8s_t, fp8u_t}` and `LutT={float, half}`. After linking against the `libraft` binary and including `raft/neighbors/specializations.cuh` in your source file, you can invoke the `raft::neighbors::ivf_pq` functions and compile your code. If the specializations are working, you should be able to use `nm -g -C --defined-only /path/to/your/binary | grep raft::neighbors::ivf_pq::detail::get_compute_similarity::kernel` and you shouldn't see any results, because those symbols should be coming from the `libraft` binary and skipped from compiling into your binary. + +## Runtime APIs + +RAFT contains a growing list of runtime APIs that, unlike the pre-compiled template specializations, allow you to link against `libraft` and invoke RAFT directly from `cpp` files. The benefit to RAFT's runtime APIs are two-fold- unlike the template specializations, which still require your code be compiled with the CUDA compiler (`nvcc`), the `runtime` APIs are the lightweight wrappers which enable `pylibraft`. + +Similar to the pre-compiled template specializations, RAFT's runtime APIs \ No newline at end of file From e35b51a1e05258e5bd0ab099c2753ca892328f86 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 10 Mar 2023 20:44:40 -0500 Subject: [PATCH 5/6] Fixing compile error in fused l2 knn --- cpp/test/neighbors/fused_l2_knn.cu | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/test/neighbors/fused_l2_knn.cu b/cpp/test/neighbors/fused_l2_knn.cu index 5f0ae152de..61bee62235 100644 --- a/cpp/test/neighbors/fused_l2_knn.cu +++ b/cpp/test/neighbors/fused_l2_knn.cu @@ -26,6 +26,12 @@ #include #endif +#ifdef RAFT_DISTANCE_COMPILED +#include +#endif + +#include + #include #include From b5cf3c5778b24b93698b442d200fe2c2b34e9589 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Sat, 11 Mar 2023 16:09:35 -0500 Subject: [PATCH 6/6] Small follow-on cleanup --- cpp/bench/neighbors/knn.cuh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/bench/neighbors/knn.cuh b/cpp/bench/neighbors/knn.cuh index 0a59da3970..e505d4d456 100644 --- a/cpp/bench/neighbors/knn.cuh +++ b/cpp/bench/neighbors/knn.cuh @@ -27,12 +27,10 @@ #if defined RAFT_DISTANCE_COMPILED #include -// TODO: Legacy. Remove when FAISS is removed -#include #endif #if defined RAFT_NN_COMPILED -#include +// TODO: Legacy. Remove when FAISS is removed #include #endif