From fadba70d1d232329a598f126b9bd92ff8a7fa464 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 15 Nov 2023 11:34:21 -0500 Subject: [PATCH 1/5] Updating ivf-flat serialization --- cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh index 61a6046273..c5218549ad 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh @@ -45,7 +45,7 @@ struct check_index_layout { "paste in the new size and consider updating the serialization logic"); }; -template struct check_index_layout), 328>; +template struct check_index_layout), 368>; /** * Save the index to file. From c7a57a04b5535a33ee23c3f4a89238b6e09a1656 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 15 Nov 2023 11:48:10 -0500 Subject: [PATCH 2/5] Updating ivf-Pq serialize --- cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh index f01035cad3..0012ac18f9 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh @@ -48,7 +48,7 @@ struct check_index_layout { }; // TODO: Recompute this and come back to it. -template struct check_index_layout), 480>; +template struct check_index_layout), 536>; /** * Write the index to an output stream From fb108df790c32a80ec013b0a82bfcb4a12c68049 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 15 Nov 2023 12:36:53 -0600 Subject: [PATCH 3/5] Update cagra_serialize.cuh. --- cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh index eb21b75d3a..439d10cd5e 100644 --- a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh @@ -31,7 +31,7 @@ namespace raft::neighbors::cagra::detail { -constexpr int serialization_version = 3; +constexpr int serialization_version = 4; // NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error // message. @@ -42,7 +42,7 @@ struct check_index_layout { "paste in the new size and consider updating the serialization logic"); }; -constexpr size_t expected_size = 200; +constexpr size_t expected_size = 216; template struct check_index_layout), expected_size>; /** From 54f55b732b48338c0782cb5e546bccf7dd0cb07a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 15 Nov 2023 12:38:02 -0600 Subject: [PATCH 4/5] Increment IVF serialization versions. --- cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh | 4 ++-- cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh index c5218549ad..e6d47393fd 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh @@ -29,12 +29,12 @@ namespace raft::neighbors::ivf_flat::detail { -// Serialization version 3 +// Serialization version // No backward compatibility yet; that is, can't add additional fields without breaking // backward compatibility. // TODO(hcho3) Implement next-gen serializer for IVF that allows for expansion in a backward // compatible fashion. -constexpr int serialization_version = 4; +constexpr int serialization_version = 5; // NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error // message. diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh index 0012ac18f9..34446a98e4 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh @@ -36,7 +36,7 @@ namespace raft::neighbors::ivf_pq::detail { // backward compatibility. // TODO(hcho3) Implement next-gen serializer for IVF that allows for expansion in a backward // compatible fashion. -constexpr int kSerializationVersion = 3; +constexpr int kSerializationVersion = 4; // NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error // message. From 9a65de2a1088c1280e5bb980321b78783c1917b8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 15 Nov 2023 12:40:58 -0600 Subject: [PATCH 5/5] Remove static checks for index size. --- .../neighbors/detail/cagra/cagra_serialize.cuh | 14 +------------- .../raft/neighbors/detail/ivf_flat_serialize.cuh | 13 +------------ .../raft/neighbors/detail/ivf_pq_serialize.cuh | 14 +------------- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh index 439d10cd5e..51c9475434 100644 --- a/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/cagra_serialize.cuh @@ -31,19 +31,7 @@ namespace raft::neighbors::cagra::detail { -constexpr int serialization_version = 4; - -// NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error -// message. -template -struct check_index_layout { - static_assert(RealSize == ExpectedSize, - "The size of the index struct has changed since the last update; " - "paste in the new size and consider updating the serialization logic"); -}; - -constexpr size_t expected_size = 216; -template struct check_index_layout), expected_size>; +constexpr int serialization_version = 3; /** * Save the index to file. diff --git a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh index e6d47393fd..aaf48ae830 100644 --- a/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_flat_serialize.cuh @@ -34,18 +34,7 @@ namespace raft::neighbors::ivf_flat::detail { // backward compatibility. // TODO(hcho3) Implement next-gen serializer for IVF that allows for expansion in a backward // compatible fashion. -constexpr int serialization_version = 5; - -// NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error -// message. -template -struct check_index_layout { - static_assert(RealSize == ExpectedSize, - "The size of the index struct has changed since the last update; " - "paste in the new size and consider updating the serialization logic"); -}; - -template struct check_index_layout), 368>; +constexpr int serialization_version = 4; /** * Save the index to file. diff --git a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh index 34446a98e4..038bf8d7cc 100644 --- a/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh +++ b/cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh @@ -36,19 +36,7 @@ namespace raft::neighbors::ivf_pq::detail { // backward compatibility. // TODO(hcho3) Implement next-gen serializer for IVF that allows for expansion in a backward // compatible fashion. -constexpr int kSerializationVersion = 4; - -// NB: we wrap this check in a struct, so that the updated RealSize is easy to see in the error -// message. -template -struct check_index_layout { - static_assert(RealSize == ExpectedSize, - "The size of the index struct has changed since the last update; " - "paste in the new size and consider updating the serialization logic"); -}; - -// TODO: Recompute this and come back to it. -template struct check_index_layout), 536>; +constexpr int kSerializationVersion = 3; /** * Write the index to an output stream