Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

IVF-Flat reconstruction #1270

Open
wants to merge 48 commits into
base: branch-23.08
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
328a179
Initial index splitting
lowener Feb 7, 2023
a83aca4
Adapt `extend`
lowener Feb 7, 2023
843904a
Refactoring: build and extend fix
lowener Feb 9, 2023
f09b3a0
Refactor ivf flat search for index splitting
lowener Feb 10, 2023
fdc9395
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Feb 10, 2023
93d5b35
Use mdpsan/mdarray aliases
lowener Feb 12, 2023
5fcf564
Add serialization
lowener Feb 13, 2023
c49bf67
Deserialize ivf_flat and style fix
lowener Feb 16, 2023
6ba87d4
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Feb 19, 2023
7e2d80b
Integrate ivf::list to ivf_flat index splitting
lowener Feb 21, 2023
74e0a8c
Update refine
lowener Feb 21, 2023
7b36742
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Feb 22, 2023
135a9b6
Use std vector for ivf flat index list
lowener Feb 24, 2023
81b2cbf
Test second variant of `ivf_flat::extend`
lowener Feb 24, 2023
fd33dbc
Use ValueT template on spec
lowener Feb 25, 2023
539fbc5
Use second variant of ivf_extend
lowener Feb 26, 2023
31815d7
Merge branch 'branch-23.04' into 23.04-flat-split
cjnolet Feb 27, 2023
33bfb82
Fix spec template
lowener Feb 28, 2023
adb96e4
Revert ValueT on ivfpq
lowener Mar 6, 2023
32936c9
Make ivf::list more flexible
achirkin Mar 7, 2023
23e0f84
Merge pull request #1 from achirkin/23.04-flat-split
lowener Mar 7, 2023
acf1888
Use new list API for ivf flat
lowener Mar 7, 2023
0283d25
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Mar 7, 2023
f790abf
Fix adaptive centers
lowener Mar 9, 2023
6a05196
Fix adaptive center norms testing
lowener Mar 9, 2023
eda7923
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Mar 9, 2023
f0a7031
Fix list spec template after merge
lowener Mar 9, 2023
a6c54a4
Add serialization overload
lowener Mar 9, 2023
85b4aa1
Fix list size override
lowener Mar 10, 2023
692af0d
Fix resizelist with interleaved format
lowener Mar 10, 2023
d892ebb
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Mar 10, 2023
a8b96a7
Fix refine list resize operation
lowener Mar 12, 2023
1870541
Merge branch 'branch-23.04' into 23.04-flat-split
lowener Mar 12, 2023
beb9264
IVF-Flat reconstruction
viclafargue Mar 14, 2023
2e8bc92
Merge branch 'branch-23.04' into ivf-reconstruction
viclafargue Mar 15, 2023
ebda975
Merge branch 'branch-23.04' into ivf-reconstruction
cjnolet Mar 23, 2023
81d51be
Merge branch 'branch-23.04' into ivf-reconstruction
cjnolet Mar 25, 2023
d250ae9
Merge branch 'branch-23.06' into ivf-reconstruction
cjnolet Apr 12, 2023
e9df346
Merge branch 'branch-23.06' into ivf-reconstruction
viclafargue May 22, 2023
f798657
addressing review + expose function
viclafargue May 23, 2023
080c784
Cluster reconstruction
viclafargue May 29, 2023
0f6cdbe
addressing review 1/2
viclafargue Jul 12, 2023
cad88b3
addressing review 2/2
viclafargue Jul 12, 2023
6cd1abc
Fix style
viclafargue Jul 12, 2023
de6c13d
addressing review
viclafargue Jul 14, 2023
7064703
Merge branch 'branch-23.08' into ivf-reconstruction
viclafargue Jul 14, 2023
5e02d7f
Fix style
viclafargue Jul 14, 2023
87edda0
Fix doc
viclafargue Jul 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'branch-23.04' into 23.04-flat-split
lowener committed Mar 9, 2023
commit eda7923c28b5fe497dab782495269f213ad458c9
46 changes: 7 additions & 39 deletions cpp/include/raft/neighbors/detail/ivf_pq_serialize.cuh
Original file line number Diff line number Diff line change
@@ -114,39 +114,7 @@ void serialize(raft::device_resources const& handle_,
std::ofstream of(filename, std::ios::out | std::ios::binary);
if (!of) { RAFT_FAIL("Cannot open file %s", filename.c_str()); }

RAFT_LOG_DEBUG("Size %zu, dim %d, pq_dim %d, pq_bits %d",
static_cast<size_t>(index.size()),
static_cast<int>(index.dim()),
static_cast<int>(index.pq_dim()),
static_cast<int>(index.pq_bits()));

serialize_scalar(handle_, of, kSerializationVersion);
serialize_scalar(handle_, of, index.size());
serialize_scalar(handle_, of, index.dim());
serialize_scalar(handle_, of, index.pq_bits());
serialize_scalar(handle_, of, index.pq_dim());
serialize_scalar(handle_, of, index.conservative_memory_allocation());

serialize_scalar(handle_, of, index.metric());
serialize_scalar(handle_, of, index.codebook_kind());
serialize_scalar(handle_, of, index.n_lists());

serialize_mdspan(handle_, of, index.pq_centers());
serialize_mdspan(handle_, of, index.centers());
serialize_mdspan(handle_, of, index.centers_rot());
serialize_mdspan(handle_, of, index.rotation_matrix());

auto sizes_host = make_host_mdarray<uint32_t, uint32_t, row_major>(index.list_sizes().extents());
copy(sizes_host.data_handle(),
index.list_sizes().data_handle(),
sizes_host.size(),
handle_.get_stream());
handle_.sync_stream();
serialize_mdspan(handle_, of, sizes_host.view());
auto list_store_spec = list_spec<uint32_t, IdxT>{index.pq_bits(), index.pq_dim(), true};
for (uint32_t label = 0; label < index.n_lists(); label++) {
ivf::serialize_list(handle_, of, index.lists()[label], list_store_spec, sizes_host(label));
}
detail::serialize(handle_, of, index);

of.close();
if (!of) { RAFT_FAIL("Error writing output %s", filename.c_str()); }
@@ -189,15 +157,15 @@ auto deserialize(raft::device_resources const& handle_, std::istream& is) -> ind
auto index = raft::neighbors::ivf_pq::index<IdxT>(
handle_, metric, codebook_kind, n_lists, dim, pq_bits, pq_dim, cma);

deserialize_mdspan(handle_, infile, index.pq_centers());
deserialize_mdspan(handle_, infile, index.centers());
deserialize_mdspan(handle_, infile, index.centers_rot());
deserialize_mdspan(handle_, infile, index.rotation_matrix());
deserialize_mdspan(handle_, infile, index.list_sizes());
deserialize_mdspan(handle_, is, index.pq_centers());
deserialize_mdspan(handle_, is, index.centers());
deserialize_mdspan(handle_, is, index.centers_rot());
deserialize_mdspan(handle_, is, index.rotation_matrix());
deserialize_mdspan(handle_, is, index.list_sizes());
auto list_device_spec = list_spec<uint32_t, IdxT>{pq_bits, pq_dim, cma};
auto list_store_spec = list_spec<uint32_t, IdxT>{pq_bits, pq_dim, true};
for (auto& list : index.lists()) {
ivf::deserialize_list(handle_, infile, list, list_store_spec, list_device_spec);
ivf::deserialize_list(handle_, is, list, list_store_spec, list_device_spec);
}

handle_.sync_stream();
You are viewing a condensed version of this merge commit. You can view the full changes here.