Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refine
Browse files Browse the repository at this point in the history
Signed-off-by: Lloyd-Pottiger <[email protected]>
Lloyd-Pottiger committed Oct 25, 2024

Verified

This commit was signed with the committer’s verified signature.
paulober Paul
1 parent d4d25db commit 11dc897
Showing 3 changed files with 13 additions and 19 deletions.
4 changes: 2 additions & 2 deletions dbms/src/Storages/DeltaMerge/File/DMFileVectorIndexReader.cpp
Original file line number Diff line number Diff line change
@@ -190,7 +190,7 @@ std::vector<VectorIndexViewer::Key> DMFileVectorIndexReader::loadVectorSearchRes

void DMFileVectorIndexReader::read(
MutableColumnPtr & vec_column,
const std::vector<VectorIndexViewer::Key> & row_ids,
const std::span<const VectorIndexViewer::Key> & selected_rows,
size_t start_offset,
size_t column_size)
{
@@ -200,7 +200,7 @@ void DMFileVectorIndexReader::read(
vec_column->reserve(column_size);
std::vector<Float32> value;
size_t current_rowid = start_offset;
for (auto rowid : row_ids)
for (auto rowid : selected_rows)
{
vec_index->get(rowid, value);
if (rowid > current_rowid)
Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ class DMFileVectorIndexReader
// return the real number of rows read.
void read(
MutableColumnPtr & vec_column,
const std::vector<VectorIndexViewer::Key> & row_ids,
const std::span<const VectorIndexViewer::Key> & selected_rows,
size_t start_offset,
size_t column_size);

Original file line number Diff line number Diff line change
@@ -168,16 +168,13 @@ std::tuple<Block, size_t> DMFileWithVectorIndexBlockInputStream::readByIndexRead
size_t read_rows = index_reader_next_row_id - block_start_row_id;
auto vec_column = vec_cd.type->createColumn();

std::vector<VectorIndexViewer::Key> row_ids;
for (auto row_id : sorted_results)
{
if (row_id >= block_start_row_id && row_id < index_reader_next_row_id)
row_ids.push_back(row_id);
}
vec_index_reader->read(vec_column, row_ids, block_start_row_id, read_rows);
auto begin = std::lower_bound(sorted_results.cbegin(), sorted_results.cend(), block_start_row_id);
auto end = std::lower_bound(begin, sorted_results.cend(), index_reader_next_row_id);
const std::span block_selected_rows{begin, end};
vec_index_reader->read(vec_column, block_selected_rows, block_start_row_id, read_rows);

block.insert(ColumnWithTypeAndName{std::move(vec_column), vec_cd.type, vec_cd.name, vec_cd.id});
return {block, row_ids.size()};
return {block, block_selected_rows.size()};
}

std::tuple<Block, size_t> DMFileWithVectorIndexBlockInputStream::readByFollowingOtherColumns()
@@ -199,19 +196,16 @@ std::tuple<Block, size_t> DMFileWithVectorIndexBlockInputStream::readByFollowing
auto vec_column = vec_cd.type->createColumn();

// Then read from vector index for the same pack.
std::vector<VectorIndexViewer::Key> row_ids;
for (auto row_id : sorted_results)
{
if (row_id >= block_others.startOffset() && row_id < block_others.startOffset() + read_rows)
row_ids.push_back(row_id);
}
vec_index_reader->read(vec_column, row_ids, block_others.startOffset(), read_rows);
auto begin = std::lower_bound(sorted_results.cbegin(), sorted_results.cend(), block_others.startOffset());
auto end = std::lower_bound(begin, sorted_results.cend(), block_others.startOffset() + read_rows);
const std::span block_selected_rows{begin, end};
vec_index_reader->read(vec_column, block_selected_rows, block_others.startOffset(), read_rows);

// Re-assemble block using the same layout as header.
// Insert the vector column into the block.
auto index = header.getPositionByName(vec_cd.name);
block_others.insert(index, ColumnWithTypeAndName(std::move(vec_column), vec_cd.type, vec_cd.name));
return {block_others, row_ids.size()};
return {block_others, block_selected_rows.size()};
}

void DMFileWithVectorIndexBlockInputStream::load()

0 comments on commit 11dc897

Please sign in to comment.