diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index 971e8f36e2a..bd67e59df77 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -300,7 +300,7 @@ std::string Block::dumpNames() const out << ", "; out << it->name; } - return out.str(); + return out.releaseStr(); } @@ -313,7 +313,21 @@ std::string Block::dumpStructure() const out << ", "; it->dumpStructure(out); } - return out.str(); + return out.releaseStr(); +} + +std::string Block::dumpJsonStructure() const +{ + WriteBufferFromOwnString out; + out << "["; + for (auto it = data.begin(); it != data.end(); ++it) + { + if (it != data.begin()) + out << ","; + it->dumpJsonStructure(out); + } + out << "]"; + return out.releaseStr(); } diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index 713ae85d082..04378c7553f 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -118,6 +118,8 @@ class Block /** List of names, types and lengths of columns. Designed for debugging. */ std::string dumpStructure() const; + std::string dumpJsonStructure() const; + /** Get the same block, but empty. */ Block cloneEmpty() const; diff --git a/dbms/src/Core/ColumnWithTypeAndName.cpp b/dbms/src/Core/ColumnWithTypeAndName.cpp index 644b8cff0bf..56f8e1a36e8 100644 --- a/dbms/src/Core/ColumnWithTypeAndName.cpp +++ b/dbms/src/Core/ColumnWithTypeAndName.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -48,15 +49,29 @@ void ColumnWithTypeAndName::dumpStructure(WriteBuffer & out) const out << ' ' << column->dumpStructure(); else out << " nullptr"; - - out << " " << column_id; } String ColumnWithTypeAndName::dumpStructure() const { WriteBufferFromOwnString out; dumpStructure(out); - return out.str(); + return out.releaseStr(); +} + +void ColumnWithTypeAndName::dumpJsonStructure(WriteBuffer & out) const +{ + out << fmt::format(R"json({{"name":"{}","id":{},"type":{},"column":{}}})json", + name, + column_id, + (type ? "\"" + type->getName() + "\"" : "null"), + (column ? "\"" + column->dumpStructure() + "\"" : "null")); +} + +String ColumnWithTypeAndName::dumpJsonStructure() const +{ + WriteBufferFromOwnString out; + dumpJsonStructure(out); + return out.releaseStr(); } } // namespace DB diff --git a/dbms/src/Core/ColumnWithTypeAndName.h b/dbms/src/Core/ColumnWithTypeAndName.h index 8fff3b1453a..ea889cdb2dd 100644 --- a/dbms/src/Core/ColumnWithTypeAndName.h +++ b/dbms/src/Core/ColumnWithTypeAndName.h @@ -63,6 +63,9 @@ struct ColumnWithTypeAndName void dumpStructure(WriteBuffer & out) const; String dumpStructure() const; + + void dumpJsonStructure(WriteBuffer & out) const; + String dumpJsonStructure() const; }; } // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h index 33433da10f8..4e1a7b6fd3d 100644 --- a/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h +++ b/dbms/src/Storages/DeltaMerge/DMSegmentThreadInputStream.h @@ -104,7 +104,13 @@ class DMSegmentThreadInputStream : public IProfilingBlockInputStream cur_segment = task->segment; if (is_raw) { - cur_stream = cur_segment->getInputStreamRaw(*dm_context, columns_to_read, task->read_snapshot, task->ranges, filter, do_delete_mark_filter_for_raw); + cur_stream = cur_segment->getInputStreamRaw( + *dm_context, + columns_to_read, + task->read_snapshot, + task->ranges, + filter, + do_delete_mark_filter_for_raw); } else { diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h index 77335e6d9f0..ad585d684c7 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h @@ -46,7 +46,7 @@ inline Handle encodeToPK(T v) inline size_t getPosByColumnId(const Block & block, ColId col_id) { size_t pos = 0; - for (auto & c : block) + for (const auto & c : block) { if (c.column_id == col_id) return pos; @@ -57,7 +57,7 @@ inline size_t getPosByColumnId(const Block & block, ColId col_id) inline ColumnWithTypeAndName tryGetByColumnId(const Block & block, ColId col_id) { - for (auto & c : block) + for (const auto & c : block) { if (c.column_id == col_id) return c; @@ -68,7 +68,7 @@ inline ColumnWithTypeAndName tryGetByColumnId(const Block & block, ColId col_id) // TODO: we should later optimize getByColumnId. inline const ColumnWithTypeAndName & getByColumnId(const Block & block, ColId col_id) { - for (auto & c : block) + for (const auto & c : block) { if (c.column_id == col_id) return c; @@ -105,7 +105,7 @@ inline PaddedPODArray const * toColumnVectorDataPtr(const ColumnPtr & column) { if (column->isColumnConst()) { - auto * const_col = static_cast(column.get()); + const auto * const_col = static_cast(column.get()); const ColumnVector & c = assert_cast &>(const_col->getDataColumn()); return &c.getData(); @@ -191,7 +191,7 @@ inline Block genBlock(const ColumnDefines & column_defines, const Columns & colu Block block; for (size_t i = 0; i < column_defines.size(); ++i) { - auto & c = column_defines[i]; + const auto & c = column_defines[i]; addColumnToBlock(block, c.id, c.name, c.type, columns[i], c.default_value); } return block; @@ -200,7 +200,7 @@ inline Block genBlock(const ColumnDefines & column_defines, const Columns & colu inline Block getNewBlockByHeader(const Block & header, const Block & block) { Block new_block; - for (auto & c : header) + for (const auto & c : header) new_block.insert(block.getByName(c.name)); return new_block; } @@ -215,7 +215,7 @@ inline ColumnDefines getColumnDefinesFromBlock(const Block & block) inline bool hasColumn(const ColumnDefines & columns, const ColId & col_id) { - for (auto & c : columns) + for (const auto & c : columns) { if (c.id == col_id) return true; @@ -231,8 +231,8 @@ inline bool isSameSchema(const Block & a, const Block & b) return false; for (size_t i = 0; i < a.columns(); ++i) { - auto & ca = a.getByPosition(i); - auto & cb = b.getByPosition(i); + const auto & ca = a.getByPosition(i); + const auto & cb = b.getByPosition(i); bool col_ok = ca.column_id == cb.column_id; bool name_ok = ca.name == cb.name; diff --git a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h index 84fafbc46ef..6c6343cb96a 100644 --- a/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h +++ b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h @@ -41,6 +41,8 @@ namespace tests // Add this so that we can call typeFromString under namespace DB::DM::tests using DB::tests::typeFromString; +using namespace DB::tests; + /// helper functions for comparing HandleRange inline ::testing::AssertionResult HandleRangeCompare( const char * lhs_expr, @@ -96,6 +98,18 @@ inline std::vector createSignedNumbers(size_t beg, size_t end) return values; } +// Mock a common_pk_col that composed by number `rowkey_column_size` of int64 value +inline String genMockCommonHandle(Int64 value, size_t rowkey_column_size) +{ + WriteBufferFromOwnString ss; + for (size_t index = 0; index < rowkey_column_size; ++index) + { + ::DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); + ::DB::EncodeInt64(value, ss); + } + return ss.releaseStr(); +} + class DMTestEnv { public: @@ -285,13 +299,7 @@ class DMTestEnv for (size_t i = 0; i < num_rows; i++) { Int64 value = reversed ? end - 1 - i : beg + i; - WriteBufferFromOwnString ss; - for (size_t index = 0; index < rowkey_column_size; index++) - { - ::DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - ::DB::EncodeInt64(value, ss); - } - values.emplace_back(ss.releaseStr()); + values.emplace_back(genMockCommonHandle(value, rowkey_column_size)); } block.insert(DB::tests::createColumn( std::move(values), @@ -410,16 +418,7 @@ class DMTestEnv const size_t num_rows = 1; if (is_common_handle) { - Strings values; - { - WriteBufferFromOwnString ss; - for (size_t index = 0; index < rowkey_column_size; index++) - { - ::DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - ::DB::EncodeInt64(pk, ss); - } - values.emplace_back(ss.releaseStr()); - } + Strings values{genMockCommonHandle(pk, rowkey_column_size)}; block.insert(DB::tests::createColumn( std::move(values), pk_name, @@ -466,20 +465,8 @@ class DMTestEnv static RowKeyRange getRowKeyRangeForClusteredIndex(Int64 start, Int64 end, size_t rowkey_column_size) { - WriteBufferFromOwnString ss; - for (size_t i = 0; i < rowkey_column_size; i++) - { - EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - EncodeInt64(start, ss); - } - RowKeyValue start_key = RowKeyValue(true, std::make_shared(ss.releaseStr())); - ss.restart(); - for (size_t i = 0; i < rowkey_column_size; i++) - { - EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - EncodeInt64(end, ss); - } - RowKeyValue end_key = RowKeyValue(true, std::make_shared(ss.releaseStr())); + RowKeyValue start_key = RowKeyValue(true, std::make_shared(genMockCommonHandle(start, rowkey_column_size))); + RowKeyValue end_key = RowKeyValue(true, std::make_shared(genMockCommonHandle(end, rowkey_column_size))); return RowKeyRange(start_key, end_key, true, rowkey_column_size); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp index f6ca7900a13..4ed58b03e73 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_column_filter.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include namespace DB { @@ -45,7 +47,7 @@ class DebugBlockInputStream : public BlocksListBlockInputStream bool is_common_handle; }; -BlockInputStreamPtr genColumnFilterInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) +BlockInputStreamPtr genColumnProjInputStream(BlocksList & blocks, const ColumnDefines & columns, bool is_common_handle) { ColumnDefine handle_define( TiDBPkColumnID, @@ -85,34 +87,16 @@ TEST(DeleteFilterTest, NormalCase) ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); - { - auto in = genDeleteFilterInputStream(blocks, columns, false); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 1); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "hello"); - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, "world"); - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, "TiFlash"); - - block = in->read(); - ASSERT_FALSE(block); // ensure the stream is ended - in->readSuffix(); - } + auto in = genDeleteFilterInputStream(blocks, columns, false); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({str_col_name}), + createColumns({ + createColumn({"hello", "world", "TiFlash"}), + })); } -TEST(ColumnFilterTest, NormalCase) +TEST(ColumnProjectionTest, NormalCase) { BlocksList blocks; @@ -125,47 +109,22 @@ TEST(ColumnFilterTest, NormalCase) blocks.push_back(DMTestEnv::prepareOneRowBlock(pk_value, 40, 1, str_col_name, "Storage", false, 1)); } + // Only keep the column `str_col_name` ColumnDefines columns = getColumnDefinesFromBlock(blocks.back()); - + for (auto iter = columns.begin(); iter != columns.end(); /**/) { - auto in = genColumnFilterInputStream(blocks, columns, false); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 1); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "hello"); - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, "world"); - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, ""); - - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, "TiFlash"); - - block = in->read(); - ASSERT_EQ(block.rows(), 1); - col = block.getByName(str_col_name); - val = col.column->getDataAt(0); - ASSERT_EQ(val, "Storage"); - - block = in->read(); - ASSERT_FALSE(block); // ensure the stream is ended - in->readSuffix(); + if (iter->name != str_col_name) + iter = columns.erase(iter); + else + iter++; } + + ASSERT_INPUTSTREAM_BLOCK_UR( + genColumnProjInputStream(blocks, columns, false), + Block({ + createColumn({"hello", "world", "", "TiFlash", "Storage"}, str_col_name), + })); } } // namespace tests } // namespace DM -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp index 00f31bc97e7..9b30bbded9d 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB { @@ -23,7 +24,7 @@ namespace DM { namespace tests { -TEST(PKSquash_test, WithExtraSort) +TEST(PKSquashTest, WithExtraSort) { BlocksList blocks; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_index_manager.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_index_manager.cpp index 7b90d23036f..2ccecbe591c 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_index_manager.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_index_manager.cpp @@ -22,10 +22,10 @@ namespace DM { namespace tests { -class DeltaIndexManager_test : public ::testing::Test +class DeltaIndexManagerTest : public ::testing::Test { public: - DeltaIndexManager_test() + DeltaIndexManagerTest() : one_node_size(DefaultDeltaTree().getBytes()) {} @@ -41,7 +41,7 @@ DeltaIndexPtr genDeltaIndex() } -TEST_F(DeltaIndexManager_test, LRU) +TEST_F(DeltaIndexManagerTest, LRU) try { DeltaIndexManager manager(one_node_size * 100); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index 19b9e73f4e7..1e48dd4e4af 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -14,12 +14,21 @@ #include #include +#include #include +#include +#include #include #include +#include #include +#include +#include +#include +#include #include +#include namespace DB { @@ -39,7 +48,6 @@ namespace DM { namespace tests { - String testModeToString(const ::testing::TestParamInfo & info) { const auto mode = info.param; @@ -120,6 +128,7 @@ try DMTestEnv::PkType::PkIsHandleInt32, }) { + SCOPED_TRACE(fmt::format("Test case for {}", DMTestEnv::PkTypeToString(pk_type))); LOG_FMT_INFO(log, "Test case for {} begin.", DMTestEnv::PkTypeToString(pk_type)); auto cols = DMTestEnv::getDefaultColumns(pk_type); @@ -144,8 +153,7 @@ try block1 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block1)); ASSERT_EQ(block1.rows(), nrows); ASSERT_TRUE(block1.has(EXTRA_HANDLE_COLUMN_NAME)); - for (const auto & c : block1) - ASSERT_EQ(c.column->size(), nrows); + ASSERT_NO_THROW({ block1.checkNumberOfRows(); }); // Make a block that is overlapped with `block1` and it should be squashed by `PKSquashingBlockInputStream` size_t nrows_2 = 2; @@ -161,27 +169,12 @@ try block2 = DeltaMergeStore::addExtraColumnIfNeed(*db_context, store->getHandle(), std::move(block2)); ASSERT_EQ(block2.rows(), nrows_2); ASSERT_TRUE(block2.has(EXTRA_HANDLE_COLUMN_NAME)); - for (const auto & c : block2) - ASSERT_EQ(c.column->size(), nrows_2); + ASSERT_NO_THROW({ block2.checkNumberOfRows(); }); BlockInputStreamPtr stream = std::make_shared(BlocksList{block1, block2}); stream = std::make_shared>(stream, EXTRA_HANDLE_COLUMN_ID, store->isCommonHandle()); - - size_t num_rows_read = 0; - stream->readPrefix(); - while (Block block = stream->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - ASSERT_EQ(c->size(), block.rows()) - << "unexpected num of rows for column [name=" << iter.name << "] " << DMTestEnv::PkTypeToString(pk_type); - } - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, nrows + nrows_2); + ASSERT_INPUTSTREAM_NROWS(stream, nrows + nrows_2); LOG_FMT_INFO(log, "Test case for {} done.", DMTestEnv::PkTypeToString(pk_type)); } @@ -268,73 +261,28 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - //printf("pk:%lld\n", c->getInt(i)); - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_str_define.name) - { - //printf("%s:%s\n", col_str_define.name.c_str(), c->getDataAt(i).data); - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - //printf("%s:%lld\n", col_i8_define.name.c_str(), c->getInt(i)); - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_str_define.name, col_i8_define.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } { // test readRaw const auto & columns = store->getTableColumns(); BlockInputStreamPtr in = store->readRaw(*db_context, db_context->getSettingsRef(), columns, 1, /* keep_order= */ false)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_str_define.name) - { - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_str_define.name, col_i8_define.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } } CATCH @@ -412,15 +360,7 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 0); + ASSERT_INPUTSTREAM_NROWS(in, 0); } } CATCH @@ -499,38 +439,14 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - //printf("pk:%lld\n", c->getInt(i)); - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_str_define.name) - { - //printf("%s:%s\n", col_str_define.name.c_str(), c->getDataAt(i).data); - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - //printf("%s:%lld\n", col_i8_define.name.c_str(), c->getInt(i)); - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_str_define.name, col_i8_define.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } } CATCH @@ -572,24 +488,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } // Delete range [0, 64) const size_t num_deleted_rows = 64; @@ -611,25 +515,13 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - // Range after deletion is [64, 128) - ASSERT_EQ(c->getInt(i), i + Int64(num_deleted_rows)); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + // Range after deletion is [64, 128) + createColumn(createNumbers(num_deleted_rows, num_rows_write)), + })); } } CATCH @@ -698,24 +590,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, 3 * num_write_rows)), + })); } store = reload(); @@ -785,24 +665,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, 3 * num_write_rows)), + })); } // Read with version { @@ -818,24 +686,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - - ASSERT_EQ(num_rows_read, 2 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, 2 * num_write_rows)), + })); } } CATCH @@ -875,23 +731,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto & iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); i++) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - } - } - } - ASSERT_EQ(num_rows_read, 8UL); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, 8)), + })); } { @@ -916,30 +761,12 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - // block_num represents index of current segment - int block_num = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto & iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); i++) - { - if (iter.name == DMTestEnv::pk_name && block_num == 0) - { - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == DMTestEnv::pk_name && block_num == 1) - { - EXPECT_EQ(c->getInt(i), i + 4); - } - } - } - block_num++; - } - ASSERT_EQ(num_rows_read, 9UL); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, 9)), + })); } } CATCH @@ -979,13 +806,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, num_rows_tso1 + num_rows_tso2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_tso1 + num_rows_tso2); } { @@ -1004,13 +825,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, num_rows_tso1 + num_rows_tso2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_tso1 + num_rows_tso2); } { @@ -1029,13 +844,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, num_rows_tso1); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_tso1); } { @@ -1054,13 +863,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, 0UL); + ASSERT_INPUTSTREAM_NROWS(in, 0); } } CATCH @@ -1117,27 +920,14 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expect_pk = 0; - UInt64 expect_tso = tso1; - while (Block block = in->read()) - { - ASSERT_TRUE(block.has(DMTestEnv::pk_name)); - ASSERT_TRUE(block.has(VERSION_COLUMN_NAME)); - auto pk_c = block.getByName(DMTestEnv::pk_name); - auto v_c = block.getByName(VERSION_COLUMN_NAME); - for (size_t i = 0; i < block.rows(); ++i) - { - // std::cerr << "pk:" << pk_c.column->getInt(i) << ", ver:" << v_c.column->getInt(i) << std::endl; - ASSERT_EQ(pk_c.column->getInt(i), expect_pk++); - ASSERT_EQ(v_c.column->getUInt(i), expect_tso); - } - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32UL) << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + createColumns({ + createColumn(createNumbers(0, 32)), + createColumn(std::vector(32, tso1)), + })) + << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; } { @@ -1156,28 +946,14 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expect_pk = 0; - UInt64 expect_tso = tso1; - while (Block block = in->read()) - { - ASSERT_TRUE(block.has(DMTestEnv::pk_name)); - ASSERT_TRUE(block.has(VERSION_COLUMN_NAME)); - auto pk_c = block.getByName(DMTestEnv::pk_name); - auto v_c = block.getByName(VERSION_COLUMN_NAME); - for (size_t i = 0; i < block.rows(); ++i) - { - // std::cerr << "pk:" << pk_c.column->getInt(i) << ", ver:" << v_c.column->getInt(i) << std::endl; - ASSERT_EQ(pk_c.column->getInt(i), expect_pk++); - ASSERT_EQ(v_c.column->getUInt(i), expect_tso); - } - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32UL) << "Data [32, 128) after ingest with tso less than: " << tso2 - << " are erased, should only get [0, 32)"; + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + createColumns({ + createColumn(createNumbers(0, 32)), + createColumn(std::vector(32, tso1)), + })) + << fmt::format("Data [32, 128) after ingest with tso less than: {} are erased, should only get [0, 32)", tso2); } { @@ -1196,15 +972,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32UL + 16) << "The rows number after ingest with tso less than " << tso3 << " is not match"; + ASSERT_INPUTSTREAM_NROWS(in, 32 + 16) << fmt::format("The rows number after ingest with tso less than {} is not match", tso3); } { @@ -1223,13 +991,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32UL + (48 - 32) + (256UL - 80)) << "The rows number after ingest is not match"; + ASSERT_INPUTSTREAM_NROWS(in, 32 + (48 - 32) + (256 - 80)) << "The rows number after ingest is not match"; } { @@ -1250,13 +1012,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1UL); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, 2UL) << "The rows number of two point get is not match"; + ASSERT_INPUTSTREAM_NROWS(in, 2) << "The rows number of two point get is not match"; } } CATCH @@ -1313,27 +1069,14 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expect_pk = 0; - UInt64 expect_tso = tso1; - while (Block block = in->read()) - { - ASSERT_TRUE(block.has(DMTestEnv::pk_name)); - ASSERT_TRUE(block.has(VERSION_COLUMN_NAME)); - auto pk_c = block.getByName(DMTestEnv::pk_name); - auto v_c = block.getByName(VERSION_COLUMN_NAME); - for (size_t i = 0; i < block.rows(); ++i) - { - // std::cerr << "pk:" << pk_c.column->getInt(i) << ", ver:" << v_c.column->getInt(i) << std::endl; - ASSERT_EQ(pk_c.column->getInt(i), expect_pk++); - ASSERT_EQ(v_c.column->getUInt(i), expect_tso); - } - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32) << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + createColumns({ + createColumn(createNumbers(0, 32)), + createColumn(std::vector(32, tso1)), + })) + << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; } { @@ -1352,28 +1095,14 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expect_pk = 0; - UInt64 expect_tso = tso1; - while (Block block = in->read()) - { - ASSERT_TRUE(block.has(DMTestEnv::pk_name)); - ASSERT_TRUE(block.has(VERSION_COLUMN_NAME)); - auto pk_c = block.getByName(DMTestEnv::pk_name); - auto v_c = block.getByName(VERSION_COLUMN_NAME); - for (size_t i = 0; i < block.rows(); ++i) - { - // std::cerr << "pk:" << pk_c.column->getInt(i) << ", ver:" << v_c.column->getInt(i) << std::endl; - ASSERT_EQ(pk_c.column->getInt(i), expect_pk++); - ASSERT_EQ(v_c.column->getUInt(i), expect_tso); - } - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32) << "Data [32, 128) after ingest with tso less than: " << tso2 - << " are erased, should only get [0, 32)"; + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + createColumns({ + createColumn(createNumbers(0, 32)), + createColumn(std::vector(32, tso1)), + })) + << fmt::format("Data [32, 128) after ingest with tso less than: {} are erased, should only get [0, 32)", tso2); } { @@ -1392,13 +1121,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32 + 128 - 32) << "The rows number after ingest is not match"; + ASSERT_INPUTSTREAM_NROWS(in, 32 + 128 - 32) << "The rows number after ingest is not match"; } } CATCH @@ -1450,27 +1173,14 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expect_pk = 0; - UInt64 expect_tso = tso1; - while (Block block = in->read()) - { - ASSERT_TRUE(block.has(DMTestEnv::pk_name)); - ASSERT_TRUE(block.has(VERSION_COLUMN_NAME)); - auto pk_c = block.getByName(DMTestEnv::pk_name); - auto v_c = block.getByName(VERSION_COLUMN_NAME); - for (size_t i = 0; i < block.rows(); ++i) - { - // std::cerr << "pk:" << pk_c.column->getInt(i) << ", ver:" << v_c.column->getInt(i) << std::endl; - ASSERT_EQ(pk_c.column->getInt(i), expect_pk++); - ASSERT_EQ(v_c.column->getUInt(i), expect_tso); - } - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32) << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, VERSION_COLUMN_NAME}), + createColumns({ + createColumn(createNumbers(0, 32)), + createColumn(std::vector(32, tso1)), + })) + << "Data [32, 128) before ingest should be erased, should only get [0, 32)"; } { @@ -1489,13 +1199,7 @@ try /* expected_block_size= */ 1024); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - num_rows_read += block.rows(); - in->readSuffix(); - EXPECT_EQ(num_rows_read, 32) << "The rows number after ingest is not match"; + ASSERT_INPUTSTREAM_NROWS(in, 32) << "The rows number after ingest is not match"; } } CATCH @@ -1579,33 +1283,12 @@ try BlockInputStreamPtr in = ins[0]; LOG_FMT_TRACE(&Poco::Logger::get(GET_GTEST_FULL_NAME), "start to check data of [1,{}]", num_rows_write_in_total); - - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expected_row_pk = 1; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - for (size_t i = 0; i < c->size(); ++i) - { - auto expected = expected_row_pk++; - auto value = c->getInt(i); - if (value != expected) - { - // Convenient for debug. - EXPECT_EQ(expected, value); - } - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write_in_total); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(1, num_rows_write_in_total + 1)), + })); LOG_FMT_TRACE(&Poco::Logger::get(GET_GTEST_FULL_NAME), "done checking data of [1,{}]", num_rows_write_in_total); } @@ -1695,33 +1378,13 @@ try ASSERT_EQ(col.column_id, col_id_ddl); ASSERT_TRUE(col.type->equals(*col_type_after_ddl)); } - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (size_t i = 0; i < block.rows(); ++i) - { - for (auto && iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - // printf("pk:%lld\n", c->getInt(i)); - EXPECT_EQ(c->getInt(i), Int64(i)); - } - else if (iter.name == col_name_ddl) - { - // printf("%s:%lld\n", col_name_ddl.c_str(), c->getInt(i)); - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_ddl}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } } CATCH @@ -1800,26 +1463,12 @@ try const Block head = in->getHeader(); ASSERT_FALSE(head.has(col_name_to_drop)); } - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } } CATCH @@ -1902,35 +1551,14 @@ try ASSERT_TRUE(col.type->equals(*col_type_to_add)); } } - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_name_c1) - { - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - else if (iter.name == col_name_to_add) - { - EXPECT_EQ(c->getInt(i), 0); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_c1, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 0)), + })); } } CATCH @@ -1990,25 +1618,13 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(tmp.get(), 1.123456); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 1.123456)), + })); } } CATCH @@ -2068,25 +1684,13 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(tmp.get(), 1.123456); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 1.123456)), + })); } } CATCH @@ -2145,26 +1749,13 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(tmp.get(), 1.125); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 1.125)), + })); } } CATCH @@ -2223,26 +1814,13 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(tmp.get(), 1); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 1)), + })); } } CATCH @@ -2302,25 +1880,13 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(tmp.get(), 1); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 1)), + })); } } CATCH @@ -2378,22 +1944,17 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); i++) - { - EXPECT_EQ((*col.column)[i].get(), mydatetime_uint); // Timestamp for '1999-09-09 12:34:56' - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + std::vector datetime_data( + num_rows_write, + MyDateTime(1999, 9, 9, 12, 34, 56, 0).toPackedUInt()); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(/*data_type_args=*/std::make_tuple(0), datetime_data), + })); } } CATCH @@ -2452,25 +2013,13 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - EXPECT_EQ(tmp.get(), String("test_add_string_col")); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(Strings(num_rows_write, "test_add_string_col")), + })); } } CATCH @@ -2557,32 +2106,13 @@ try ASSERT_THROW(head.getByName(col_name_before_ddl), ::DB::Exception); } - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - //printf("pk:%lld\n", c->getInt(i)); - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_name_after_ddl) - { - //printf("col2:%s\n", c->getDataAt(i).data); - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_after_ddl}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } } CATCH @@ -2692,27 +2222,12 @@ try // check old col name is not exist ASSERT_THROW(head.getByName(col_name_before_ddl), ::DB::Exception); } - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == col_name_after_ddl) - { - //printf("col2:%s\n", c->getDataAt(i).data); - EXPECT_EQ(c->getInt(i), Int64(i)); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({col_name_after_ddl}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } { @@ -2755,27 +2270,12 @@ try // check old col name is not exist ASSERT_THROW(head.getByName(col_name_before_ddl), ::DB::Exception); } - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == col_name_after_ddl) - { - //printf("col2:%s\n", c->getDataAt(i).data); - EXPECT_EQ(c->getInt(i), Int64(i)); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write * 2); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({col_name_after_ddl}), + createColumns({ + createColumn(createNumbers(0, num_rows_write * 2)), + })); } } } @@ -2844,26 +2344,13 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - for (size_t i = 0; i < block.rows(); ++i) - { - Field tmp; - col.column->get(i, tmp); - // There is some loss of precision during the convertion, so we just do a rough comparison - EXPECT_FLOAT_EQ(std::abs(tmp.get()), 1.125); - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(std::vector{0}), + createColumn(std::vector{1.125}), + })); } { @@ -2900,21 +2387,14 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write * 2); - ASSERT_TRUE(block.has(col_name_to_add)); - const auto & col = block.getByName(col_name_to_add); - ASSERT_DATATYPE_EQ(col.type, col_type_to_add); - ASSERT_EQ(col.name, col_name_to_add); - Field tmp; - tmp = (*col.column)[0]; - EXPECT_FLOAT_EQ(tmp.get(), 1.125); // fill with default value - tmp = (*col.column)[1]; - EXPECT_FLOAT_EQ(tmp.get(), 3.1415); // keep the value we inserted - } - in->readSuffix(); + // FIXME!!! + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_name_to_add}), + createColumns({ + createColumn(std::vector{0, 1}), + createColumn(std::vector{1.125, 3.1415}), + })); } } CATCH @@ -2946,7 +2426,7 @@ try { const ColumnDefine col_str_define(2, "col2", std::make_shared()); const ColumnDefine col_i8_define(3, "i8", std::make_shared()); - size_t rowkey_column_size = 2; + const size_t rowkey_column_size = 2; { auto table_column_defines = DMTestEnv::getDefaultColumns(DMTestEnv::PkType::CommonHandle); table_column_defines->emplace_back(col_str_define); @@ -3018,71 +2498,42 @@ try /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - else if (iter.name == col_str_define.name) - { - //printf("%s:%s\n", col_str_define.name.c_str(), c->getDataAt(i).data); - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - //printf("%s:%lld\n", col_i8_define.name.c_str(), c->getInt(i)); - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, num_rows_write); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_i8_define.name, col_str_define.name}), + createColumns({ + createColumn(common_handle_coldata), + createColumn(createSignedNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + })); } { // test readRaw const auto & columns = store->getTableColumns(); BlockInputStreamPtr in = store->readRaw(*db_context, db_context->getSettingsRef(), columns, 1, /* keep_order= */ false)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - else if (iter.name == col_str_define.name) - { - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, num_rows_write); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_i8_define.name, col_str_define.name}), + createColumns({ + createColumn(common_handle_coldata), + createColumn(createSignedNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + })); } } CATCH @@ -3148,24 +2599,20 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - } - } - } - - ASSERT_EQ(num_rows_read, 3 * num_write_rows); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, 3 * num_write_rows); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } store = reload(table_column_defines, true, rowkey_column_size); @@ -3224,24 +2671,20 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - } - } - } - - ASSERT_EQ(num_rows_read, 3 * num_write_rows); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, 3 * num_write_rows); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } // Read with version { @@ -3257,24 +2700,20 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - } - } - } - - ASSERT_EQ(num_rows_read, 2 * num_write_rows); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, 2 * num_write_rows); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), 2 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH @@ -3283,7 +2722,7 @@ TEST_P(DeltaMergeStoreRWTest, DeleteReadWithCommonHandle) try { const size_t num_rows_write = 128; - size_t rowkey_column_size = 2; + const size_t rowkey_column_size = 2; { // Create a block with sequential Int64 handle in range [0, 128) auto table_column_difines = DMTestEnv::getDefaultColumns(DMTestEnv::PkType::CommonHandle); @@ -3317,42 +2756,27 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write); + // mock common handle + auto common_handle_coldata = []() { + auto tmp = createNumbers(0, num_rows_write); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } // Delete range [0, 64) const size_t num_deleted_rows = 64; { - WriteBufferFromOwnString ss; - DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - DB::EncodeInt64(Int64(0), ss); - DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - DB::EncodeInt64(Int64(0), ss); - RowKeyValue start(true, std::make_shared(ss.releaseStr())); - - ss.restart(); - DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - DB::EncodeInt64(Int64(num_deleted_rows), ss); - DB::EncodeUInt(static_cast(TiDB::CodecFlagInt), ss); - DB::EncodeInt64(Int64(num_deleted_rows), ss); - RowKeyValue end(true, std::make_shared(ss.str())); - RowKeyRange range(start, end, true, 2); + RowKeyValue start(true, std::make_shared(genMockCommonHandle(0, rowkey_column_size))); + RowKeyValue end(true, std::make_shared(genMockCommonHandle(num_deleted_rows, rowkey_column_size))); + RowKeyRange range(start, end, true, rowkey_column_size); store->deleteRange(*db_context, db_context->getSettingsRef(), range); } // Read after deletion @@ -3369,28 +2793,20 @@ try /* keep_order= */ false, /* is_fast_mode= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - // Range after deletion is [64, 128) - DMTestEnv::verifyClusteredIndexValue( - c->operator[](i).get(), - i + Int64(num_deleted_rows), - rowkey_column_size); - } - } - } - } - - ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + // mock common handle, data range after deletion is [64, 128) + auto common_handle_coldata = []() { + auto tmp = createNumbers(num_deleted_rows, num_rows_write); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), num_rows_write - num_deleted_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH @@ -3449,33 +2865,12 @@ try LOG_FMT_TRACE(&Poco::Logger::get(GET_GTEST_FULL_NAME), "start to check data of [1,{}]", num_rows_write_in_total); - size_t num_rows_read = 0; - in->readPrefix(); - Int64 expected_row_pk = 1; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - for (size_t i = 0; i < c->size(); ++i) - { - auto expected = expected_row_pk++; - auto value = c->getInt(i); - if (value != expected) - { - // Convenient for debug. - EXPECT_EQ(expected, value); - } - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write_in_total); - + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(1, num_rows_write_in_total + 1)), + })); LOG_FMT_TRACE(&Poco::Logger::get(GET_GTEST_FULL_NAME), "done checking data of [1,{}]", num_rows_write_in_total); } diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp index 8cfa7c07642..3206f179349 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store_for_fast_mode.cpp @@ -14,22 +14,20 @@ #include #include +#include #include +#include +#include /// This test file is mainly test on the correctness of read in fast mode. /// Because the basic functions are tested in gtest_dm_delta_merge_storage.cpp, we will not cover it here. namespace DB { -namespace FailPoints -{ -} // namespace FailPoints - namespace DM { namespace tests { - TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithoutRangeFilter) { /// test under only insert data (no update, no delete) with all range @@ -92,35 +90,14 @@ TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithoutRangeFilter) /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_str_define.name) - { - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_str_define.name, col_i8_define.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createNumberStrings(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write)), + })); } } @@ -173,22 +150,23 @@ TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithRangeFilter) } { // read all columns from store with row key range in fast mode + auto read_nums_limit = 64; WriteBufferFromOwnString start_key_ss; DB::EncodeInt64(0, start_key_ss); WriteBufferFromOwnString end_key_ss; - DB::EncodeInt64(64, end_key_ss); - + DB::EncodeInt64(read_nums_limit, end_key_ss); const auto & columns = store->getTableColumns(); + RowKeyRanges key_ranges{RowKeyRange( + RowKeyValue(false, std::make_shared(start_key_ss.releaseStr()), /*int_val_*/ 0), + RowKeyValue(false, std::make_shared(end_key_ss.releaseStr()), /*int_val_*/ read_nums_limit), + false, + store->getRowKeyColumnSize())}; BlockInputStreamPtr in = store->read(*db_context, db_context->getSettingsRef(), columns, - {RowKeyRange( - RowKeyValue(false, std::make_shared(start_key_ss.releaseStr()), 0), - RowKeyValue(false, std::make_shared(end_key_ss.releaseStr()), 64), - false, - store->getRowKeyColumnSize())}, + key_ranges, /* num_streams= */ 1, /* max_version= */ std::numeric_limits::max(), EMPTY_FILTER, @@ -196,35 +174,14 @@ TEST_P(DeltaMergeStoreRWTest, TestFastModeWithOnlyInsertWithRangeFilter) /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - else if (iter.name == col_str_define.name) - { - EXPECT_EQ(c->getDataAt(i), DB::toString(i)); - } - else if (iter.name == col_i8_define.name) - { - Int64 num = i * (i % 2 == 0 ? -1 : 1); - EXPECT_EQ(c->getInt(i), num); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 64); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, col_str_define.name, col_i8_define.name}), + createColumns({ + createColumn(createNumbers(0, read_nums_limit)), + createColumn(createNumberStrings(0, read_nums_limit)), + createColumn(createSignedNumbers(0, read_nums_limit)), + })); } } @@ -292,83 +249,57 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: case TestMode::V2_BlockOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_rows_read); - } - } - } - num_rows_read += block.rows(); - } + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, 3 * num_write_rows))})); break; } case TestMode::V2_FileOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_rows_read); - } - } - } - num_rows_read += block.rows(); - } + auto pk_coldata = []() { + std::vector res; + // first [0, 32) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [32, 64) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [64, 96) + tmp = createNumbers(2 * num_write_rows, 3 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } case TestMode::V2_Mix: { - int block_index = 0; - int begin_value = 0; // persist first, then memory, finally stable - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = num_write_rows * 2; - } - else if (block_index == 2) - { - begin_value = num_write_rows; - } - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + // persist first, then memory, finally stable + auto pk_coldata = []() { + std::vector res; + // first [0, 32) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [64, 96) + tmp = createNumbers(2 * num_write_rows, 3 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [32, 64) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } } - - in->readSuffix(); - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } CATCH @@ -439,82 +370,41 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: case TestMode::V2_BlockOnly: - { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - num_rows_read += block.rows(); - } - break; - } case TestMode::V2_FileOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_rows_read); - } - } - } - num_rows_read += block.rows(); - } + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, 3 * num_write_rows))})); break; } case TestMode::V2_Mix: { - int block_index = 0; - int begin_value = 0; - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = num_write_rows * 2; - } - else if (block_index == 2) - { - begin_value = num_write_rows; - } - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + auto pk_coldata = []() { + std::vector res; + // first [0, 32) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [64, 96) + tmp = createNumbers(2 * num_write_rows, 3 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [32, 64) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); break; } } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } CATCH @@ -564,7 +454,6 @@ try file_ids.insert(file_ids.cend(), file_ids3.begin(), file_ids3.end()); store->ingestFiles(dm_context, range, file_ids, false); store->write(*db_context, db_context->getSettingsRef(), block2); - break; } } @@ -587,84 +476,41 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: case TestMode::V2_BlockOnly: - { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - num_rows_read += block.rows(); - } - break; - } case TestMode::V2_FileOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_rows_read); - } - } - } - num_rows_read += block.rows(); - } + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, 3 * num_write_rows))})); break; } case TestMode::V2_Mix: { - int block_index = 0; - int begin_value = 0; - - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = num_write_rows * 2; - } - else if (block_index == 2) - { - begin_value = num_write_rows; - } - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + auto pk_coldata = []() { + std::vector res; + // first [0, 32) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [64, 96) + tmp = createNumbers(2 * num_write_rows, 3 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [32, 64) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); break; } } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } CATCH @@ -739,27 +585,10 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, 3 * num_write_rows))})); } } CATCH @@ -835,109 +664,70 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: case TestMode::V2_BlockOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (i < Int64(num_write_rows / 2)) - { - ASSERT_EQ(c->getInt(i), i); - } - else if (i < Int64(2.5 * num_write_rows)) - { - ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); - } - else - { - ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); - } - } - } - } - num_rows_read += block.rows(); - } + auto pk_coldata = []() { + std::vector res; + // first [0, 128) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128, 256) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // the pk is sorted by flush cache + std::sort(res.begin(), res.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } case TestMode::V2_FileOnly: { - auto block_index = 0; - auto begin_value = 0; - - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = num_write_rows; - } - else if (block_index == 2) - { - begin_value = num_write_rows / 2; - } - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + auto pk_coldata = []() { + std::vector res; + // first [0, 128) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128, 256) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } case TestMode::V2_Mix: { - auto block_index = 0; - auto begin_value = num_write_rows; - - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = 0; - } - else if (block_index == 2) - { - begin_value = num_write_rows / 2; - } - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } - + auto pk_coldata = []() { + std::vector res; + // first [128, 256) + auto tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [0, 128) + tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } } - - in->readSuffix(); - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } } CATCH @@ -1021,27 +811,11 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); // filter del mark = 1， thus just read the insert data before delete - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, num_rows_write))})); } store->flushCache(*db_context, RowKeyRange::newAll(store->isCommonHandle(), store->getRowKeyColumnSize())); @@ -1063,27 +837,10 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, num_rows_write))})); } } CATCH @@ -1095,7 +852,7 @@ try const size_t num_rows_write = 128; { // Create a block with sequential Int64 handle in range [0, 128) - Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); switch (mode) { @@ -1126,25 +883,10 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, num_rows_write))})); } // Delete range [0, 64) const size_t num_deleted_rows = 64; @@ -1166,28 +908,11 @@ try /* keep_order = */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - // filter del mark = 1， thus just read the insert data before delete - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, num_rows_write))})); } } CATCH @@ -1198,7 +923,7 @@ try const size_t num_rows_write = 128; { // Create a block with sequential Int64 handle in range [0, 128) - Block block = DMTestEnv::prepareSimpleWriteBlock(0, 128, false); + Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); switch (mode) { @@ -1243,27 +968,12 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_deleted_rows); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + auto pk_coldata = createNumbers(num_deleted_rows, num_rows_write); + ASSERT_EQ(pk_coldata.size(), num_rows_write - num_deleted_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); } } CATCH @@ -1346,105 +1056,70 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - in->readPrefix(); switch (mode) { case TestMode::V1_BlockOnly: case TestMode::V2_BlockOnly: { - while (Block block = in->read()) - { - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (i < Int64(num_write_rows / 2)) - { - ASSERT_EQ(c->getInt(i), i); - } - else if (i < Int64(2.5 * num_write_rows)) - { - ASSERT_EQ(c->getInt(i), (i - num_write_rows / 2) / 2 + num_write_rows / 2); - } - else - { - ASSERT_EQ(c->getInt(i), (i - num_write_rows * 2) + num_write_rows); - } - } - } - } - num_rows_read += block.rows(); - } + auto pk_coldata = []() { + std::vector res; + // first [0, 128) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128, 256) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // the pk is sorted by flush cache + std::sort(res.begin(), res.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } case TestMode::V2_FileOnly: { - auto block_index = 0; - auto begin_value = 0; - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = num_write_rows; - } - else if (block_index == 2) - { - begin_value = num_write_rows / 2; - } - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + auto pk_coldata = []() { + std::vector res; + // first [0, 128) + auto tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128, 256) + tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } case TestMode::V2_Mix: { - auto block_index = 0; - auto begin_value = num_write_rows; - while (Block block = in->read()) - { - if (block_index == 1) - { - begin_value = 0; - } - else if (block_index == 2) - { - begin_value = num_write_rows / 2; - } - for (auto && iter : block) - { - if (iter.name == DMTestEnv::pk_name) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - ASSERT_EQ(c->getInt(i), i + begin_value); - } - } - } - num_rows_read += block.rows(); - block_index += 1; - } + auto pk_coldata = []() { + std::vector res; + // first [128, 256) + auto tmp = createNumbers(num_write_rows, 2 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [0, 128) + tmp = createNumbers(0, num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + // then [128/2, 128 * 1.5) + tmp = createNumbers(num_write_rows / 2, 1.5 * num_write_rows); + res.insert(res.end(), tmp.begin(), tmp.end()); + return res; + }(); + ASSERT_EQ(pk_coldata.size(), 3 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({DMTestEnv::pk_name}), createColumns({createColumn(pk_coldata)})); break; } } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 3 * num_write_rows); } // Read with version in normal case @@ -1461,27 +1136,10 @@ try /* keep_order= */ false, /* is_raw_read= */ false, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i + num_write_rows / 2); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, 1.5 * num_write_rows); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(num_write_rows / 2, 2 * num_write_rows))})); } } CATCH @@ -1530,27 +1188,12 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto && iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); ++i) - { - if (iter.name == DMTestEnv::pk_name) - { - ASSERT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } // Delete range [0, 64) @@ -1570,7 +1213,7 @@ try { const auto & columns = store->getTableColumns(); ColumnDefines real_columns; - for (auto & col : columns) + for (const auto & col : columns) { if (col.name != EXTRA_HANDLE_COLUMN_NAME) { @@ -1589,19 +1232,10 @@ try /* keep_order= */ false, /* is_raw_read= */ true, /* expected_block_size= */ 1024)[0]; - size_t num_rows_read = 0; - - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - - ASSERT_EQ(num_rows_read, num_rows_write - num_deleted_rows); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write - num_deleted_rows); } } CATCH } // namespace tests } // namespace DM -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index 23062f4ffdf..4b29e17ec23 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -23,7 +24,10 @@ #include #include #include +#include +#include +#include #include namespace DB @@ -38,7 +42,7 @@ namespace DM { namespace tests { -TEST(DMFileWriterFlags_test, SetClearFlags) +TEST(DMFileWriterFlagsTest, SetClearFlags) { using Flags = DMFileWriter::Flags; @@ -83,12 +87,12 @@ String paramToString(const ::testing::TestParamInfo & info) using DMFileBlockOutputStreamPtr = std::shared_ptr; using DMFileBlockInputStreamPtr = std::shared_ptr; -class DMFile_Test +class DMFileTest : public DB::base::TiFlashStorageTestBasic , public testing::WithParamInterface { public: - DMFile_Test() + DMFileTest() : dm_file(nullptr) {} @@ -103,7 +107,7 @@ class DMFile_Test auto configuration = (mode == DMFileMode::DirectoryChecksum ? std::make_optional() : std::nullopt); parent_path = TiFlashStorageTestBasic::getTemporaryPath(); - path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFile_Test", false)); + path_pool = std::make_unique(db_context->getPathPool().withTable("test", "DMFileTest", false)); storage_pool = std::make_unique(*db_context, /*ns_id*/ 100, *path_pool, "test.t1"); dm_file = DMFile::create(1, parent_path, single_file_mode, std::move(configuration)); table_columns_ = std::make_shared(); @@ -160,7 +164,7 @@ class DMFile_Test }; -TEST_P(DMFile_Test, WriteRead) +TEST_P(DMFileTest, WriteRead) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -198,23 +202,12 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } /// Test restore the file from disk and read @@ -237,28 +230,17 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } } CATCH -TEST_P(DMFile_Test, GcFlag) +TEST_P(DMFileTest, GcFlag) try { // clean @@ -319,10 +301,10 @@ try } CATCH -/// DMFile_Test.InterruptedDrop_0 and InterruptedDrop_1 test that if deleting file +/// DMFileTest.InterruptedDrop_0 and InterruptedDrop_1 test that if deleting file /// is interrupted by accident, we can safely ignore those broken files. -TEST_P(DMFile_Test, InterruptedDrop_0) +TEST_P(DMFileTest, InterruptedDrop0) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -349,23 +331,12 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } FailPointHelper::enableFailPoint(FailPoints::exception_before_dmfile_remove_encryption); @@ -388,7 +359,7 @@ try } CATCH -TEST_P(DMFile_Test, InterruptedDrop_1) +TEST_P(DMFileTest, InterruptedDrop1) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -415,23 +386,12 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + })); } FailPointHelper::enableFailPoint(FailPoints::exception_before_dmfile_remove_from_disk); @@ -456,7 +416,7 @@ CATCH /// Test reading rows with some filters -TEST_P(DMFile_Test, ReadFilteredByHandle) +TEST_P(DMFileTest, ReadFilteredByHandle) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -482,13 +442,14 @@ try stream->writeSuffix(); } - HandleRanges ranges; - ranges.emplace_back(HandleRange{0, span_per_part}); // only first part - ranges.emplace_back(HandleRange{800, num_rows_write}); - ranges.emplace_back(HandleRange{256, 700}); // - ranges.emplace_back(HandleRange::newNone()); // none - ranges.emplace_back(HandleRange{0, num_rows_write}); // full range - ranges.emplace_back(HandleRange::newAll()); // full range + HandleRanges ranges{ + HandleRange{0, span_per_part}, // only first part + HandleRange{800, num_rows_write}, + HandleRange{256, 700}, // + HandleRange::newNone(), // none + HandleRange{0, num_rows_write}, // full range + HandleRange::newAll(), // full range + }; auto test_read_range = [&](const HandleRange & range) { // Test read DMFileBlockInputStreamBuilder builder(dbContext()); @@ -496,29 +457,17 @@ try .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::fromHandleRange(range)}); // Filtered by read_range - Int64 num_rows_read = 0; - stream->readPrefix(); Int64 expect_first_pk = int(std::floor(std::max(0, range.start) / span_per_part)) * span_per_part; Int64 expect_last_pk = std::min(num_rows_write, // int(std::ceil(std::min(num_rows_write, range.end) / span_per_part)) * span_per_part + (range.end % span_per_part ? span_per_part : 0)); - Int64 cur_pk = expect_first_pk; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++) - << "range: " << range.toDebugString() << ", cur_pk: " << cur_pk << ", first pk: " << expect_first_pk; - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, expect_last_pk - expect_first_pk) // - << "range: " << range.toDebugString() // - << ", first: " << expect_first_pk << ", last: " << expect_last_pk; + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(expect_first_pk, expect_last_pk)), + })) + << fmt::format("range: {}, first: {}, last: {}", range.toDebugString(), expect_first_pk, expect_last_pk); }; for (const auto & range : ranges) @@ -548,7 +497,7 @@ RSOperatorPtr toRSFilter(const ColumnDefine & cd, const HandleRange & range) } } // namespace -TEST_P(DMFile_Test, ReadFilteredByRoughSetFilter) +TEST_P(DMFileTest, ReadFilteredByRoughSetFilter) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -583,13 +532,14 @@ try stream->writeSuffix(); } - HandleRanges ranges; - ranges.emplace_back(HandleRange{0, span_per_part}); // only first part - ranges.emplace_back(HandleRange{800, num_rows_write}); - ranges.emplace_back(HandleRange{256, 700}); // - ranges.emplace_back(HandleRange::newNone()); // none - ranges.emplace_back(HandleRange{0, num_rows_write}); // full range - ranges.emplace_back(HandleRange::newAll()); // full range + HandleRanges ranges{ + HandleRange{0, span_per_part}, // only first part + HandleRange{800, num_rows_write}, + HandleRange{256, 700}, // + HandleRange::newNone(), // none + HandleRange{0, num_rows_write}, // full range + HandleRange::newAll(), // full range + }; auto test_read_filter = [&](const HandleRange & range) { // Filtered by rough set filter auto filter = toRSFilter(i64_cd, range); @@ -600,29 +550,17 @@ try .setRSOperator(filter) // Filtered by rough set filter .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - Int64 num_rows_read = 0; - stream->readPrefix(); Int64 expect_first_pk = int(std::floor(std::max(0, range.start) / span_per_part)) * span_per_part; Int64 expect_last_pk = std::min(num_rows_write, // int(std::ceil(std::min(num_rows_write, range.end) / span_per_part)) * span_per_part + (range.end % span_per_part ? span_per_part : 0)); - Int64 cur_pk = expect_first_pk; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(i64_cd.name)); - auto col = in.getByName(i64_cd.name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++) - << "range: " << range.toDebugString() << ", cur_pk: " << cur_pk << ", first pk: " << expect_first_pk; - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, expect_last_pk - expect_first_pk) // - << "range: " << range.toDebugString() // - << ", first: " << expect_first_pk << ", last: " << expect_last_pk; + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(expect_first_pk, expect_last_pk)), + })) + << fmt::format("range: {}, first: {}, last: {}", range.toDebugString(), expect_first_pk, expect_last_pk); }; for (const auto & range : ranges) @@ -642,7 +580,7 @@ try CATCH // Test rough filter with some unsupported operations -TEST_P(DMFile_Test, ReadFilteredByRoughSetFilterWithUnsupportedOperation) +TEST_P(DMFileTest, ReadFilteredByRoughSetFilterWithUnsupportedOperation) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -695,25 +633,15 @@ try .setRSOperator(filter) // Filtered by rough set filter .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - Int64 num_rows_read = 0; - stream->readPrefix(); Int64 expect_first_pk = 0; Int64 expect_last_pk = num_rows_should_read; - Int64 cur_pk = expect_first_pk; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(i64_cd.name)); - auto col = in.getByName(i64_cd.name); - auto & c = col.column; - for (size_t j = 0; j < c->size(); j++) - { - EXPECT_EQ(c->getInt(j), cur_pk++) << "cur_pk: " << cur_pk << ", first pk: " << expect_first_pk; - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, expect_last_pk - expect_first_pk) // - << "first: " << expect_first_pk << ", last: " << expect_last_pk; + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(expect_first_pk, expect_last_pk)), + })) + << fmt::format("first: {}, last: {}", expect_first_pk, expect_last_pk); }; for (size_t i = 0; i < filters.size(); ++i) @@ -736,7 +664,7 @@ try } CATCH -TEST_P(DMFile_Test, ReadFilteredByPackIndices) +TEST_P(DMFileTest, ReadFilteredByPackIndices) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -780,9 +708,8 @@ try .setReadPacks(id_set_ptr) // filter by pack index .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - Int64 num_rows_read = 0; - stream->readPrefix(); - Int64 expect_first_pk = 0, expect_last_pk = 0; + Int64 expect_first_pk = 0; + Int64 expect_last_pk = 0; if (id_set_ptr && !id_set_ptr->empty()) { expect_first_pk = *(id_set_ptr->begin()) * span_per_part; @@ -794,24 +721,13 @@ try // not filter if it is nullptr expect_last_pk = num_rows_write; } - - Int64 cur_pk = expect_first_pk; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), cur_pk++) // - << "test index: " << test_index // - << ", cur_pk: " << cur_pk << ", first pk: " << expect_first_pk; - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, expect_last_pk - expect_first_pk) // - << "test index: " << test_index << ", first: " << expect_first_pk << ", last: " << expect_last_pk; + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(createNumbers(expect_first_pk, expect_last_pk)), + })) + << fmt::format("test index: {}, first: {}, last: {}", test_index, expect_first_pk, expect_last_pk); }; for (size_t test_index = 0; test_index <= test_sets.size(); test_index++) { @@ -831,7 +747,7 @@ CATCH /// Test reading different column types -TEST_P(DMFile_Test, NumberTypes) +TEST_P(DMFileTest, NumberTypes) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -870,37 +786,24 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(i64_col.name)); - ASSERT_TRUE(in.has(f64_col.name)); - auto i64_c = in.getByName(i64_col.name).column; - auto f64_c = in.getByName(f64_col.name).column; - ASSERT_EQ(i64_c->size(), f64_c->size()); - for (size_t i = 0; i < i64_c->size(); i++) - { - EXPECT_EQ(i64_c->getInt(i), cur_pk++); - Field f = (*f64_c)[i]; - EXPECT_FLOAT_EQ(f.get(), 0.125); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name, i64_col.name, f64_col.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 0.125)), + })); } } CATCH -TEST_P(DMFile_Test, StringType) +TEST_P(DMFileTest, StringType) try { auto cols = DMTestEnv::getDefaultColumns(); // Prepare columns - ColumnDefine fixed_str_col(2, "str", typeFromString("FixedString(5)")); + ColumnDefine fixed_str_col(2, "str", typeFromString("String")); cols->push_back(fixed_str_col); reload(cols); @@ -929,28 +832,18 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(fixed_str_col.name)); - auto col = in.getByName(fixed_str_col.name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - EXPECT_EQ(value.get(), "hello"); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name, fixed_str_col.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, "hello")), + })); } } CATCH -TEST_P(DMFile_Test, NullableType) +TEST_P(DMFileTest, NullableType) try { auto cols = DMTestEnv::getDefaultColumns(); @@ -966,7 +859,7 @@ try Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); // Half of the column are filled by NULL auto col = nullable_col.type->createColumn(); - for (size_t i = 0; i < 64; i++) + for (size_t i = 0; i < num_rows_write / 2; i++) col->insert(toField(Int64(i))); for (size_t i = 64; i < num_rows_write; i++) col->insertDefault(); @@ -990,59 +883,35 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - ASSERT_TRUE(in.has(nullable_col.name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - auto ncol = in.getByName(nullable_col.name); - auto & nc = ncol.column; - for (size_t i = 0; i < c->size(); i++) - { - // check nullable column - { - const auto nested_col = typeid_cast(nc.get()); - auto nested = nested_col->getNestedColumnPtr(); - if (cur_pk < 64) - { - EXPECT_FALSE(nested_col->isNullAt(i)); - EXPECT_EQ(nested->getInt(i), cur_pk); - } - else - { - EXPECT_TRUE(nested_col->isNullAt(i)); - } - } - // check pk - EXPECT_EQ(c->getInt(i), cur_pk++); - } - num_rows_read += in.rows(); - } - ASSERT_EQ(num_rows_read, num_rows_write); - stream->readSuffix(); + std::vector nullable_coldata = createNumbers(0, num_rows_write / 2); + nullable_coldata.resize(num_rows_write); + std::vector null_map(num_rows_write, 0); + std::fill(null_map.begin() + num_rows_write / 2, null_map.end(), 1); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name, nullable_col.name}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createNullableColumn(nullable_coldata, null_map), + })); } } CATCH INSTANTIATE_TEST_CASE_P(DTFileMode, // - DMFile_Test, + DMFileTest, testing::Values(DMFileMode::SingleFile, DMFileMode::DirectoryLegacy, DMFileMode::DirectoryChecksum), paramToString); /// DMFile test for clustered index -class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic - , // - public testing::WithParamInterface +class DMFileClusteredIndexTest + : public DB::base::TiFlashStorageTestBasic + , public testing::WithParamInterface { public: - DMFile_Clustered_Index_Test() + DMFileClusteredIndexTest() : dm_file(nullptr) {} @@ -1107,7 +976,7 @@ class DMFile_Clustered_Index_Test : public DB::base::TiFlashStorageTestBasic size_t rowkey_column_size = 2; }; -TEST_P(DMFile_Clustered_Index_Test, WriteRead) +TEST_P(DMFileClusteredIndexTest, WriteRead) try { auto cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); @@ -1150,28 +1019,25 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 cur_pk = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - DMTestEnv::verifyClusteredIndexValue((*c)[i].get(), cur_pk++, rowkey_column_size); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + // mock common handle + auto common_handle_coldata = [this]() { + std::vector int_coldata = createNumbers(0, num_rows_write); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH -TEST_P(DMFile_Clustered_Index_Test, ReadFilteredByHandle) +TEST_P(DMFileClusteredIndexTest, ReadFilteredByHandle) try { auto cols = DMTestEnv::getDefaultColumns(is_common_handle ? DMTestEnv::PkType::CommonHandle : DMTestEnv::PkType::HiddenTiDBRowID); @@ -1237,41 +1103,37 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols, RowKeyRanges{range.range}); // Filtered by read_range - - Int64 num_rows_read = 0; - stream->readPrefix(); Int64 expect_first_pk = int(std::floor(std::max(0, range.start) / span_per_part)) * span_per_part; Int64 expect_last_pk = std::min(num_rows_write, // int(std::ceil(std::min(num_rows_write, range.end) / span_per_part)) * span_per_part + (range.end % span_per_part ? span_per_part : 0)); - Int64 cur_pk = expect_first_pk; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(DMTestEnv::pk_name)); - auto col = in.getByName(DMTestEnv::pk_name); - auto & c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - DMTestEnv::verifyClusteredIndexValue((*c)[i].get(), cur_pk++, rowkey_column_size); - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, expect_last_pk - expect_first_pk) // - << "range: " << range.range.toDebugString() // - << ", first: " << expect_first_pk << ", last: " << expect_last_pk; + // mock common handle + auto common_handle_coldata = [this, expect_first_pk, expect_last_pk]() { + std::vector int_coldata = createNumbers(expect_first_pk, expect_last_pk); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), expect_last_pk - expect_first_pk); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })) + << fmt::format("range: {}, first: {}, last: {}", range.range.toDebugString(), expect_first_pk, expect_last_pk); } } CATCH INSTANTIATE_TEST_CASE_P(DTFileMode, // - DMFile_Clustered_Index_Test, + DMFileClusteredIndexTest, testing::Values(DMFile::Mode::FOLDER, DMFile::Mode::SINGLE_FILE), paramToString); /// DDL test cases -class DMFile_DDL_Test : public DMFile_Test +class DMFileDDLTest : public DMFileTest { public: /// Write some data into DMFile. @@ -1291,20 +1153,18 @@ class DMFile_DDL_Test : public DMFile_Test { // Prepare write Block block = DMTestEnv::prepareSimpleWriteBlock(0, num_rows_write, false); - - auto col = i8_col.type->createColumn(); - for (size_t i = 0; i < num_rows_write; i++) + if (!i8_is_nullable) { - Field field; // Null by default - if (!i8_is_nullable || (i8_is_nullable && i < num_rows_write / 2)) - field = toField(Int64(i) * (-1 * (i % 2))); - col->insert(field); + auto i8_coldata = createSignedNumbers(0, num_rows_write); + block.insert(createColumn(i8_coldata, i8_col.name, i8_col.id)); + } + else + { + auto c = getExpectedI8Column(num_rows_write); + c.name = i8_col.name; + c.column_id = i8_col.id; + block.insert(c); } - block.insert(ColumnWithTypeAndName{ - std::move(col), - i8_col.type, - i8_col.name, - i8_col.id}); block.insert(DB::tests::createColumn( std::vector(num_rows_write, 0.125), @@ -1320,9 +1180,21 @@ class DMFile_DDL_Test : public DMFile_Test return {num_rows_write, *cols_before_ddl}; } } + + static ColumnWithTypeAndName getExpectedI8Column(size_t num_rows_write) + { + auto i8_coldata = createSignedNumbers(0, num_rows_write); + std::vector nullmap(num_rows_write, 0); + for (size_t i = 0; i < num_rows_write / 2; ++i) + { + i8_coldata[i] = 0; + nullmap[i] = 1; + } + return createNullableColumn(i8_coldata, nullmap); + } }; -TEST_P(DMFile_DDL_Test, AddColumn) +TEST_P(DMFileDDLTest, AddColumn) try { // Prepare some data before ddl @@ -1345,71 +1217,22 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols_after_ddl, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 row_number = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has("i8")); - ASSERT_TRUE(in.has("f64")); - ASSERT_TRUE(in.has(new_s_col.name)); - ASSERT_TRUE(in.has(new_i_col_with_default.name)); - { - auto col = in.getByName(new_s_col.name); - EXPECT_EQ(col.column_id, new_s_col.id); - EXPECT_TRUE(col.type->equals(*new_s_col.type)); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - ASSERT_EQ(value.getType(), Field::Types::String); - // Empty default value - ASSERT_EQ(value, new_s_col.type->getDefault()); - } - } - { - auto col = in.getByName(new_i_col_with_default.name); - EXPECT_EQ(col.column_id, new_i_col_with_default.id); - EXPECT_TRUE(col.type->equals(*new_i_col_with_default.type)); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - ASSERT_EQ(c->getInt(i), 5); // Should fill with default value - } - } - { - // Check old columns before ddl - auto col = in.getByName("i8"); - EXPECT_EQ(col.column_id, 2L); - EXPECT_TRUE(col.type->equals(*typeFromString("Int8"))); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - EXPECT_EQ(c->getInt(i), Int64(row_number * (-1 * (row_number % 2)))); - row_number++; - } - } - { - auto col = in.getByName("f64"); - EXPECT_EQ(col.column_id, 3L); - EXPECT_TRUE(col.type->equals(*typeFromString("Float64"))); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - EXPECT_FLOAT_EQ(value.get(), 0.125); - } - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({"i8", "f64", new_s_col.name, new_i_col_with_default.name}), + createColumns({ + // old cols + createColumn(createSignedNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, 0.125)), + // new cols + createColumn(Strings(num_rows_write, "")), // filled with empty + createColumn(std::vector(num_rows_write, 5)), // filled with default value + })); } } CATCH -TEST_P(DMFile_DDL_Test, UpcastColumnType) +TEST_P(DMFileDDLTest, UpcastColumnType) try { // Prepare some data before ddl @@ -1431,56 +1254,26 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols_after_ddl, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 row_number = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(new_col.name)); - ASSERT_TRUE(!in.has("i8")); - ASSERT_TRUE(in.has("f64")); - { - auto col = in.getByName(new_col.name); - EXPECT_EQ(col.column_id, new_col.id); - EXPECT_TRUE(col.type->equals(*new_col.type)); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - auto value = c->getInt(Int64(i)); - ASSERT_EQ(value, (Int64)(row_number * (-1 * (row_number % 2)))); - row_number++; - } - } - { - // Check old columns before ddl - auto col = in.getByName("f64"); - EXPECT_EQ(col.column_id, 3L); - EXPECT_TRUE(col.type->equals(*typeFromString("Float64"))); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - EXPECT_DOUBLE_EQ(value.get(), 0.125); - } - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({new_col.name, "f64"}), + createColumns({ + createColumn(createSignedNumbers(0, num_rows_write)), + // old cols + createColumn(std::vector(num_rows_write, 0.125)), + })); } } CATCH -TEST_P(DMFile_DDL_Test, NotNullToNull) +TEST_P(DMFileDDLTest, NotNullToNull) try { // Prepare some data before ddl const auto [num_rows_write, cols_before_ddl] = prepareSomeDataToDMFile(); // Mock that we achange a column type from int8 -> Nullable(int32), and its name to "i8_new" after ddl - auto cols_after_ddl = std::make_shared(); - *cols_after_ddl = cols_before_ddl; + auto cols_after_ddl = std::make_shared(cols_before_ddl); const ColumnDefine old_col = cols_before_ddl[3]; ASSERT_TRUE(old_col.type->equals(*typeFromString("Int8"))); ColumnDefine new_col = old_col; @@ -1494,48 +1287,18 @@ try auto stream = builder .setColumnCache(column_cache_) .build(dm_file, *cols_after_ddl, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 row_number = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(new_col.name)); - ASSERT_TRUE(!in.has("i8")); - ASSERT_TRUE(in.has("f64")); - { - auto col = in.getByName(new_col.name); - EXPECT_EQ(col.column_id, new_col.id); - EXPECT_TRUE(col.type->equals(*new_col.type)); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - auto value = (*c)[i]; - ASSERT_FALSE(value.isNull()); - ASSERT_EQ(value, (Int64)(row_number * (-1 * (row_number % 2)))); - row_number++; - } - } - { - auto col = in.getByName("f64"); - EXPECT_EQ(col.column_id, 3L); - EXPECT_TRUE(col.type->equals(*typeFromString("Float64"))); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - EXPECT_DOUBLE_EQ(value.get(), 0.125); - } - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({new_col.name, "f64"}), + createColumns({ + createNullableColumn(createSignedNumbers(0, num_rows_write), /*null_map=*/std::vector(num_rows_write, 0)), + createColumn(std::vector(num_rows_write, 0.125)), + })); } } CATCH -TEST_P(DMFile_DDL_Test, NullToNotNull) +TEST_P(DMFileDDLTest, NullToNotNull) try { // Prepare some data before ddl @@ -1558,57 +1321,23 @@ try .setColumnCache(column_cache_) .build(dm_file, *cols_after_ddl, RowKeyRanges{RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - stream->readPrefix(); - Int64 row_number = 0; - while (Block in = stream->read()) - { - ASSERT_TRUE(in.has(new_col.name)); - ASSERT_TRUE(!in.has("i8")); - ASSERT_TRUE(in.has("f64")); - { - auto col = in.getByName(new_col.name); - EXPECT_EQ(col.column_id, new_col.id); - EXPECT_TRUE(col.type->equals(*new_col.type)); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - auto value = (*c)[i]; - if (i < num_rows_write / 2) - { - ASSERT_FALSE(value.isNull()) << " at row: " << i; - ASSERT_EQ(value, (Int64)(row_number * (-1 * (row_number % 2)))) << " at row: " << i; - } - else - { - ASSERT_FALSE(value.isNull()) << " at row: " << i; - ASSERT_EQ(value, (Int64)0) << " at row: " << i; - } - row_number++; - } - } - { - // Check old columns before ddl - auto col = in.getByName("f64"); - EXPECT_EQ(col.column_id, 3L); - EXPECT_TRUE(col.type->equals(*typeFromString("Float64"))); - auto c = col.column; - for (size_t i = 0; i < c->size(); i++) - { - Field value = (*c)[i]; - EXPECT_DOUBLE_EQ(value.get(), 0.125); - } - } - num_rows_read += in.rows(); - } - stream->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + auto i32_coldata = createSignedNumbers(0, num_rows_write); + for (size_t i = 0; i < num_rows_write / 2; ++i) + i32_coldata[i] = 0; + ASSERT_INPUTSTREAM_COLS_UR( + stream, + Strings({DMTestEnv::pk_name, new_col.name, "f64"}), + createColumns({ + createColumn(createNumbers(0, num_rows_write)), + createColumn(i32_coldata), + createColumn(std::vector(num_rows_write, 0.125)), + })); } } CATCH INSTANTIATE_TEST_CASE_P(DTFileMode, // - DMFile_DDL_Test, + DMFileDDLTest, testing::Values(DMFileMode::SingleFile, DMFileMode::DirectoryLegacy, DMFileMode::DirectoryChecksum), paramToString); diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp index f066237d8e2..345ecff9286 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -125,9 +126,7 @@ bool checkMatch( const ColumnDefine & col_to_read = check_pk ? getExtraHandleColumnDefine(is_common_handle) : cd; auto streams = store->read(context, context.getSettingsRef(), {col_to_read}, {all_range}, 1, std::numeric_limits::max(), filter, name, false); - streams[0]->readPrefix(); - auto rows = streams[0]->read().rows(); - streams[0]->readSuffix(); + auto rows = getInputStreamNRows(streams[0]); store->drop(); return rows != 0; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index deec5646d33..f27abf157dc 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -139,14 +140,7 @@ try { // read written data (only in delta) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -157,14 +151,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } } @@ -180,14 +167,7 @@ try { // read written data (both in delta and stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write + num_rows_write_2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write + num_rows_write_2); } { @@ -198,14 +178,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write + num_rows_write_2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write + num_rows_write_2); } } } @@ -235,15 +208,8 @@ try { auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); // only write two visible pks - ASSERT_EQ(num_rows_read, 2); + ASSERT_INPUTSTREAM_NROWS(in, 2); } } CATCH @@ -278,14 +244,7 @@ try { // read written data (only in delta) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows); } { @@ -298,14 +257,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows); } } @@ -323,14 +275,7 @@ try { // read written data (both in delta and stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows_2); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows_2); } { @@ -342,14 +287,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows_2); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows_2); } } } @@ -372,18 +310,6 @@ try segment->write(dmContext(), block); }; - auto check_rows = [&](size_t expected_rows) { - auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expected_rows); - }; - { // check segment segment->check(dmContext(), "test"); @@ -391,12 +317,16 @@ try // Thread A write_rows(100); - check_rows(100); + ASSERT_INPUTSTREAM_NROWS( + segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}), + 100); auto snap = segment->createSnapshot(dmContext(), false, CurrentMetrics::DT_SnapshotOfRead); // Thread B write_rows(100); - check_rows(200); + ASSERT_INPUTSTREAM_NROWS( + segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}), + 200); // Thread A { @@ -408,14 +338,7 @@ try {}, std::numeric_limits::max(), DEFAULT_BLOCK_SIZE); - int num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, 100); + ASSERT_INPUTSTREAM_NROWS(in, 100); } } CATCH @@ -444,15 +367,7 @@ try auto get_rows = [&](const RowKeyRange & range) { auto in = segment->getInputStream(dmContext(), *tableColumns(), {range}); - in->readPrefix(); - size_t rows = 0; - while (Block block = in->read()) - { - rows += block.rows(); - } - in->readSuffix(); - - return rows; + return getInputStreamNRows(in); }; // First place the block packs, so that we can only place DeleteRange below. @@ -522,14 +437,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -550,21 +458,10 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(1), 99); - } - } - } - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn({0, 99})})); } // For the case that apply merge delta after delete range, we ensure that data on disk are compacted @@ -572,15 +469,8 @@ try { // read raw after delete range auto in = segment->getInputStreamRaw(dmContext(), *tableColumns()); - in->readPrefix(); - size_t num_rows = 0; - while (Block block = in->read()) - { - num_rows += block.rows(); - } - in->readSuffix(); // Only 2 rows are left on disk, others are compacted. - ASSERT_EQ(num_rows, 2UL); + ASSERT_INPUTSTREAM_NROWS(in, 2); } } CATCH @@ -600,14 +490,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -636,21 +519,10 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(1), 99); - } - } - } - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn({0, 99})})); } // For the case that apply merge delta after delete range, we ensure that data on disk are compacted @@ -658,15 +530,8 @@ try { // read raw after delete range auto in = segment->getInputStreamRaw(dmContext(), *tableColumns()); - in->readPrefix(); - size_t num_rows = 0; - while (Block block = in->read()) - { - num_rows += block.rows(); - } - in->readSuffix(); // Only 2 rows are left on disk, others are compacted. - ASSERT_EQ(num_rows, 2UL); + ASSERT_INPUTSTREAM_NROWS(in, 2); } } CATCH @@ -695,14 +560,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -723,21 +581,10 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(1), 99); - } - } - } - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn({0, 99})})); } // For the case that apply merge delta after delete range, we ensure that data on disk are compacted @@ -745,15 +592,8 @@ try { // read raw after delete range auto in = segment->getInputStreamRaw(dmContext(), *tableColumns()); - size_t num_rows = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows += block.rows(); - } - in->readSuffix(); // Only 2 rows are left on disk, others are compacted. - ASSERT_EQ(num_rows, 2UL); + ASSERT_INPUTSTREAM_NROWS(in, 2); } } CATCH @@ -795,23 +635,10 @@ try // Read after deletion // The deleted range has no overlap with current data, so there should be no change auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write); - for (auto & iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); i++) - { - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(i), i); - } - } - } - } - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, num_rows_write))})); } { @@ -828,22 +655,10 @@ try // Read after deletion // The deleted range has overlap range [63, 64) with current data, so the record with Handle 63 should be deleted auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 1); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(62), 62); - } - EXPECT_EQ(c->size(), 63UL); - } - } - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(createNumbers(0, 63))})); } { @@ -858,21 +673,14 @@ try { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 32); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(1), 32); - } - } - } - in->readSuffix(); + + std::vector pk_coldata{0}; + auto tmp = createNumbers(32, 63); + pk_coldata.insert(pk_coldata.end(), tmp.begin(), tmp.end()); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); } { @@ -888,21 +696,13 @@ try { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 32); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 0); - EXPECT_EQ(c->getInt(1), 32); - } - } - } - in->readSuffix(); + std::vector pk_coldata{0}; + auto tmp = createNumbers(32, 63); + pk_coldata.insert(pk_coldata.end(), tmp.begin(), tmp.end()); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); } { @@ -918,20 +718,11 @@ try { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 33); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 32); - } - } - } - in->readSuffix(); + std::vector pk_coldata = createNumbers(32, 63); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); } { @@ -945,23 +736,14 @@ try { // Read after new write auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 33 + 7); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - EXPECT_EQ(c->getInt(0), 9); - EXPECT_EQ(c->getInt(6), 15); - EXPECT_EQ(c->getInt(7), 32); - EXPECT_EQ(c->getInt(block.rows() - 1), 62); - } - } - } - in->readSuffix(); + std::vector pk_coldata = createNumbers(9, 16); + auto tmp = createNumbers(32, 63); + pk_coldata.insert(pk_coldata.end(), tmp.begin(), tmp.end()); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(pk_coldata)})); } } CATCH @@ -985,15 +767,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } const auto old_range = segment->getRowKeyRange(); @@ -1011,29 +785,9 @@ try EXPECT_EQ(*s2_range.end.value, *old_range.end.value); // TODO check segment epoch is increase - size_t num_rows_seg1 = 0; - size_t num_rows_seg2 = 0; - { - { - auto in = segment->getInputStream(dmContext(), *tableColumns(), {segment->getRowKeyRange()}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_seg1 += block.rows(); - } - in->readSuffix(); - } - { - auto in = new_segment->getInputStream(dmContext(), *tableColumns(), {new_segment->getRowKeyRange()}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_seg2 += block.rows(); - } - in->readSuffix(); - } - ASSERT_EQ(num_rows_seg1 + num_rows_seg2, num_rows_write); - } + size_t num_rows_seg1 = getInputStreamNRows(segment->getInputStream(dmContext(), *tableColumns(), {segment->getRowKeyRange()})); + size_t num_rows_seg2 = getInputStreamNRows(new_segment->getInputStream(dmContext(), *tableColumns(), {new_segment->getRowKeyRange()})); + ASSERT_EQ(num_rows_seg1 + num_rows_seg2, num_rows_write); // delete rows in the right segment { @@ -1052,15 +806,8 @@ try // TODO check segment epoch is increase } { - size_t num_rows_read = 0; auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, num_rows_seg1); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_seg1); } } } @@ -1192,7 +939,7 @@ try const time_t start_time = std::time(nullptr); - auto temp = std::vector(); + std::vector temp; for (;;) { { @@ -1227,26 +974,11 @@ try { // Read after writing auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - for (auto & iter : block) - { - auto c = iter.column; - for (size_t i = 0; i < c->size(); i++) - { - if (iter.name == DMTestEnv::pk_name) - { - auto expect = temp.at(i + num_rows_read); - EXPECT_EQ(c->getInt(Int64(i)), expect); - } - } - } - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_batches_written * (num_rows_per_write - 2), num_rows_read); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({createColumn(temp)})) + << fmt::format("num_batches_written={} num_rows_per_write={}", num_batches_written, num_rows_per_write); } { @@ -1375,15 +1107,8 @@ try }; auto read_rows = [&](const SegmentPtr & segment) { - size_t rows = 0; auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(false, 1)}); - in->readPrefix(); - while (Block block = in->read()) - { - rows += block.rows(); - } - in->readSuffix(); - return rows; + return getInputStreamNRows(in); }; write_100_rows(segment); @@ -1526,27 +1251,13 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *columns_to_read, {RowKeyRange::newAll(false, 1)}); - // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(column_name_i8_to_i32)); - const ColumnWithTypeAndName & col = block.getByName(column_name_i8_to_i32); - ASSERT_DATATYPE_EQ(col.type, column_i32_after_ddl.type); - ASSERT_EQ(col.name, column_i32_after_ddl.name); - ASSERT_EQ(col.column_id, column_i32_after_ddl.id); - for (size_t i = 0; i < block.rows(); ++i) - { - auto value = col.column->getInt(i); - const auto expected = static_cast((i % 2 == 0 ? -1 : 1) * i); - ASSERT_EQ(value, expected) << "at row: " << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, column_i32_after_ddl.name}), + createColumns({// + createColumn(createNumbers(0, num_rows_write)), + createColumn(createSignedNumbers(0, num_rows_write))})); } @@ -1574,33 +1285,18 @@ try auto in = segment->getInputStream(dmContext(), *columns_to_read, {RowKeyRange::newAll(false, 1)}); // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(column_name_i8_to_i32)); - const ColumnWithTypeAndName & col = block.getByName(column_name_i8_to_i32); - ASSERT_DATATYPE_EQ(col.type, column_i32_after_ddl.type); - ASSERT_EQ(col.name, column_i32_after_ddl.name); - ASSERT_EQ(col.column_id, column_i32_after_ddl.id); - for (size_t i = 0; i < block.rows(); ++i) - { - auto value = col.column->getInt(i); - auto expected = 0; - if (i < num_rows_write / 2) - expected = static_cast((i % 2 == 0 ? -1 : 1) * i); - else - { - auto r = i - num_rows_write / 2; - expected = static_cast((r % 2 == 0 ? -1 : 1) * r); - } - // std::cerr << " row: " << i << " "<< value << std::endl; - ASSERT_EQ(value, expected) << "at row: " << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + // [0, 50) is the old signed values, [50, 100) is replaced by newer written values + std::vector i32_columndata = createSignedNumbers(0, num_rows_write / 2); + auto tmp = createSignedNumbers(0, num_rows_write * 2 - num_rows_write / 2); + i32_columndata.insert(i32_columndata.end(), tmp.begin(), tmp.end()); + ASSERT_EQ(i32_columndata.size(), 2 * num_rows_write); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, column_i32_after_ddl.name}), + createColumns({// + createColumn(createNumbers(0, 2 * num_rows_write)), + createColumn(i32_columndata)})); } // Flush cache and apply delta-merge, then read again @@ -1615,33 +1311,17 @@ try auto in = segment->getInputStream(dmContext(), *columns_to_read, {RowKeyRange::newAll(false, 1)}); // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(column_name_i8_to_i32)); - const ColumnWithTypeAndName & col = block.getByName(column_name_i8_to_i32); - ASSERT_DATATYPE_EQ(col.type, column_i32_after_ddl.type); - ASSERT_EQ(col.name, column_i32_after_ddl.name); - ASSERT_EQ(col.column_id, column_i32_after_ddl.id); - for (size_t i = 0; i < block.rows(); ++i) - { - auto value = col.column->getInt(i); - auto expected = 0; - if (i < num_rows_write / 2) - expected = static_cast((i % 2 == 0 ? -1 : 1) * i); - else - { - auto r = i - num_rows_write / 2; - expected = static_cast((r % 2 == 0 ? -1 : 1) * r); - } - // std::cerr << " row: " << i << " "<< value << std::endl; - ASSERT_EQ(value, expected) << "at row: " << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + std::vector i32_columndata = createSignedNumbers(0, num_rows_write / 2); + auto tmp = createSignedNumbers(0, num_rows_write * 2 - num_rows_write / 2); + i32_columndata.insert(i32_columndata.end(), tmp.begin(), tmp.end()); + ASSERT_EQ(i32_columndata.size(), 2 * num_rows_write); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, column_i32_after_ddl.name}), + createColumns({// + createColumn(createNumbers(0, 2 * num_rows_write)), + createColumn(i32_columndata)})); } } CATCH @@ -1696,23 +1376,12 @@ try auto in = segment->getInputStream(dmContext(), *columns_after_ddl, {RowKeyRange::newAll(false, 1)}); // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - const ColumnWithTypeAndName & col = block.getByName(new_column_define.name); - ASSERT_TRUE(col.type->equals(*new_column_define.type)); - ASSERT_EQ(col.name, new_column_define.name); - ASSERT_EQ(col.column_id, new_column_define.id); - for (size_t i = 0; i < block.rows(); ++i) - { - auto value = col.column->getInt(i); - ASSERT_EQ(value, new_column_default_value_int) << "at row:" << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, new_column_define.name}), + createColumns({// + createColumn(createNumbers(0, num_rows_write)), + createColumn(std::vector(num_rows_write, new_column_default_value_int))})); } @@ -1742,33 +1411,19 @@ try auto in = segment->getInputStream(dmContext(), *columns_after_ddl, {RowKeyRange::newAll(false, 1)}); // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(new_column_name)); - const ColumnWithTypeAndName & col = block.getByName(new_column_name); - ASSERT_DATATYPE_EQ(col.type, new_column_define.type); - ASSERT_EQ(col.name, new_column_define.name); - ASSERT_EQ(col.column_id, new_column_define.id); - for (size_t i = 0; i < block.rows(); ++i) - { - int8_t value = col.column->getInt(i); - int8_t expected = 0; - if (i < num_rows_write / 2) - expected = new_column_default_value_int; - else - { - auto r = i - num_rows_write / 2; - expected = static_cast((r % 2 == 0 ? -1 : 1) * r); - } - // std::cerr << " row: " << i << " "<< value << std::endl; - ASSERT_EQ(value, expected) << "at row: " << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + // First 50 values are default value + std::vector i8_columndata(num_rows_write / 2, new_column_default_value_int); + // then fill with signed number sequence + auto tmp = createSignedNumbers(0, num_rows_write * 2 - num_rows_write / 2); + i8_columndata.insert(i8_columndata.end(), tmp.begin(), tmp.end()); + ASSERT_EQ(i8_columndata.size(), 2 * num_rows_write); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, new_column_define.name}), + createColumns({// + createColumn(createNumbers(0, 2 * num_rows_write)), + createColumn(i8_columndata)})); } // Flush cache and apply delta-merge, then read again @@ -1783,33 +1438,19 @@ try auto in = segment->getInputStream(dmContext(), *columns_after_ddl, {RowKeyRange::newAll(false, 1)}); // check that we can read correct values - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - ASSERT_TRUE(block.has(new_column_name)); - const ColumnWithTypeAndName & col = block.getByName(new_column_name); - ASSERT_DATATYPE_EQ(col.type, new_column_define.type); - ASSERT_EQ(col.name, new_column_define.name); - ASSERT_EQ(col.column_id, new_column_define.id); - for (size_t i = 0; i < block.rows(); ++i) - { - int8_t value = col.column->getInt(i); - int8_t expected = 0; - if (i < num_rows_write / 2) - expected = new_column_default_value_int; - else - { - auto r = i - num_rows_write / 2; - expected = static_cast((r % 2 == 0 ? -1 : 1) * r); - } - // std::cerr << " row: " << i << " "<< value << std::endl; - ASSERT_EQ(value, expected) << "at row: " << i; - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, (size_t)(num_rows_write * 2)); + // First 50 values are default value + std::vector i8_columndata(num_rows_write / 2, new_column_default_value_int); + // then fill with signed number sequence + auto tmp = createSignedNumbers(0, num_rows_write * 2 - num_rows_write / 2); + i8_columndata.insert(i8_columndata.end(), tmp.begin(), tmp.end()); + ASSERT_EQ(i8_columndata.size(), 2 * num_rows_write); + + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name, new_column_define.name}), + createColumns({// + createColumn(createNumbers(0, 2 * num_rows_write)), + createColumn(i8_columndata)})); } } CATCH diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index 6359a3db184..1944a1bf7e1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -17,7 +17,9 @@ #include #include #include +#include #include +#include #include #include @@ -28,12 +30,10 @@ namespace DM { namespace tests { -class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic +class SegmentCommonHandleTest : public DB::base::TiFlashStorageTestBasic { public: - Segment_Common_Handle_test() - : storage_pool() - {} + SegmentCommonHandleTest() {} public: static void SetUpTestCase() {} @@ -95,10 +95,10 @@ class Segment_Common_Handle_test : public DB::base::TiFlashStorageTestBasic // the segment we are going to test SegmentPtr segment; bool is_common_handle = true; - size_t rowkey_column_size = 2; + const size_t rowkey_column_size = 2; }; -TEST_F(Segment_Common_Handle_test, WriteRead) +TEST_F(SegmentCommonHandleTest, WriteRead) try { const size_t num_rows_write = 100; @@ -115,11 +115,11 @@ try // write to segment segment->write(dmContext(), block); // estimate segment - auto estimatedRows = segment->getEstimatedRows(); - ASSERT_EQ(estimatedRows, block.rows()); + auto estimated_rows = segment->getEstimatedRows(); + ASSERT_EQ(estimated_rows, block.rows()); - auto estimatedBytes = segment->getEstimatedBytes(); - ASSERT_EQ(estimatedBytes, block.bytes()); + auto estimated_bytes = segment->getEstimatedBytes(); + ASSERT_EQ(estimated_bytes, block.bytes()); } { @@ -131,14 +131,7 @@ try { // read written data (only in delta) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -149,14 +142,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } } @@ -180,14 +166,7 @@ try { // read written data (both in delta and stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write + num_rows_write_2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write + num_rows_write_2); } { @@ -198,20 +177,13 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write + num_rows_write_2); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write + num_rows_write_2); } } } CATCH -TEST_F(Segment_Common_Handle_test, WriteReadMultiRange) +TEST_F(SegmentCommonHandleTest, WriteReadMultiRange) try { const size_t num_rows_write = 100; @@ -228,11 +200,11 @@ try // write to segment segment->write(dmContext(), block); // estimate segment - auto estimatedRows = segment->getEstimatedRows(); - ASSERT_EQ(estimatedRows, block.rows()); + auto estimated_rows = segment->getEstimatedRows(); + ASSERT_EQ(estimated_rows, block.rows()); - auto estimatedBytes = segment->getEstimatedBytes(); - ASSERT_EQ(estimatedBytes, block.bytes()); + auto estimated_bytes = segment->getEstimatedBytes(); + ASSERT_EQ(estimated_bytes, block.bytes()); } { @@ -240,23 +212,17 @@ try segment->check(dmContext(), "test"); } - RowKeyRanges read_ranges; - read_ranges.emplace_back(RowKeyRange::fromHandleRange(HandleRange(0, 10), true)); - read_ranges.emplace_back(RowKeyRange::fromHandleRange(HandleRange(20, 30), true)); - read_ranges.emplace_back(RowKeyRange::fromHandleRange(HandleRange(110, 130), true)); + RowKeyRanges read_ranges{ + (RowKeyRange::fromHandleRange(HandleRange(0, 10), true)), + (RowKeyRange::fromHandleRange(HandleRange(20, 30), true)), + (RowKeyRange::fromHandleRange(HandleRange(110, 130), true)), + }; const size_t expect_read_rows = 20; { // Round 1 { // read written data (only in delta) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows); } { @@ -269,14 +235,7 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows); } } @@ -302,14 +261,7 @@ try { // read written data (both in delta and stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows_2); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows_2); } { @@ -321,20 +273,13 @@ try { // read written data (only in stable) auto in = segment->getInputStream(dmContext(), *tableColumns(), read_ranges); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expect_read_rows_2); + ASSERT_INPUTSTREAM_NROWS(in, expect_read_rows_2); } } } CATCH -class SegmentDeletion_Common_Handle_test : public Segment_Common_Handle_test +class SegmentDeletion_Common_Handle_test : public SegmentCommonHandleTest , // public testing::WithParamInterface> { @@ -363,14 +308,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -389,21 +327,21 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](1).get(), 99, rowkey_column_size); - } - } - } - in->readSuffix(); + const size_t nrows_after_delete = 2; + // mock common handle + auto common_handle_coldata = [this]() { + auto tmp = std::vector{0, 99}; + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), nrows_after_delete); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH @@ -431,14 +369,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -465,21 +396,21 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](1).get(), 99, rowkey_column_size); - } - } - } - in->readSuffix(); + const size_t nrows_after_delete = 2; + // mock common handle + auto common_handle_coldata = [this]() { + auto tmp = std::vector{0, 99}; + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), nrows_after_delete); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH @@ -524,14 +455,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } { @@ -550,21 +474,21 @@ try { // read after delete range auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), 2UL); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](1).get(), 99, rowkey_column_size); - } - } - } - in->readSuffix(); + const size_t nrows_after_delete = 2; + // mock common handle + auto common_handle_coldata = [this]() { + auto tmp = std::vector{0, 99}; + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), nrows_after_delete); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH @@ -573,7 +497,7 @@ INSTANTIATE_TEST_CASE_P(WhetherReadOrMergeDeltaBeforeDeleteRange, SegmentDeletion_Common_Handle_test, testing::Combine(testing::Bool(), testing::Bool())); -TEST_F(Segment_Common_Handle_test, DeleteRead) +TEST_F(SegmentCommonHandleTest, DeleteRead) try { const size_t num_rows_write = 64; @@ -599,6 +523,7 @@ try // Test delete range [70, 100) segment->write(dmContext(), {DMTestEnv::getRowKeyRangeForClusteredIndex(70, 100, rowkey_column_size)}); // flush segment + segment->flushCache(dmContext()); segment = segment->mergeDelta(dmContext(), tableColumns()); } @@ -606,29 +531,27 @@ try // Read after deletion // The deleted range has no overlap with current data, so there should be no change auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write); - for (auto & iter : block) - { - auto c = iter.column; - for (Int64 i = 0; i < Int64(c->size()); i++) - { - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](i).get(), i, rowkey_column_size); - } - } - } - } - in->readSuffix(); + // mock common handle + auto common_handle_coldata = [this]() { + auto tmp = createNumbers(0, num_rows_write); + Strings res; + std::transform(tmp.begin(), tmp.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), num_rows_write); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } { // Test delete range [63, 70) segment->write(dmContext(), {DMTestEnv::getRowKeyRangeForClusteredIndex(63, 70, rowkey_column_size)}); // flush segment + segment->flushCache(dmContext()); segment = segment->mergeDelta(dmContext(), tableColumns()); } @@ -636,49 +559,49 @@ try // Read after deletion // The deleted range has overlap range [63, 64) with current data, so the record with Handle 63 should be deleted auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 1); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](62).get(), 62, rowkey_column_size); - } - EXPECT_EQ(c->size(), 63UL); - } - } - in->readSuffix(); + // mock common handle + auto common_handle_coldata = [this]() { + std::vector int_coldata = createNumbers(0, 63); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_EQ(common_handle_coldata.size(), num_rows_write - 1); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } { // Test delete range [1, 32) segment->write(dmContext(), {DMTestEnv::getRowKeyRangeForClusteredIndex(1, 32, rowkey_column_size)}); // flush segment + segment->flushCache(dmContext()); segment = segment->mergeDelta(dmContext(), tableColumns()); } { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 32); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](1).get(), 32, rowkey_column_size); - } - } - } - in->readSuffix(); + // mock common handle + auto common_handle_coldata = [this]() { + // the result should be [0, 32,33,34,...62] + std::vector int_coldata{0}; + auto tmp = createNumbers(32, 63); + int_coldata.insert(int_coldata.end(), tmp.begin(), tmp.end()); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } { @@ -686,27 +609,28 @@ try // delete should be idempotent segment->write(dmContext(), {DMTestEnv::getRowKeyRangeForClusteredIndex(1, 32, rowkey_column_size)}); // flush segment + segment->flushCache(dmContext()); segment = segment->mergeDelta(dmContext(), tableColumns()); } { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 32); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 0, rowkey_column_size); - DMTestEnv::verifyClusteredIndexValue(c->operator[](1).get(), 32, rowkey_column_size); - } - } - } - in->readSuffix(); + // mock common handle + auto common_handle_coldata = [this]() { + std::vector int_coldata{0}; + auto tmp = createNumbers(32, 63); + int_coldata.insert(int_coldata.end(), tmp.begin(), tmp.end()); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } { @@ -714,31 +638,31 @@ try // There is an overlap range [0, 1) segment->write(dmContext(), {DMTestEnv::getRowKeyRangeForClusteredIndex(0, 2, rowkey_column_size)}); // flush segment + segment->flushCache(dmContext()); segment = segment->mergeDelta(dmContext(), tableColumns()); } { // Read after deletion auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - ASSERT_EQ(block.rows(), num_rows_write - 33); - for (auto & iter : block) - { - auto c = iter.column; - if (iter.name == DMTestEnv::pk_name) - { - DMTestEnv::verifyClusteredIndexValue(c->operator[](0).get(), 32, rowkey_column_size); - } - } - } - in->readSuffix(); + // mock common handle + auto common_handle_coldata = [this]() { + std::vector int_coldata = createNumbers(32, 63); + Strings res; + std::transform(int_coldata.begin(), int_coldata.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } } CATCH -TEST_F(Segment_Common_Handle_test, Split) +TEST_F(SegmentCommonHandleTest, Split) try { const size_t num_rows_write = 100; @@ -759,15 +683,7 @@ try { // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } const auto old_range = segment->getRowKeyRange(); @@ -785,29 +701,9 @@ try EXPECT_EQ(*s2_range.end.value, *old_range.end.value); // TODO check segment epoch is increase - size_t num_rows_seg1 = 0; - size_t num_rows_seg2 = 0; - { - { - auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_seg1 += block.rows(); - } - in->readSuffix(); - } - { - auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_seg2 += block.rows(); - } - in->readSuffix(); - } - ASSERT_EQ(num_rows_seg1 + num_rows_seg2, num_rows_write); - } + size_t num_rows_seg1 = getInputStreamNRows(segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)})); + size_t num_rows_seg2 = getInputStreamNRows(segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)})); + ASSERT_EQ(num_rows_seg1 + num_rows_seg2, num_rows_write); // merge segments // TODO: enable merge test! @@ -822,21 +718,14 @@ try // TODO check segment epoch is increase } { - size_t num_rows_read = 0; auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - EXPECT_EQ(num_rows_read, num_rows_write); + ASSERT_INPUTSTREAM_NROWS(in, num_rows_write); } } } CATCH -TEST_F(Segment_Common_Handle_test, Restore) +TEST_F(SegmentCommonHandleTest, Restore) try { // compare will compares the given segments. @@ -935,7 +824,7 @@ try } CATCH -TEST_F(Segment_Common_Handle_test, MassiveSplit) +TEST_F(SegmentCommonHandleTest, MassiveSplit) try { Settings settings = dmContext().db_context.getSettings(); @@ -991,26 +880,19 @@ try { // Read after writing auto in = segment->getInputStream(dmContext(), *tableColumns(), {RowKeyRange::newAll(is_common_handle, rowkey_column_size)}); - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - for (auto & iter : block) - { - auto c = iter.column; - for (size_t i = 0; i < c->size(); i++) - { - if (iter.name == DMTestEnv::pk_name) - { - auto expect = temp.at(i + num_rows_read); - DMTestEnv::verifyClusteredIndexValue(c->operator[](Int64(i)).get(), expect, rowkey_column_size); - } - } - } - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_batches_written * (num_rows_per_write - 2), num_rows_read); + ASSERT_EQ(temp.size(), num_batches_written * (num_rows_per_write - 2)); + // mock common handle + auto common_handle_coldata = [this, &temp]() { + Strings res; + std::transform(temp.begin(), temp.end(), std::back_inserter(res), [this](Int64 v) { return genMockCommonHandle(v, rowkey_column_size); }); + return res; + }(); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({DMTestEnv::pk_name}), + createColumns({ + createColumn(common_handle_coldata), + })); } { diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index f929e153847..ac03b509f18 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include @@ -57,13 +58,14 @@ namespace tests TEST(StorageDeltaMergeTest, ReadWriteCase1) try { + size_t num_rows_write = 100; // prepare block data Block sample; sample.insert(DB::tests::createColumn( - createNumbers(0, 100, /*reversed*/ true), + createNumbers(0, num_rows_write, /*reversed*/ true), "col1")); sample.insert(DB::tests::createColumn( - Strings(100, "a"), + Strings(num_rows_write, "a"), "col2")); Context ctx = DMTestEnv::getContext(); @@ -120,30 +122,10 @@ try BlockInputStreams ins = storage->read(column_names, query_info, ctx, stage2, 8192, 1); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - in->readPrefix(); - - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - for (auto & iter : block) - { - auto c = iter.column; - for (unsigned int i = 0; i < c->size(); i++) - { - if (iter.name == "col1") - { - ASSERT_EQ(c->getInt(i), i); - } - else if (iter.name == "col2") - { - ASSERT_EQ(c->getDataAt(i), "a"); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, sample.rows()); + ASSERT_INPUTSTREAM_BLOCK_UR( + in, + Block({createColumn(createNumbers(0, num_rows_write), "col1"), + createColumn(Strings(num_rows_write, "a"), "col2")})); auto store_status = storage->status(); Block status = store_status->read(); @@ -157,7 +139,7 @@ try } else if (col_name->getDataAt(i) == String("total_rows")) { - EXPECT_EQ(col_value->getDataAt(i), String(DB::toString(num_rows_read))); + EXPECT_EQ(col_value->getDataAt(i), String(DB::toString(num_rows_write))); } } auto delta_store = storage->getStore(); @@ -167,7 +149,7 @@ try { total_segment_rows += stat.rows; } - EXPECT_EQ(total_segment_rows, num_rows_read); + EXPECT_EQ(total_segment_rows, num_rows_write); storage->drop(); // remove the storage from TiFlash context manually storage->removeFromTMTContext(); @@ -609,12 +591,13 @@ TEST(StorageDeltaMergeTest, ReadExtraPhysicalTableID) try { // prepare block data + size_t num_rows_write = 100; Block sample; sample.insert(DB::tests::createColumn( - createNumbers(0, 100, /*reversed*/ true), + createNumbers(0, num_rows_write, /*reversed*/ true), "col1")); sample.insert(DB::tests::createColumn( - Strings(100, "a"), + Strings(num_rows_write, "a"), "col2")); constexpr TiDB::TableID table_id = 1; const String table_name = fmt::format("t_{}", table_id); @@ -676,38 +659,14 @@ try BlockInputStreams ins = storage->read(read_columns, query_info, ctx, stage2, 8192, 1); ASSERT_EQ(ins.size(), 1); BlockInputStreamPtr in = ins[0]; - in->readPrefix(); + ASSERT_INPUTSTREAM_BLOCK_UR( + in, + Block({ + createColumn(createNumbers(0, num_rows_write), "col1"), + createConstColumn>(num_rows_write, table_id, EXTRA_TABLE_ID_COLUMN_NAME), + createColumn(Strings(num_rows_write, "a"), "col2"), + })); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - ASSERT_EQ(block.getByPosition(1).name, EXTRA_TABLE_ID_COLUMN_NAME); - num_rows_read += block.rows(); - for (auto & iter : block) - { - auto c = iter.column; - for (unsigned int i = 0; i < c->size(); i++) - { - if (iter.name == "col1") - { - ASSERT_EQ(c->getInt(i), i); - } - else if (iter.name == "col2") - { - ASSERT_EQ(c->getDataAt(i), "a"); - } - else if (iter.name == EXTRA_TABLE_ID_COLUMN_NAME) - { - Field res; - c->get(i, res); - ASSERT_TRUE(!res.isNull()); - ASSERT_EQ(res.get(), table_id); - } - } - } - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, sample.rows()); storage->drop(); // remove the storage from TiFlash context manually storage->removeFromTMTContext(); @@ -801,15 +760,7 @@ try query_info.mvcc_query_info = std::make_unique(ctx.getSettingsRef().resolve_locks, std::numeric_limits::max()); Names read_columns = {"col1", EXTRA_TABLE_ID_COLUMN_NAME, "col2"}; BlockInputStreams ins = storage->read(read_columns, query_info, ctx, stage2, 8192, 1); - BlockInputStreamPtr in = ins[0]; - in->readPrefix(); - size_t num_rows_read = 0; - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - return num_rows_read; + return getInputStreamNRows(ins[0]); }; // create table diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp index c676f2e08d5..a46df50b9a4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_segment_test_basic.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace DB @@ -63,30 +64,14 @@ size_t SegmentTestBasic::getSegmentRowNumWithoutMVCC(PageId segment_id) { auto segment = segments[segment_id]; auto in = segment->getInputStreamRaw(dmContext(), *tableColumns()); - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - return num_rows_read; + return getInputStreamNRows(in); } size_t SegmentTestBasic::getSegmentRowNum(PageId segment_id) { auto segment = segments[segment_id]; auto in = segment->getInputStream(dmContext(), *tableColumns(), {segment->getRowKeyRange()}); - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - return num_rows_read; + return getInputStreamNRows(in); } void SegmentTestBasic::checkSegmentRow(PageId segment_id, size_t expected_row_num) @@ -94,15 +79,7 @@ void SegmentTestBasic::checkSegmentRow(PageId segment_id, size_t expected_row_nu auto segment = segments[segment_id]; // read written data auto in = segment->getInputStream(dmContext(), *tableColumns(), {segment->getRowKeyRange()}); - - size_t num_rows_read = 0; - in->readPrefix(); - while (Block block = in->read()) - { - num_rows_read += block.rows(); - } - in->readSuffix(); - ASSERT_EQ(num_rows_read, expected_row_num); + ASSERT_INPUTSTREAM_NROWS(in, expected_row_num); } std::optional SegmentTestBasic::splitSegment(PageId segment_id) diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp index 16b1729bea1..d96085942eb 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include namespace DB { @@ -77,7 +79,7 @@ BlockInputStreamPtr genInputStream(const BlocksList & blocks, const ColumnDefine } // namespace -TEST(VersionFilter_test, MVCC) +TEST(VersionFilterTest, MVCC) { BlocksList blocks; @@ -93,48 +95,27 @@ TEST(VersionFilter_test, MVCC) { auto in = genInputStream(blocks, columns, 40, false); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "Flash"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"Flash"})})); } { auto in = genInputStream(blocks, columns, 30, false); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 0UL); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({})})); } { auto in = genInputStream(blocks, columns, 20, false); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "world"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"world"})})); } { auto in = genInputStream(blocks, columns, 10, false); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "hello"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"hello"})})); } { auto in = genInputStream(blocks, columns, 9, false); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 0UL); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({})})); } } -TEST(VersionFilter_test, MVCCCommonHandle) +TEST(VersionFilterTest, MVCCCommonHandle) { BlocksList blocks; @@ -150,48 +131,27 @@ TEST(VersionFilter_test, MVCCCommonHandle) { auto in = genInputStream(blocks, columns, 40, true); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "Flash"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"Flash"})})); } { auto in = genInputStream(blocks, columns, 30, true); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 0UL); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({})})); } { auto in = genInputStream(blocks, columns, 20, true); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "world"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"world"})})); } { auto in = genInputStream(blocks, columns, 10, true); - in->readPrefix(); - Block block = in->read(); - auto col = block.getByName(str_col_name); - auto val = col.column->getDataAt(0); - ASSERT_EQ(val, "hello"); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({"hello"})})); } { auto in = genInputStream(blocks, columns, 9, true); - in->readPrefix(); - Block block = in->read(); - ASSERT_EQ(block.rows(), 0UL); - in->readSuffix(); + ASSERT_INPUTSTREAM_COLS_UR(in, Strings({str_col_name}), createColumns({createColumn({})})); } } -TEST(VersionFilter_test, Compact) +TEST(VersionFilterTest, Compact) { // TODO: currently it just test data statistics, add test for data correctness BlocksList blocks; @@ -294,7 +254,7 @@ TEST(VersionFilter_test, Compact) } } -TEST(VersionFilter_test, CompactCommonHandle) +TEST(VersionFilterTest, CompactCommonHandle) { // TODO: currently it just test data statistics, add test for data correctness BlocksList blocks; diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 1c8b0242bfa..a7bb3d15f29 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -46,30 +46,42 @@ ::testing::AssertionResult assertEqual( { if (expected_v != actual_v) { - auto expected_str = fmt::format("\n{}: {}", expected_expr, expected_display); - auto actual_str = fmt::format("\n{}: {}", actual_expr, actual_display); + auto expected_str = fmt::format("\n {}:\n {}", expected_expr, expected_display); + auto actual_str = fmt::format("\n {}:\n {}", actual_expr, actual_display); return ::testing::AssertionFailure() << title << expected_str << actual_str; } return ::testing::AssertionSuccess(); } -#define ASSERT_EQUAL_WITH_TEXT(expected_value, actual_value, title, expected_display, actual_display) \ - do \ - { \ - auto result = assertEqual(#expected_value, #actual_value, (expected_value), (actual_value), (expected_display), (actual_display), title); \ - if (!result) \ - return result; \ +#define ASSERT_EQUAL_WITH_TEXT(expected_value, actual_value, title, expected_display, actual_display) \ + do \ + { \ + if (auto result = assertEqual(#expected_value, \ + #actual_value, \ + (expected_value), \ + (actual_value), \ + (expected_display), \ + (actual_display), \ + title); \ + !result) \ + return result; \ } while (false) -#define ASSERT_EQUAL(expected_value, actual_value, title) \ - do \ - { \ - auto expected_v = (expected_value); \ - auto actual_v = (actual_value); \ - auto result = assertEqual(#expected_value, #actual_value, expected_v, actual_v, expected_v, actual_v, title); \ - if (!result) \ - return result; \ +#define ASSERT_EQUAL(expected_value, actual_value, title) \ + do \ + { \ + auto expected_v = (expected_value); \ + auto actual_v = (actual_value); \ + if (auto result = assertEqual(#expected_value, \ + #actual_value, \ + expected_v, \ + actual_v, \ + expected_v, \ + actual_v, \ + title); \ + !result) \ + return result; \ } while (false) ::testing::AssertionResult dataTypeEqual( @@ -82,7 +94,8 @@ ::testing::AssertionResult dataTypeEqual( ::testing::AssertionResult columnEqual( const ColumnPtr & expected, - const ColumnPtr & actual) + const ColumnPtr & actual, + bool is_floating_point) { ASSERT_EQUAL(expected->getName(), actual->getName(), "Column name mismatch"); ASSERT_EQUAL(expected->size(), actual->size(), "Column size mismatch"); @@ -92,7 +105,22 @@ ::testing::AssertionResult columnEqual( auto expected_field = (*expected)[i]; auto actual_field = (*actual)[i]; - ASSERT_EQUAL_WITH_TEXT(expected_field, actual_field, fmt::format("Value {} mismatch", i), expected_field.toString(), actual_field.toString()); + if (!is_floating_point) + { + ASSERT_EQUAL_WITH_TEXT(expected_field, actual_field, fmt::format("Value at index {} mismatch", i), expected_field.toString(), actual_field.toString()); + } + else + { + auto expected_field_expr = expected_field.toString(); + auto actual_field_expr = actual_field.toString(); + if (auto res = ::testing::internal::CmpHelperFloatingPointEQ( + expected_field_expr.c_str(), + actual_field_expr.c_str(), + expected_field.safeGet(), + actual_field.safeGet()); + !res) + return ::testing::AssertionFailure() << fmt::format("Value at index {} mismatch, ", i) << res.message(); + } } return ::testing::AssertionSuccess(); } @@ -101,11 +129,10 @@ ::testing::AssertionResult columnEqual( const ColumnWithTypeAndName & expected, const ColumnWithTypeAndName & actual) { - auto ret = dataTypeEqual(expected.type, actual.type); - if (!ret) + if (auto ret = dataTypeEqual(expected.type, actual.type); !ret) return ret; - return columnEqual(expected.column, actual.column); + return columnEqual(expected.column, actual.column, expected.type->isFloatingPoint()); } ::testing::AssertionResult blockEqual( @@ -115,7 +142,10 @@ ::testing::AssertionResult blockEqual( size_t columns = actual.columns(); size_t expected_columns = expected.columns(); - ASSERT_EQUAL(expected_columns, columns, "Block size mismatch"); + ASSERT_EQUAL( + expected_columns, + columns, + fmt::format("Block column size mismatch\nexpected_structure: {}\nstructure: {}", expected.dumpJsonStructure(), actual.dumpJsonStructure())); for (size_t i = 0; i < columns; ++i) { @@ -381,7 +411,7 @@ String getColumnsContent(const ColumnsWithTypeAndName & cols) { if (cols.size() <= 0) return ""; - return getColumnsContent(cols, 0, cols[0].column->size() - 1); + return getColumnsContent(cols, 0, cols[0].column->size()); } String getColumnsContent(const ColumnsWithTypeAndName & cols, size_t begin, size_t end) @@ -392,7 +422,7 @@ String getColumnsContent(const ColumnsWithTypeAndName & cols, size_t begin, size const size_t col_size = cols[0].column->size(); assert(begin <= end); - assert(col_size > end); + assert(col_size >= end); assert(col_size > begin); bool is_same = true; @@ -411,7 +441,7 @@ String getColumnsContent(const ColumnsWithTypeAndName & cols, size_t begin, size { /// Push the column name fmt_buf.append(fmt::format("{}: (", cols[i].name)); - for (size_t j = begin; j <= end; ++j) + for (size_t j = begin; j < end; ++j) col_content.push_back(std::make_pair(j, (*cols[i].column)[j].toString())); /// Add content diff --git a/dbms/src/TestUtils/FunctionTestUtils.h b/dbms/src/TestUtils/FunctionTestUtils.h index e75acabe56c..50ec3dcda07 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.h +++ b/dbms/src/TestUtils/FunctionTestUtils.h @@ -516,7 +516,7 @@ ColumnWithTypeAndName createConstColumn( String getColumnsContent(const ColumnsWithTypeAndName & cols); -/// We can designate the range of columns printed with begin and end. range: [begin, end] +/// We can designate the range of columns printed with begin and end. range: [begin, end) String getColumnsContent(const ColumnsWithTypeAndName & cols, size_t begin, size_t end); // This wrapper function only serves to construct columns input for function-like macros, @@ -531,7 +531,8 @@ ::testing::AssertionResult dataTypeEqual( ::testing::AssertionResult columnEqual( const ColumnPtr & expected, - const ColumnPtr & actual); + const ColumnPtr & actual, + bool is_floating_point = false); // ignore name ::testing::AssertionResult columnEqual( @@ -581,7 +582,11 @@ DataTypePtr getReturnTypeForFunction( bool raw_function_test = false); template -ColumnWithTypeAndName createNullableColumn(InferredDataVector init_vec, const std::vector & null_map, const String name = "") +ColumnWithTypeAndName createNullableColumn( + InferredDataVector init_vec, + const std::vector & null_map, + const String name = "", + Int64 column_id = 0) { static_assert(TypeTraits::is_nullable == false); auto updated_vec = InferredDataVector>(); @@ -593,15 +598,19 @@ ColumnWithTypeAndName createNullableColumn(InferredDataVector init_vec, const else updated_vec.push_back(init_vec[i]); } - return createColumn>(updated_vec, name); + return createColumn>(updated_vec, name, column_id); } template -ColumnWithTypeAndName createNullableColumn(InferredDataInitializerList init, const std::vector & null_map, const String name = "") +ColumnWithTypeAndName createNullableColumn( + InferredDataInitializerList init, + const std::vector & null_map, + const String name = "", + Int64 column_id = 0) { static_assert(TypeTraits::is_nullable == false); auto vec = InferredDataVector(init); - return createNullableColumn(vec, null_map, name); + return createNullableColumn(vec, null_map, name, column_id); } template diff --git a/dbms/src/TestUtils/InputStreamTestUtils.cpp b/dbms/src/TestUtils/InputStreamTestUtils.cpp new file mode 100644 index 00000000000..761efa8232e --- /dev/null +++ b/dbms/src/TestUtils/InputStreamTestUtils.cpp @@ -0,0 +1,350 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +namespace DB +{ +namespace tests +{ +size_t getInputStreamNRows(const BlockInputStreamPtr & stream) +{ + RUNTIME_CHECK(stream != nullptr, Exception(fmt::format("The inputstream is nullptr!"))); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (true) + { + Block block = stream->read(); + // No more blocks + if (!block) + break; + block.checkNumberOfRows(); + num_rows_read += block.rows(); + } + stream->readSuffix(); + return num_rows_read; +} + +::testing::AssertionResult InputStreamRowsLengthCompare( + const char * stream_expr, + const char * nrows_expr, + const BlockInputStreamPtr & stream, + const size_t num_rows_expect) +{ + RUNTIME_CHECK(stream != nullptr, Exception(fmt::format("The first param of ASSERT_INPUTSTREAM_NROWS, `{}` is nullptr!", stream_expr))); + + size_t num_rows_read = 0; + stream->readPrefix(); + while (true) + { + try + { + Block read_block = stream->read(); + if (!read_block) + break; + read_block.checkNumberOfRows(); + num_rows_read += read_block.rows(); + } + catch (...) + { + return ::testing::AssertionFailure() << fmt::format("exception thrown while reading from {}. Error: {}", stream_expr, getCurrentExceptionMessage(true, false)); + } + } + stream->readSuffix(); + + if (num_rows_expect == num_rows_read) + return ::testing::AssertionSuccess(); + + auto reason = fmt::format(R"r( ({}).read() return num of rows + Which is: {} + {} + Which is: {})r", + stream_expr, + num_rows_read, + nrows_expr, + num_rows_expect); + return ::testing::AssertionFailure() << reason; +} + +::testing::AssertionResult InputStreamVSBlocksCompare( + const char * stream_expr, + const char * blocks_expr, + const BlockInputStreamPtr & stream, + const Blocks & blocks) +{ + RUNTIME_CHECK(stream != nullptr, Exception(fmt::format("The first param of ASSERT_INPUTSTREAM_NROWS, `{}` is nullptr!", stream_expr))); + + size_t block_idx = 0; + size_t num_rows_expect = 0; + size_t num_rows_read = 0; + stream->readPrefix(); + while (Block read_block = stream->read()) + { + read_block.checkNumberOfRows(); + + if (block_idx == blocks.size()) + { + auto reason = fmt::format(R"r( ({}).read() return more blocks as expected + {} only has {} blocks)r", + stream_expr, + blocks_expr, + blocks.size()); + return ::testing::AssertionFailure() << reason; + } + + blocks[block_idx].checkNumberOfRows(); // check the input + if (auto res = DB::tests::blockEqual(blocks[block_idx], read_block); !res) + { + auto reason = fmt::format(R"r( + ({}).read() return block is not equal to + the {} block in ({}))r", + stream_expr, + blocks_expr, + block_idx); + return res << reason; + } + + // continue to compare next block + num_rows_read += read_block.rows(); + num_rows_expect += blocks[block_idx].rows(); + block_idx++; + } + stream->readSuffix(); + + if (num_rows_expect == num_rows_read) + return ::testing::AssertionSuccess(); + + auto reason = fmt::format(R"r( ({}).read() return num of rows + Which is: {} + sum( ({}).rows() ) + Which is: {})r", + stream_expr, + num_rows_read, + blocks_expr, + num_rows_expect); + return ::testing::AssertionFailure() << reason; +} + +::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompare( + const char * stream_expr, + const char * block_expr, + const BlockInputStreamPtr & stream, + const Block & expect_block) +{ + RUNTIME_CHECK(stream != nullptr, Exception(fmt::format("The first param of ASSERT_INPUTSTREAM_NROWS, `{}` is nullptr!", stream_expr))); + expect_block.checkNumberOfRows(); // check the input + + size_t num_rows_expect = expect_block.rows(); + size_t num_rows_read = 0; + size_t prev_num_rows_read = 0; + stream->readPrefix(); + while (Block read_block = stream->read()) + { + read_block.checkNumberOfRows(); + num_rows_read += read_block.rows(); + // hot path, first block from inputstream and the rows is as expected + if (prev_num_rows_read == 0 && read_block.rows() == num_rows_expect) + { + if (auto res = DB::tests::blockEqual(expect_block, read_block); !res) + { + auto reason = fmt::format(R"r( + ({}).read() return block is not equal + structure() == {} + to the expect block ({}) + structure() == {})r", + stream_expr, + read_block.dumpJsonStructure(), + block_expr, + expect_block.dumpJsonStructure()); + return res << reason; + } + } + + if (num_rows_read > num_rows_expect) + { + auto reason = fmt::format(R"r( + ({}).read() return more rows({}) than expected + ({}).rows() + Which is: {} + last block is: {})r", + stream_expr, + num_rows_read, + block_expr, + num_rows_expect, + getColumnsContent(read_block.getColumnsWithTypeAndName())); + return ::testing::AssertionFailure() << reason; + } + + // else, compare the the `block` to the [prev_num_rows_read, num_rows_read) rows of `rhs` + { + // num of columns + auto read_cols_expr = fmt::format("{}.read().columns()", stream_expr); + auto rcols_expr = fmt::format("{}.columns()", block_expr); + if (auto res = ::testing::internal::EqHelper::Compare( + read_cols_expr.c_str(), + rcols_expr.c_str(), + read_block.columns(), + expect_block.columns()); + !res) + { + return res; + } + for (size_t i = 0; i < expect_block.columns(); ++i) + { + const auto & actual_col = read_block.getByPosition(i); + const auto & expected_full_col = expect_block.getByPosition(i); + if (expected_full_col.column->isColumnConst() != actual_col.column->isColumnConst()) + { + // One is ColumnConst but the other is not + return ::testing::AssertionFailure() << fmt::format( + " block[{}].isColumnConst() from actual block\n {}\n expect_block[{}].isColumnConst()\n {}", + actual_col.name, + actual_col.column->isColumnConst(), + expected_full_col.name, + expected_full_col.column->isColumnConst()); + } + else if (expected_full_col.column->isColumnConst() && actual_col.column->isColumnConst()) + { + if (auto res = dataTypeEqual(expected_full_col.type, actual_col.type); !res) + return res; + if (auto res = ::testing::internal::EqHelper::Compare("", "", actual_col.column->size(), expected_full_col.column->size()); !res) + { + return res; + } + if (actual_col.column->compareAt(0, 0, *expected_full_col.column, -1) != 0) + { + return ::testing::AssertionFailure() << "Column Const data mismatch"; + } + } + else + { + auto expect_col = expected_full_col.cloneEmpty(); + auto column_data = expect_col.type->createColumn(); + column_data->insertRangeFrom(*expected_full_col.column, prev_num_rows_read, num_rows_read - prev_num_rows_read); + expect_col.column = std::move(column_data); + if (auto res = DB::tests::columnEqual(expect_col, actual_col); !res) + { + return res; + } + } + } + } + + prev_num_rows_read += read_block.rows(); + } + stream->readSuffix(); + + if (num_rows_expect == num_rows_read) + return ::testing::AssertionSuccess(); + + // Less rows than expected + auto reason = fmt::format(R"r( ({}).read() return num of rows + Which is: {} + the num rows of ({}) + Which is: {})r", + stream_expr, + num_rows_read, + block_expr, + num_rows_expect); + return ::testing::AssertionFailure() << reason; +} + +::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompareColumns( + const char * stream_expr, + const char * /*colnames_expr*/, + const char * columns_expr, + const BlockInputStreamPtr & stream, + const Strings & colnames, + const ColumnsWithTypeAndName & columns) +{ + RUNTIME_CHECK(stream != nullptr, Exception(fmt::format("The first param of ASSERT_INPUTSTREAM_COLS_UR, `{}` is nullptr!", stream_expr))); + RUNTIME_CHECK( + colnames.size() == columns.size(), + Exception(fmt::format("The length of second and thrid param of ASSERT_INPUTSTREAM_COLS_UR not match! {} != {}", colnames.size(), columns.size()))); + + Block expect_block(columns); + expect_block.checkNumberOfRows(); // check the input + + size_t num_rows_expect = expect_block.rows(); + size_t num_rows_read = 0; + size_t prev_num_rows_read = 0; + stream->readPrefix(); + while (Block read_block = stream->read()) + { + num_rows_read += read_block.rows(); + + if (num_rows_read > num_rows_expect) + { + auto reason = fmt::format(R"r( + ({}).read() return more rows({}) than expected + ({}).rows() + Which is: {} + last block is: {})r", + stream_expr, + num_rows_read, + columns_expr, + num_rows_expect, + getColumnsContent(read_block.getColumnsWithTypeAndName())); + return ::testing::AssertionFailure() << reason; + } + + // else, compare the the `read_block` to the [prev_num_rows_read, num_rows_read) rows of `expect_block` + for (size_t col_idx = 0; col_idx < colnames.size(); ++col_idx) + { + const auto & col_name = colnames[col_idx]; + // Copy the [prev_num_rows_read, num_rows_read) of `expect_block` + const auto & expect_full_col = expect_block.getByPosition(col_idx); + auto expect_col = expect_full_col.cloneEmpty(); + auto column_data = expect_col.type->createColumn(); + column_data->insertRangeFrom(*expect_full_col.column, prev_num_rows_read, num_rows_read - prev_num_rows_read); + expect_col.column = std::move(column_data); + + const auto & actual_col = read_block.getByName(col_name); + if (auto res = DB::tests::columnEqual(expect_col, actual_col); !res) + { + auto expect_expr = fmt::format("expect block: {}", getColumnsContent(expect_block.getColumnsWithTypeAndName(), prev_num_rows_read, num_rows_read)); + Block actual_block_to_cmp; + for (const auto & col_name : colnames) + actual_block_to_cmp.insert(read_block.getByName(col_name)); + auto actual_expr = fmt::format("actual block: {}", getColumnsContent(actual_block_to_cmp.getColumnsWithTypeAndName())); + return res << fmt::format("\n details: [column={}] [prev_nrows={}] [cur_nrows={}]:\n {}\n {}", col_name, prev_num_rows_read, num_rows_read, expect_expr, actual_expr); + } + } + + prev_num_rows_read += read_block.rows(); + } + stream->readSuffix(); + + if (num_rows_expect == num_rows_read) + return ::testing::AssertionSuccess(); + + // Less rows than expected + auto reason = fmt::format(R"r( ({}).read() return num of rows + Which is: {} + the num rows of ({}) + Which is: {})r", + stream_expr, + num_rows_read, + columns_expr, + num_rows_expect); + return ::testing::AssertionFailure() << reason; +} + +} // namespace tests +} // namespace DB diff --git a/dbms/src/TestUtils/InputStreamTestUtils.h b/dbms/src/TestUtils/InputStreamTestUtils.h new file mode 100644 index 00000000000..82b2768df16 --- /dev/null +++ b/dbms/src/TestUtils/InputStreamTestUtils.h @@ -0,0 +1,75 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include +namespace DB +{ +class IBlockInputStream; +using BlockInputStreamPtr = std::shared_ptr; + +namespace tests +{ +/// helper functions for comparing the result of input stream + +// Get the num of rows read from inputstream +size_t getInputStreamNRows(const BlockInputStreamPtr & stream); + +// Checking the num of rows read from inputstream +::testing::AssertionResult InputStreamRowsLengthCompare( + const char * stream_expr, + const char * nrows_expr, + const BlockInputStreamPtr & stream, + const size_t num_rows_expect); +#define ASSERT_INPUTSTREAM_NROWS(val1, val2) ASSERT_PRED_FORMAT2(::DB::tests::InputStreamRowsLengthCompare, val1, val2) + +// Checking the blocks read from inputstream. +// The inputstream must return the blocks strictly equal to `blocks`. +::testing::AssertionResult InputStreamVSBlocksCompare( + const char * stream_expr, + const char * blocks_expr, + const BlockInputStreamPtr & stream, + const Blocks & blocks); +#define ASSERT_INPUTSTREAM_BLOCKS(val1, val2) ASSERT_PRED_FORMAT2(::DB::tests::InputStreamVSBlocksCompare, val1, val2) + +// Unrestrictly checking the blocks read from inputstream. +// Allowing the inputstream break the rows into serval smaller blocks. +::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompare( + const char * stream_expr, + const char * block_expr, + const BlockInputStreamPtr & stream, + const Block & expect_block); +#define ASSERT_INPUTSTREAM_BLOCK_UR(val1, val2) ASSERT_PRED_FORMAT2(::DB::tests::InputStreamVSBlockUnrestrictlyCompare, val1, val2) + +// Unrestrictly checking a part of columns read from inputstream. +// Allowing the inputstream break the rows into serval smaller blocks. +// Only check the read columns with name in `colnames` +// The size of `colnames` must be the same as `colunms` +::testing::AssertionResult InputStreamVSBlockUnrestrictlyCompareColumns( + const char * stream_expr, + const char * colnames_expr, + const char * columns_expr, + const BlockInputStreamPtr & stream, + const Strings & colnames, + const ColumnsWithTypeAndName & columns); +#define ASSERT_INPUTSTREAM_COLS_UR(stream, colnames, columns) \ + ASSERT_PRED_FORMAT3(::DB::tests::InputStreamVSBlockUnrestrictlyCompareColumns, stream, colnames, columns) + +} // namespace tests +} // namespace DB diff --git a/dbms/src/TestUtils/tests/gtest_function_test_utils.cpp b/dbms/src/TestUtils/tests/gtest_function_test_utils.cpp index b7fd8cb7e7f..40b8f41d48d 100644 --- a/dbms/src/TestUtils/tests/gtest_function_test_utils.cpp +++ b/dbms/src/TestUtils/tests/gtest_function_test_utils.cpp @@ -18,11 +18,16 @@ namespace DB { namespace tests { -class TestFunctionTestUtils : public ::testing::Test +TEST(TestFunctionTestUtils, CompareFloat64Column) +try { -}; + ASSERT_COLUMN_EQ( + createColumn({1.23456789}), + createColumn({1.23456789})); +} +CATCH -TEST_F(TestFunctionTestUtils, ParseDecimal) +TEST(TestFunctionTestUtils, ParseDecimal) try { using DecimalField64 = DecimalField; @@ -58,7 +63,7 @@ try } CATCH -TEST_F(TestFunctionTestUtils, CreateDecimalColumn) +TEST(TestFunctionTestUtils, CreateDecimalColumn) try { using DecimalField64 = DecimalField; diff --git a/dbms/src/TestUtils/tests/gtest_inputstream_test_utils.cpp b/dbms/src/TestUtils/tests/gtest_inputstream_test_utils.cpp new file mode 100644 index 00000000000..64b57c3a5f9 --- /dev/null +++ b/dbms/src/TestUtils/tests/gtest_inputstream_test_utils.cpp @@ -0,0 +1,302 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +namespace DB +{ +namespace tests +{ +TEST(InputStreamTestUtilsTest, RawFuncPass) +try +{ + ASSERT_ANY_THROW(InputStreamVSBlockUnrestrictlyCompare("", "", nullptr, Block());); + + { + // pass + BlocksList blocks{ + Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")}), + }; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + EXPECT_TRUE( + InputStreamVSBlockUnrestrictlyCompare( + "", + "", + in, + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2"), + }))); + } + { + // pass + BlocksList blocks{ + Block({createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2")}), + }; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + EXPECT_TRUE( + InputStreamVSBlockUnrestrictlyCompare( + "", + "", + in, + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2"), + }))); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, RawFuncColumnNotMatch) +try +{ + { + // column num not match + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + auto res = InputStreamVSBlockUnrestrictlyCompare( + "in", + "block", + std::make_unique(std::move(blocks)), + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2"), + createColumn({123, 456, 1, 2, 3, 4}, "col3"), + })); + EXPECT_FALSE(res); + } + { + // column num not match + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + auto res = InputStreamVSBlockUnrestrictlyCompare( + "in", + "block", + std::make_unique(std::move(blocks)), + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + })); + EXPECT_FALSE(res); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, RawFuncRowsNotMatch) +try +{ + { + // rows not match + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + auto res = InputStreamVSBlockUnrestrictlyCompare( + "in", + "block", + std::make_unique(std::move(blocks)), + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash", "br"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4, 5}, "col2"), + })); + EXPECT_FALSE(res); + } + { + // rows not match + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + auto res = InputStreamVSBlockUnrestrictlyCompare( + "in", + "block", + std::make_unique(std::move(blocks)), + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd"}, "col1"), + createColumn({123, 456, 1, 2, 3}, "col2"), + })); + EXPECT_FALSE(res); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, RawFuncCellNotMatch) +try +{ + ASSERT_ANY_THROW(InputStreamVSBlockUnrestrictlyCompare("", "", nullptr, Block());); + + { + BlocksList blocks{ + Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")}), + }; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + auto res = InputStreamVSBlockUnrestrictlyCompare( + "", + "", + in, + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, /*wrong value*/ 5}, "col2"), + })); + EXPECT_FALSE(res); + } + { + // pass + BlocksList blocks{ + Block({createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2")}), + }; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + auto res = InputStreamVSBlockUnrestrictlyCompare( + "", + "", + in, + Block({ + createColumn({"hello", "world", "tikv", "tidb", "pd", /*wrong value*/ "flash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2"), + })); + EXPECT_FALSE(res); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, CompareInputStreamNRows) +try +{ + // Only check number of rows + { + BlocksList blocks{}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_NROWS(in, 0); + } + { + BlocksList blocks{Block({createConstColumn(100, "test_data", "col1"), + createConstColumn(100, 123, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_NROWS(in, 100); + } + { + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_NROWS(in, 6); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, CompareInputStreamBlock) +try +{ + // Check the block read + { + BlocksList blocks{}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_BLOCKS(in, Blocks()); + } + { + BlocksList blocks{Block({createConstColumn(100, "test_data", "col1"), + createConstColumn(100, 123, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_BLOCKS(in, Blocks({Block{createConstColumn(100, "test_data", "col1"), createConstColumn(100, 123, "col2")}})); + } + { + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_BLOCKS( + in, + Blocks({ + Block({createColumn({"hello", "world"}, "col1"), createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), createColumn({1, 2, 3, 4}, "col2")}), + })); + } + + // unrestrictly check + { + //the input stream return smaller blocks + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_BLOCK_UR( + in, + Block({// + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2")})); + } + { + //the input stream return excatly the same block + BlocksList blocks{Block({createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_BLOCK_UR( + in, + Block({// + createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({123, 456, 1, 2, 3, 4}, "col2")})); + } +} +CATCH + +TEST(InputStreamTestUtilsTest, CompareInputStreamColumns) +try +{ + // unrestrictly check a part of columns + { + //the input stream return smaller blocks, only check col1 + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({"col1"}), + createColumns({createColumn({"hello", "world", "tikv", "tidb", "pd", "tiflash"})})); + } + { + //the input stream return smaller blocks, only check col2 + BlocksList blocks{Block({createColumn({"hello", "world"}, "col1"), + createColumn({123, 456}, "col2")}), + Block({createColumn({"tikv", "tidb", "pd", "tiflash"}, "col1"), + createColumn({1, 2, 3, 4}, "col2")})}; + BlockInputStreamPtr in = std::make_unique(std::move(blocks)); + ASSERT_INPUTSTREAM_COLS_UR( + in, + Strings({"col2"}), + createColumns({createColumn({123, 456, 1, 2, 3, 4})})); + } +} +CATCH + +} // namespace tests +} // namespace DB diff --git a/dbms/src/TestUtils/tests/gtest_print_columns.cpp b/dbms/src/TestUtils/tests/gtest_print_columns.cpp index 50631fc4f4a..4930d0adefc 100644 --- a/dbms/src/TestUtils/tests/gtest_print_columns.cpp +++ b/dbms/src/TestUtils/tests/gtest_print_columns.cpp @@ -19,7 +19,6 @@ namespace DB { namespace tests { - class PrintColumnsTest : public DB::tests::ExecutorTest { public: @@ -47,9 +46,9 @@ TEST_F(PrintColumnsTest, SimpleTest) try { EXPECT_EQ(getColumnsContent(test_cols), result1); - EXPECT_EQ(getColumnsContent(test_cols, 0, col_len - 1), result2); - EXPECT_EQ(getColumnsContent(test_cols, 0, 0), result3); - EXPECT_EQ(getColumnsContent(test_cols, 1, col_len - 2), result4); + EXPECT_EQ(getColumnsContent(test_cols, 0, col_len), result2); + EXPECT_EQ(getColumnsContent(test_cols, 0, 1), result3); + EXPECT_EQ(getColumnsContent(test_cols, 1, col_len - 1), result4); } CATCH