From 75909831961a5b93dd81bcdf6ddbbc4989bd29d8 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 27 May 2022 15:25:17 +0800 Subject: [PATCH 1/5] Add more test into mix mode tests --- .../V3/tests/gtest_page_storage_mix_mode.cpp | 255 +++++++++++++++++- 1 file changed, 252 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 97eac841018..b172b721608 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -82,6 +82,16 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic return run_mode; } + PageReader newMixedPageReader(PageStorage::SnapshotPtr & snapshot) + { + return storage_pool_mix->newLogReader(nullptr, snapshot); + } + + PageReader newMixedPageReader() + { + return storage_pool_mix->newLogReader(nullptr, true, "PageStorageMixedTest"); + } + void reloadV2StoragePool() { db_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2); @@ -597,7 +607,7 @@ try // Thread A create snapshot for read auto snapshot_mix_before_merge_delta = page_reader_mix->getSnapshot("ReadWithSnapshotAfterMergeDelta"); { - auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_before_merge_delta); + auto page_reader_mix_with_snap = newMixedPageReader(snapshot_mix_before_merge_delta); const auto & page1 = page_reader_mix_with_snap.read(1); const auto & page2 = page_reader_mix_with_snap.read(2); const auto & page3 = page_reader_mix_with_snap.read(3); @@ -606,7 +616,7 @@ try ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3); } { - auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, true, "ReadWithSnapshotAfterMergeDelta"); + auto page_reader_mix_with_snap = newMixedPageReader(); const auto & page1 = page_reader_mix_with_snap.read(1); const auto & page2 = page_reader_mix_with_snap.read(2); const auto & page3 = page_reader_mix_with_snap.read(3); @@ -625,7 +635,7 @@ try } // Thread A continue to read 1, 3 { - auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_before_merge_delta); + auto page_reader_mix_with_snap = newMixedPageReader(snapshot_mix_before_merge_delta); // read 1, 3 with snapshot, should be success const auto & page1 = page_reader_mix_with_snap.read(1); const auto & page3 = page_reader_mix_with_snap.read(3); @@ -643,6 +653,245 @@ try } CATCH +TEST_F(PageStorageMixedTest, testDTWriteRead) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_mix = page_reader_mix->getSnapshot(""); + { + WriteBatch batch; + batch.delPage(1); + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2}); + ASSERT_EQ(page_maps.size(), 2); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); + } +} +CATCH + +TEST_F(PageStorageMixedTest, testDTWriteRead2) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + auto snapshot_mix = page_reader_mix->getSnapshot(""); + { + WriteBatch batch; + batch.delPage(1); + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2}); + ASSERT_EQ(page_maps.size(), 2); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); + } +} +CATCH + +TEST_F(PageStorageMixedTest, testDTWriteRead3) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + // to keep mix mode + batch.putExternal(10, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.delPage(1); + batch.delPage(2); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_mix = page_reader_mix->getSnapshot(""); + { + WriteBatch batch; + batch.delPage(1); + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2}); + ASSERT_EQ(page_maps.size(), 2); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); + } +} +CATCH + +TEST_F(PageStorageMixedTest, testDTWriteRead4) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page1 = page_reader_mix->read(1); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } +} +CATCH + + +TEST_F(PageStorageMixedTest, testDTWriteRead5) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page1 = page_reader_mix->read(2); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } +} +CATCH } // namespace PS::V3::tests } // namespace DB From 8f592b5394e38eb23d7779bfa2ca59c1920ceafb Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 27 May 2022 16:36:48 +0800 Subject: [PATCH 2/5] Add more tests for test mix mode --- .../V3/tests/gtest_page_storage_mix_mode.cpp | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index b172b721608..a5e81caee13 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -851,7 +851,6 @@ try } CATCH - TEST_F(PageStorageMixedTest, testDTWriteRead5) try { @@ -876,6 +875,47 @@ try page_writer_v2->write(std::move(batch), nullptr); } + { + WriteBatch batch; + batch.delPage(1); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + auto page1 = page_reader_mix->read(2); + + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } +} +CATCH + +TEST_F(PageStorageMixedTest, testDTWriteRead6) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + // Change to mix mode here ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); From 3aca42e35f1d31309f7ea1b2573194e11de88f55 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 27 May 2022 16:45:38 +0800 Subject: [PATCH 3/5] change name --- .../V3/tests/gtest_page_storage_mix_mode.cpp | 55 ++----------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index a5e81caee13..57fd3bb90ea 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -653,52 +653,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, testDTWriteRead) -try -{ - UInt64 tag = 0; - const size_t buf_sz = 1024; - char c_buff[buf_sz]; - for (size_t i = 0; i < buf_sz; ++i) - { - c_buff[i] = i % 0xff; - } - - { - WriteBatch batch; - ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); - batch.putPage(1, tag, buff, buf_sz); - page_writer_v2->write(std::move(batch), nullptr); - } - - // Change to mix mode here - ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); - - { - WriteBatch batch; - batch.putRefPage(2, 1); - page_writer_mix->write(std::move(batch), nullptr); - } - - auto snapshot_mix = page_reader_mix->getSnapshot(""); - { - WriteBatch batch; - batch.delPage(1); - batch.delPage(2); - page_writer_mix->write(std::move(batch), nullptr); - } - - { - auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2}); - ASSERT_EQ(page_maps.size(), 2); - - ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1); - ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2); - } -} -CATCH - -TEST_F(PageStorageMixedTest, testDTWriteRead2) +TEST_F(PageStorageMixedTest, refWithSnapshot2) try { UInt64 tag = 0; @@ -743,7 +698,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, testDTWriteRead3) +TEST_F(PageStorageMixedTest, refWithSnapshot3) try { UInt64 tag = 0; @@ -810,7 +765,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, testDTWriteRead4) +TEST_F(PageStorageMixedTest, refWithSnapshot4) try { UInt64 tag = 0; @@ -851,7 +806,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, testDTWriteRead5) +TEST_F(PageStorageMixedTest, refWithSnapshot5) try { UInt64 tag = 0; @@ -892,7 +847,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, testDTWriteRead6) +TEST_F(PageStorageMixedTest, refWithSnapshot6) try { UInt64 tag = 0; From ca2be9e72ae50496daf6ea3acb758ac82c1f2c34 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Fri, 27 May 2022 17:21:25 +0800 Subject: [PATCH 4/5] add ReadWithSnapshot --- .../V3/tests/gtest_page_storage_mix_mode.cpp | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 57fd3bb90ea..8d13794d412 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -643,6 +643,7 @@ try ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3); ASSERT_THROW(page_reader_mix_with_snap.read(4), DB::Exception); } + { // Revert v3 WriteBatch batch; @@ -888,5 +889,55 @@ try } CATCH +TEST_F(PageStorageMixedTest, ReadWithSnapshot) +try +{ + UInt64 tag = 0; + const size_t buf_sz = 1; + char c_buff1[buf_sz]; + c_buff1[0] = 1; + + char c_buff2[buf_sz]; + c_buff2[0] = 2; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff1, buf_sz); + batch.putPage(1, tag, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + // Change to mix mode here + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + auto snapshot_mix = page_reader_mix->getSnapshot(""); + { + WriteBatch batch; + batch.delPage(1); + ReadBufferPtr buff = std::make_shared(c_buff2, buf_sz); + batch.putPage(1, tag, buff, buf_sz); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page1 = newMixedPageReader(snapshot_mix).read(1); + ASSERT_PAGE_EQ(c_buff1, buf_sz, page1, 1); + } + + { + auto page1 = page_reader_mix->read(1); + ASSERT_PAGE_EQ(c_buff2, buf_sz, page1, 1); + } + + { + // Revert v3 + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } +} +CATCH + + } // namespace PS::V3::tests } // namespace DB From 1656c4f6f781978bf378bcb685dfcf85c733faf1 Mon Sep 17 00:00:00 2001 From: jiaqizho Date: Tue, 31 May 2022 15:35:34 +0800 Subject: [PATCH 5/5] update --- dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index 8d13794d412..e2e9ea2ffd3 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -889,7 +889,7 @@ try } CATCH -TEST_F(PageStorageMixedTest, ReadWithSnapshot) +TEST_F(PageStorageMixedTest, ReadWithSnapshot2) try { UInt64 tag = 0;