Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more test in mix mode #5017

Merged
merged 9 commits into from
Jun 17, 2022
Merged

Add more test in mix mode #5017

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7590983
Add more test into mix mode tests
jiaqizho May 27, 2022
8f592b5
Add more tests for test mix mode
jiaqizho May 27, 2022
3aca42e
change name
jiaqizho May 27, 2022
ca2be9e
add ReadWithSnapshot
jiaqizho May 27, 2022
33cd045
Merge branch 'master' into add-more-test-in-mix-mode
jiaqizho May 30, 2022
1656c4f
update
jiaqizho May 31, 2022
1d14027
Merge branch 'add-more-test-in-mix-mode' of https://github.com/jiaqiz…
jiaqizho May 31, 2022
791cb84
Merge branch 'master' into add-more-test-in-mix-mode
JaySon-Huang Jun 15, 2022
de199c0
Merge branch 'master' into add-more-test-in-mix-mode
ti-chi-bot Jun 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
301 changes: 298 additions & 3 deletions dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,16 @@ class PageStorageMixedTest : public DB::base::TiFlashStorageTestBasic
return run_mode;
}

PageReader newMixedPageReader(PageStorage::SnapshotPtr & snapshot)
{
return storage_pool_mix->newLogReader(nullptr, snapshot);
}

PageReader newMixedPageReader()
{
return storage_pool_mix->newLogReader(nullptr, true, "PageStorageMixedTest");
}

void reloadV2StoragePool()
{
db_context->setPageStorageRunMode(PageStorageRunMode::ONLY_V2);
Expand Down Expand Up @@ -1035,7 +1045,7 @@ try
// Thread A create snapshot for read
auto snapshot_mix_before_merge_delta = page_reader_mix->getSnapshot("ReadWithSnapshotAfterMergeDelta");
{
auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_before_merge_delta);
auto page_reader_mix_with_snap = newMixedPageReader(snapshot_mix_before_merge_delta);
const auto & page1 = page_reader_mix_with_snap.read(1);
const auto & page2 = page_reader_mix_with_snap.read(2);
const auto & page3 = page_reader_mix_with_snap.read(3);
Expand All @@ -1044,7 +1054,7 @@ try
ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3);
}
{
auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, true, "ReadWithSnapshotAfterMergeDelta");
auto page_reader_mix_with_snap = newMixedPageReader();
const auto & page1 = page_reader_mix_with_snap.read(1);
const auto & page2 = page_reader_mix_with_snap.read(2);
const auto & page3 = page_reader_mix_with_snap.read(3);
Expand All @@ -1063,14 +1073,15 @@ try
}
// Thread A continue to read 1, 3
{
auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_before_merge_delta);
auto page_reader_mix_with_snap = newMixedPageReader(snapshot_mix_before_merge_delta);
// read 1, 3 with snapshot, should be success
const auto & page1 = page_reader_mix_with_snap.read(1);
const auto & page3 = page_reader_mix_with_snap.read(3);
ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1);
ASSERT_PAGE_EQ(c_buff2, buf_sz2, page3, 3);
ASSERT_THROW(page_reader_mix_with_snap.read(4), DB::Exception);
}

{
// Revert v3
WriteBatch batch;
Expand All @@ -1081,6 +1092,290 @@ try
}
CATCH

TEST_F(PageStorageMixedTest, refWithSnapshot2)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1024;
char c_buff[buf_sz];
for (size_t i = 0; i < buf_sz; ++i)
{
c_buff[i] = i % 0xff;
}

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

auto snapshot_mix = page_reader_mix->getSnapshot("");
{
WriteBatch batch;
batch.delPage(1);
batch.delPage(2);
page_writer_mix->write(std::move(batch), nullptr);
}

{
auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2});
ASSERT_EQ(page_maps.size(), 2);

ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1);
ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2);
}
}
CATCH

TEST_F(PageStorageMixedTest, refWithSnapshot3)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1024;
char c_buff[buf_sz];
for (size_t i = 0; i < buf_sz; ++i)
{
c_buff[i] = i % 0xff;
}

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
// to keep mix mode
batch.putExternal(10, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.delPage(1);
batch.delPage(2);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
page_writer_mix->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_mix->write(std::move(batch), nullptr);
}

auto snapshot_mix = page_reader_mix->getSnapshot("");
{
WriteBatch batch;
batch.delPage(1);
batch.delPage(2);
page_writer_mix->write(std::move(batch), nullptr);
}

{
auto page_maps = newMixedPageReader(snapshot_mix).read({1, 2});
ASSERT_EQ(page_maps.size(), 2);

ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[1], 1);
ASSERT_PAGE_EQ(c_buff, buf_sz, page_maps[2], 2);
}
}
CATCH

TEST_F(PageStorageMixedTest, refWithSnapshot4)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1024;
char c_buff[buf_sz];
for (size_t i = 0; i < buf_sz; ++i)
{
c_buff[i] = i % 0xff;
}

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

{
WriteBatch batch;
batch.delPage(2);
page_writer_mix->write(std::move(batch), nullptr);
}

{
auto page1 = page_reader_mix->read(1);

ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1);
}
}
CATCH

TEST_F(PageStorageMixedTest, refWithSnapshot5)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1024;
char c_buff[buf_sz];
for (size_t i = 0; i < buf_sz; ++i)
{
c_buff[i] = i % 0xff;
}

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.delPage(1);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

{
auto page1 = page_reader_mix->read(2);

ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2);
}
}
CATCH

TEST_F(PageStorageMixedTest, refWithSnapshot6)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1024;
char c_buff[buf_sz];
for (size_t i = 0; i < buf_sz; ++i)
{
c_buff[i] = i % 0xff;
}

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
batch.putPage(1, tag, buff, buf_sz);
page_writer_v2->write(std::move(batch), nullptr);
}

{
WriteBatch batch;
batch.putRefPage(2, 1);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

{
WriteBatch batch;
batch.delPage(1);
page_writer_mix->write(std::move(batch), nullptr);
}

{
auto page1 = page_reader_mix->read(2);

ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2);
}
}
CATCH

TEST_F(PageStorageMixedTest, ReadWithSnapshot2)
try
{
UInt64 tag = 0;
const size_t buf_sz = 1;
char c_buff1[buf_sz];
c_buff1[0] = 1;

char c_buff2[buf_sz];
c_buff2[0] = 2;

{
WriteBatch batch;
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff1, buf_sz);
batch.putPage(1, tag, buff, buf_sz);
page_writer_v2->write(std::move(batch), nullptr);
}

// Change to mix mode here
ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE);

auto snapshot_mix = page_reader_mix->getSnapshot("");
{
WriteBatch batch;
batch.delPage(1);
ReadBufferPtr buff = std::make_shared<ReadBufferFromMemory>(c_buff2, buf_sz);
batch.putPage(1, tag, buff, buf_sz);
page_writer_mix->write(std::move(batch), nullptr);
}

{
auto page1 = newMixedPageReader(snapshot_mix).read(1);
ASSERT_PAGE_EQ(c_buff1, buf_sz, page1, 1);
}

{
auto page1 = page_reader_mix->read(1);
ASSERT_PAGE_EQ(c_buff2, buf_sz, page1, 1);
}

{
// Revert v3
WriteBatch batch;
batch.delPage(1);
page_writer_mix->write(std::move(batch), nullptr);
}
}
CATCH


} // namespace PS::V3::tests
} // namespace DB