Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix some fail cases when enable TASN #5086

Merged
merged 6 commits into from
Jun 9, 2022
Merged

Fix some fail cases when enable TASN #5086

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
60bd915
Fix some fail case when enable tasn
jiaqizho Jun 8, 2022
8bca609
Fix data race in fail point
jiaqizho Jun 8, 2022
07a1c34
changed
jiaqizho Jun 8, 2022
4c5a7b0
fix build
jiaqizho Jun 8, 2022
b47be23
changed
jiaqizho Jun 9, 2022
81eb177
Merge branch 'master' into fix-tasn
ti-chi-bot Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions dbms/src/Flash/Management/tests/gtest_manual_compact.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ class BasicManualCompactTest
setupStorage();

// In tests let's only compact one segment.
db_context->setSetting("manual_compact_more_until_ms", UInt64(0));
db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(0)));

// Split into 4 segments, and prepare some delta data for first 3 segments.
helper = std::make_unique<DM::tests::MultiSegmentTestUtil>(*db_context);
Expand Down Expand Up @@ -117,7 +117,7 @@ class BasicManualCompactTest

DM::tests::DMTestEnv::PkType pk_type;

[[maybe_unused]] Poco::Logger * log;
Poco::Logger * log;
jiaqizho marked this conversation as resolved.
Show resolved Hide resolved
};


Expand Down Expand Up @@ -315,7 +315,7 @@ CATCH
TEST_P(BasicManualCompactTest, CompactMultiple)
try
{
db_context->setSetting("manual_compact_more_until_ms", UInt64(60 * 1000)); // Hope it's long enough!
db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(60 * 1000))); // Hope it's long enough!

auto request = ::kvrpcpb::CompactRequest();
request.set_physical_table_id(TABLE_ID);
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/DeltaMerge/StoragePool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,6 +624,7 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons

auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false);
fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, {
std::lock_guard lock_for_fail_point(mutex_for_fail_point);
jiaqizho marked this conversation as resolved.
Show resolved Hide resolved
static size_t fail_point_called = 0;
if (existed_path.empty() && fail_point_called % 10 == 0)
{
Expand Down
1 change: 1 addition & 0 deletions dbms/src/Storages/DeltaMerge/StoragePool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ class StoragePool : private boost::noncopyable
std::atomic<Timepoint> last_try_gc_time = Clock::now();

std::mutex mutex;
std::mutex mutex_for_fail_point;

Context & global_context;

Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
// Check there is only one segment
ASSERT_EQ(store->segments.size(), 1);
const auto & [_key, seg] = *store->segments.begin();
(void)_key;
ASSERT_EQ(seg->getDelta()->getRows(), n_avg_rows_per_segment * 4);
ASSERT_EQ(seg->getStable()->getRows(), 0);

Expand All @@ -108,6 +109,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
auto segment_idx = 0;
for (auto & [_key, seg] : store->segments)
{
(void)_key;
LOG_FMT_INFO(log, "Segment #{}: Range = {}", segment_idx, seg->getRowKeyRange().toDebugString());
ASSERT_EQ(seg->getDelta()->getRows(), 0);
ASSERT_GT(seg->getStable()->getRows(), 0); // We don't check the exact rows of each segment.
Expand Down Expand Up @@ -147,6 +149,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
auto segment_idx = 0;
for (auto & [_key, seg] : store->segments)
{
(void)_key;
ASSERT_EQ(seg->getDelta()->getRows(), expected_delta_rows[segment_idx]) << "Assert failed for segment #" << segment_idx;
ASSERT_EQ(seg->getStable()->getRows(), expected_stable_rows[segment_idx]) << "Assert failed for segment #" << segment_idx;
segment_idx++;
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3608,7 +3608,7 @@ class DeltaMergeStoreMergeDeltaBySegmentTest
UInt64 ps_ver;
DMTestEnv::PkType pk_type;

[[maybe_unused]] Poco::Logger * log;
Poco::Logger * log;
jiaqizho marked this conversation as resolved.
Show resolved Hide resolved
};

INSTANTIATE_TEST_CASE_P(
Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,20 +82,23 @@ try
stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 128,
.padded_size = 0,
.tag = 0,
.offset = 1024,
.checksum = 0x4567,
});
stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
});
stats.restoreByEntry(PageEntryV3{
.file_id = file_id2,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
Expand Down
102 changes: 51 additions & 51 deletions dbms/src/Storages/Page/V3/tests/gtest_page_directory.cpp
View file
Open in desktop

Large diffs are not rendered by default.

48 changes: 24 additions & 24 deletions dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ namespace DB::PS::V3::tests
{
TEST(WALSeriTest, AllPuts)
{
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
PageEntriesEdit edit;
edit.put(1, entry_p1);
Expand All @@ -56,8 +56,8 @@ TEST(WALSeriTest, AllPuts)
TEST(WALSeriTest, PutsAndRefsAndDels)
try
{
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
PageEntriesEdit edit;
edit.put(3, entry_p3);
Expand Down Expand Up @@ -104,9 +104,9 @@ CATCH

TEST(WALSeriTest, Upserts)
{
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
PageEntriesEdit edit;
Expand Down Expand Up @@ -164,7 +164,7 @@ TEST(WALSeriTest, RefExternalAndEntry)

{
PageEntriesEdit edit;
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
edit.varEntry(1, ver1_0, entry_p1_2, 2);
edit.varDel(1, ver2_0);
edit.varRef(2, ver3_0, 1);
Expand Down Expand Up @@ -405,8 +405,8 @@ try
ASSERT_NE(wal, nullptr);

// Stage 2. Apply with only puts
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
{
PageEntriesEdit edit;
Expand Down Expand Up @@ -435,8 +435,8 @@ try
}

// Stage 3. Apply with puts and refs
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
{
PageEntriesEdit edit;
Expand Down Expand Up @@ -468,9 +468,9 @@ try


// Stage 4. Apply with delete and upsert
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
{
Expand Down Expand Up @@ -514,8 +514,8 @@ try

std::vector<size_t> size_each_edit;
// Stage 1. Apply with only puts
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
{
PageEntriesEdit edit;
Expand All @@ -526,8 +526,8 @@ try
}

// Stage 2. Apply with puts and refs
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
{
PageEntriesEdit edit;
Expand All @@ -540,9 +540,9 @@ try
}

// Stage 3. Apply with delete and upsert
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
{
Expand Down Expand Up @@ -615,7 +615,7 @@ try
PageVersion ver(/*seq*/ 32);
for (size_t i = 0; i < num_edits_test; ++i)
{
PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntriesEdit edit;
const size_t num_pages_put = d_20(rd);
for (size_t p = 0; p < num_pages_put; ++p)
Expand Down Expand Up @@ -660,7 +660,7 @@ try
.persisted_log_files = persisted_log_files};

PageEntriesEdit snap_edit;
PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
std::uniform_int_distribution<> d_10000(0, 10000);
// just fill in some random entry
for (size_t i = 0; i < 70; ++i)
Expand Down
2 changes: 2 additions & 0 deletions dbms/src/TestUtils/ColumnsToTiPBExpr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ void columnToTiPBExpr(tipb::Expr * expr, const ColumnWithTypeAndName column, siz
if (column.column->isColumnNullable())
{
auto [col, null_map] = removeNullable(column.column.get());
(void)null_map;
is_const = col->isColumnConst();
}
}
Expand Down Expand Up @@ -97,6 +98,7 @@ void columnsToTiPBExprForTiDBCast(
if (type_column.column->isColumnNullable())
{
auto [col, null_map] = removeNullable(type_column.column.get());
(void)null_map;
is_const = col->isColumnConst();
}
}
Expand Down