Skip to content

Commit

Permalink
Fix some fail cases when enable TASN (pingcap#5086)
Browse files Browse the repository at this point in the history 
close pingcap#5085
  • Loading branch information
@jiaqizho @JaySon-Huang
jiaqizho authored and JaySon-Huang committed Nov 28, 2022
1 parent 23bbf21 commit 82c581b
Show file tree
Hide file tree
Showing 7 changed files with 239 additions and 35 deletions.
8 changes: 2 additions & 6 deletions dbms/src/Flash/Management/tests/gtest_manual_compact.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

#include <Common/SyncPoint/SyncPoint.h>
#include <Flash/Management/ManualCompact.h>
#include <Poco/Logger.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/DeltaMerge/RowKeyRange.h>
#include <Storages/DeltaMerge/tests/MultiSegmentTestUtil.h>
Expand Down Expand Up @@ -45,7 +44,6 @@ class BasicManualCompactTest

BasicManualCompactTest()
{
log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName());
pk_type = GetParam();
}

Expand All @@ -60,7 +58,7 @@ class BasicManualCompactTest
setupStorage();

// In tests let's only compact one segment.
db_context->setSetting("manual_compact_more_until_ms", UInt64(0));
db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(0)));

// Split into 4 segments, and prepare some delta data for first 3 segments.
helper = std::make_unique<DM::tests::MultiSegmentTestUtil>(*db_context);
Expand Down Expand Up @@ -113,8 +111,6 @@ class BasicManualCompactTest
std::unique_ptr<DB::Management::ManualCompactManager> manager;

DM::tests::DMTestEnv::PkType pk_type;

[[maybe_unused]] Poco::Logger * log;
};


Expand Down Expand Up @@ -312,7 +308,7 @@ CATCH
TEST_P(BasicManualCompactTest, CompactMultiple)
try
{
db_context->setSetting("manual_compact_more_until_ms", UInt64(60 * 1000)); // Hope it's long enough!
db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(60 * 1000))); // Hope it's long enough!

auto request = ::kvrpcpb::CompactRequest();
request.set_physical_table_id(TABLE_ID);
Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Storages/DeltaMerge/StoragePool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,8 +629,8 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons

auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false);
fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, {
static size_t fail_point_called = 0;
if (existed_path.empty() && fail_point_called % 10 == 0)
static std::atomic<UInt64> fail_point_called(0);
if (existed_path.empty() && fail_point_called.load() % 10 == 0)
{
existed_path = "<mock for existed path>";
}
Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
// Check there is only one segment
ASSERT_EQ(store->segments.size(), 1);
const auto & [_key, seg] = *store->segments.begin();
(void)_key;
ASSERT_EQ(seg->getDelta()->getRows(), n_avg_rows_per_segment * 4);
ASSERT_EQ(seg->getStable()->getRows(), 0);

Expand All @@ -135,6 +136,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
ASSERT_EQ(expected_delta_rows[i], 0);
ASSERT_GT(expected_stable_rows[i], 0); // We don't check the exact rows of each segment.
total_stable_rows += expected_stable_rows[i];
(void)_key;
}
ASSERT_EQ(total_stable_rows, 4 * n_avg_rows_per_segment);
}
Expand Down Expand Up @@ -166,6 +168,7 @@ class MultiSegmentTestUtil : private boost::noncopyable
auto segment_idx = 0;
for (auto & [_key, seg] : store->segments)
{
(void)_key;
ASSERT_EQ(seg->getDelta()->getRows(), expected_delta_rows[segment_idx]) << "Assert failed for segment #" << segment_idx;
ASSERT_EQ(seg->getStable()->getRows(), expected_stable_rows[segment_idx]) << "Assert failed for segment #" << segment_idx;
segment_idx++;
Expand Down
3 changes: 0 additions & 3 deletions dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3374,7 +3374,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest
public:
DeltaMergeStoreMergeDeltaBySegmentTest()
{
log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName());
std::tie(ps_ver, pk_type) = GetParam();
}

Expand Down Expand Up @@ -3417,8 +3416,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest

UInt64 ps_ver;
DMTestEnv::PkType pk_type;

[[maybe_unused]] Poco::Logger * log;
};

INSTANTIATE_TEST_CASE_P(
Expand Down
3 changes: 3 additions & 0 deletions dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,23 @@ try
stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 128,
.padded_size = 0,
.tag = 0,
.offset = 1024,
.checksum = 0x4567,
});
stats.restoreByEntry(PageEntryV3{
.file_id = file_id1,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
});
stats.restoreByEntry(PageEntryV3{
.file_id = file_id2,
.size = 512,
.padded_size = 0,
.tag = 0,
.offset = 2048,
.checksum = 0x4567,
Expand Down
48 changes: 24 additions & 24 deletions dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ namespace DB::PS::V3::tests
{
TEST(WALSeriTest, AllPuts)
{
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
PageEntriesEdit edit;
edit.put(1, entry_p1);
Expand All @@ -56,8 +56,8 @@ TEST(WALSeriTest, AllPuts)
TEST(WALSeriTest, PutsAndRefsAndDels)
try
{
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
PageEntriesEdit edit;
edit.put(3, entry_p3);
Expand Down Expand Up @@ -104,9 +104,9 @@ CATCH

TEST(WALSeriTest, Upserts)
{
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
PageEntriesEdit edit;
Expand Down Expand Up @@ -164,7 +164,7 @@ TEST(WALSeriTest, RefExternalAndEntry)

{
PageEntriesEdit edit;
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
edit.varEntry(1, ver1_0, entry_p1_2, 2);
edit.varDel(1, ver2_0);
edit.varRef(2, ver3_0, 1);
Expand Down Expand Up @@ -405,8 +405,8 @@ try
ASSERT_NE(wal, nullptr);

// Stage 2. Apply with only puts
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
{
PageEntriesEdit edit;
Expand Down Expand Up @@ -435,8 +435,8 @@ try
}

// Stage 3. Apply with puts and refs
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
{
PageEntriesEdit edit;
Expand Down Expand Up @@ -468,9 +468,9 @@ try


// Stage 4. Apply with delete and upsert
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
{
Expand Down Expand Up @@ -514,8 +514,8 @@ try

std::vector<size_t> size_each_edit;
// Stage 1. Apply with only puts
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20(/*seq=*/20);
{
PageEntriesEdit edit;
Expand All @@ -526,8 +526,8 @@ try
}

// Stage 2. Apply with puts and refs
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver21(/*seq=*/21);
{
PageEntriesEdit edit;
Expand All @@ -540,9 +540,9 @@ try
}

// Stage 3. Apply with delete and upsert
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1);
PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1);
{
Expand Down Expand Up @@ -615,7 +615,7 @@ try
PageVersion ver(/*seq*/ 32);
for (size_t i = 0; i < num_edits_test; ++i)
{
PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntriesEdit edit;
const size_t num_pages_put = d_20(rd);
for (size_t p = 0; p < num_pages_put; ++p)
Expand Down Expand Up @@ -660,7 +660,7 @@ try
.persisted_log_files = persisted_log_files};

PageEntriesEdit snap_edit;
PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567};
PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567};
std::uniform_int_distribution<> d_10000(0, 10000);
// just fill in some random entry
for (size_t i = 0; i < 70; ++i)
Expand Down
Loading

0 comments on commit 82c581b

Please sign in to comment.