diff --git a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp index 48fd9fe5789..2f916a2706d 100644 --- a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp +++ b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -45,7 +44,6 @@ class BasicManualCompactTest BasicManualCompactTest() { - log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName()); pk_type = GetParam(); } @@ -60,7 +58,7 @@ class BasicManualCompactTest setupStorage(); // In tests let's only compact one segment. - db_context->setSetting("manual_compact_more_until_ms", UInt64(0)); + db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(0))); // Split into 4 segments, and prepare some delta data for first 3 segments. helper = std::make_unique(*db_context); @@ -113,8 +111,6 @@ class BasicManualCompactTest std::unique_ptr manager; DM::tests::DMTestEnv::PkType pk_type; - - [[maybe_unused]] Poco::Logger * log; }; @@ -312,7 +308,7 @@ CATCH TEST_P(BasicManualCompactTest, CompactMultiple) try { - db_context->setSetting("manual_compact_more_until_ms", UInt64(60 * 1000)); // Hope it's long enough! + db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(60 * 1000))); // Hope it's long enough! auto request = ::kvrpcpb::CompactRequest(); request.set_physical_table_id(TABLE_ID); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index 08c38659636..310162e7476 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -629,8 +629,8 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false); fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, { - static size_t fail_point_called = 0; - if (existed_path.empty() && fail_point_called % 10 == 0) + static std::atomic fail_point_called(0); + if (existed_path.empty() && fail_point_called.load() % 10 == 0) { existed_path = ""; } diff --git a/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h b/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h index 9343e78def3..15c28b1f2fe 100644 --- a/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h +++ b/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h @@ -109,6 +109,7 @@ class MultiSegmentTestUtil : private boost::noncopyable // Check there is only one segment ASSERT_EQ(store->segments.size(), 1); const auto & [_key, seg] = *store->segments.begin(); + (void)_key; ASSERT_EQ(seg->getDelta()->getRows(), n_avg_rows_per_segment * 4); ASSERT_EQ(seg->getStable()->getRows(), 0); @@ -135,6 +136,7 @@ class MultiSegmentTestUtil : private boost::noncopyable ASSERT_EQ(expected_delta_rows[i], 0); ASSERT_GT(expected_stable_rows[i], 0); // We don't check the exact rows of each segment. total_stable_rows += expected_stable_rows[i]; + (void)_key; } ASSERT_EQ(total_stable_rows, 4 * n_avg_rows_per_segment); } @@ -166,6 +168,7 @@ class MultiSegmentTestUtil : private boost::noncopyable auto segment_idx = 0; for (auto & [_key, seg] : store->segments) { + (void)_key; ASSERT_EQ(seg->getDelta()->getRows(), expected_delta_rows[segment_idx]) << "Assert failed for segment #" << segment_idx; ASSERT_EQ(seg->getStable()->getRows(), expected_stable_rows[segment_idx]) << "Assert failed for segment #" << segment_idx; segment_idx++; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index fbb38980dea..07f2e1f54b4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -3374,7 +3374,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest public: DeltaMergeStoreMergeDeltaBySegmentTest() { - log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName()); std::tie(ps_ver, pk_type) = GetParam(); } @@ -3417,8 +3416,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest UInt64 ps_ver; DMTestEnv::PkType pk_type; - - [[maybe_unused]] Poco::Logger * log; }; INSTANTIATE_TEST_CASE_P( diff --git a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp index 216a6fe8aa9..78389180419 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp @@ -99,6 +99,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id1, .size = 128, + .padded_size = 0, .tag = 0, .offset = 1024, .checksum = 0x4567, @@ -106,6 +107,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id1, .size = 512, + .padded_size = 0, .tag = 0, .offset = 2048, .checksum = 0x4567, @@ -113,6 +115,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id2, .size = 512, + .padded_size = 0, .tag = 0, .offset = 2048, .checksum = 0x4567, diff --git a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp index 6d47adabbc5..b4e6c2d9204 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp @@ -34,8 +34,8 @@ namespace DB::PS::V3::tests { TEST(WALSeriTest, AllPuts) { - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); PageEntriesEdit edit; edit.put(1, entry_p1); @@ -56,8 +56,8 @@ TEST(WALSeriTest, AllPuts) TEST(WALSeriTest, PutsAndRefsAndDels) try { - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); PageEntriesEdit edit; edit.put(3, entry_p3); @@ -104,9 +104,9 @@ CATCH TEST(WALSeriTest, Upserts) { - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); PageEntriesEdit edit; @@ -164,7 +164,7 @@ TEST(WALSeriTest, RefExternalAndEntry) { PageEntriesEdit edit; - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; edit.varEntry(1, ver1_0, entry_p1_2, 2); edit.varDel(1, ver2_0); edit.varRef(2, ver3_0, 1); @@ -405,8 +405,8 @@ try ASSERT_NE(wal, nullptr); // Stage 2. Apply with only puts - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); { PageEntriesEdit edit; @@ -435,8 +435,8 @@ try } // Stage 3. Apply with puts and refs - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); { PageEntriesEdit edit; @@ -468,9 +468,9 @@ try // Stage 4. Apply with delete and upsert - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); { @@ -514,8 +514,8 @@ try std::vector size_each_edit; // Stage 1. Apply with only puts - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); { PageEntriesEdit edit; @@ -526,8 +526,8 @@ try } // Stage 2. Apply with puts and refs - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); { PageEntriesEdit edit; @@ -540,9 +540,9 @@ try } // Stage 3. Apply with delete and upsert - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); { @@ -615,7 +615,7 @@ try PageVersion ver(/*seq*/ 32); for (size_t i = 0; i < num_edits_test; ++i) { - PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageEntriesEdit edit; const size_t num_pages_put = d_20(rd); for (size_t p = 0; p < num_pages_put; ++p) @@ -660,7 +660,7 @@ try .persisted_log_files = persisted_log_files}; PageEntriesEdit snap_edit; - PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; std::uniform_int_distribution<> d_10000(0, 10000); // just fill in some random entry for (size_t i = 0; i < 70; ++i) diff --git a/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp new file mode 100644 index 00000000000..ea19ff08dd3 --- /dev/null +++ b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp @@ -0,0 +1,205 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +namespace DB +{ +namespace tests +{ +namespace +{ +void columnToTiPBExpr(tipb::Expr * expr, const ColumnWithTypeAndName column, size_t index) +{ + ColumnInfo ci = reverseGetColumnInfo({column.name, column.type}, 0, Field(), true); + bool is_const = false; + if (column.column != nullptr) + { + is_const = column.column->isColumnConst(); + if (!is_const) + { + if (column.column->isColumnNullable()) + { + auto [col, null_map] = removeNullable(column.column.get()); + (void)null_map; + is_const = col->isColumnConst(); + } + } + } + if (is_const) + { + Field val_field; + column.column->get(0, val_field); + literalFieldToTiPBExpr(ci, val_field, expr, 0); + } + else + { + *(expr->mutable_field_type()) = columnInfoToFieldType(ci); + expr->set_tp(tipb::ExprType::ColumnRef); + WriteBufferFromOwnString ss; + encodeDAGInt64(index, ss); + expr->set_val(ss.releaseStr()); + } +} +void columnsToTiPBExprForRegExp( + tipb::Expr * expr, + const String &, + const ColumnNumbers & argument_column_number, + const ColumnsWithTypeAndName & columns, + const TiDB::TiDBCollatorPtr & collator) +{ + expr->set_tp(tipb::ExprType::ScalarFunc); + if (collator == nullptr || !collator->isBinary()) + expr->set_sig(tipb::ScalarFuncSig::RegexpUTF8Sig); + else + expr->set_sig(tipb::ScalarFuncSig::RegexpSig); + for (size_t i = 0; i < argument_column_number.size(); ++i) + { + auto * argument_expr = expr->add_children(); + columnToTiPBExpr(argument_expr, columns[argument_column_number[i]], i); + } + /// since we don't know the type, just set a fake one + expr->mutable_field_type()->set_tp(TiDB::TypeLongLong); + if (collator != nullptr) + expr->mutable_field_type()->set_collate(-collator->getCollatorId()); +} +void columnsToTiPBExprForTiDBCast( + tipb::Expr * expr, + const String & func_name, + const ColumnNumbers & argument_column_number, + const ColumnsWithTypeAndName & columns, + const TiDB::TiDBCollatorPtr & collator) +{ + expr->set_tp(tipb::ExprType::ScalarFunc); + expr->set_sig(reverseGetFuncSigByFuncName(func_name)); + assert(argument_column_number.size() == 2); + const auto & type_column = columns[argument_column_number[1]]; + bool is_const = false; + if (type_column.column != nullptr) + { + is_const = type_column.column->isColumnConst(); + if (!is_const) + { + if (type_column.column->isColumnNullable()) + { + auto [col, null_map] = removeNullable(type_column.column.get()); + (void)null_map; + is_const = col->isColumnConst(); + } + } + } + assert(is_const && removeNullable(type_column.type)->isString()); + Field val; + type_column.column->get(0, val); + String type_string = val.safeGet(); + DataTypePtr target_type = DataTypeFactory::instance().get(type_string); + auto * argument_expr = expr->add_children(); + columnToTiPBExpr(argument_expr, columns[argument_column_number[0]], 0); + ColumnInfo ci = reverseGetColumnInfo({type_string, target_type}, 0, Field(), true); + *(expr->mutable_field_type()) = columnInfoToFieldType(ci); + if (collator != nullptr) + expr->mutable_field_type()->set_collate(-collator->getCollatorId()); +} + +const std::unordered_map date_add_sub_map({ + {"addDays", "DAY"}, + {"addWeeks", "WEEK"}, + {"addMonths", "MONTH"}, + {"addYears", "YEAR"}, + {"addHours", "HOUR"}, + {"addMinutes", "MINUTE"}, + {"addSeconds", "SECOND"}, + {"subtractDays", "DAY"}, + {"subtractWeeks", "WEEK"}, + {"subtractMonths", "MONTH"}, + {"subtractYears", "YEAR"}, + {"subtractHours", "HOUR"}, + {"subtractMinutes", "MINUTE"}, + {"subtractSeconds", "SECOND"}, +}); + +void columnsToTiPBExprForDateAddSub( + tipb::Expr * expr, + const String & func_name, + const ColumnNumbers & argument_column_number, + const ColumnsWithTypeAndName & columns, + const TiDB::TiDBCollatorPtr & collator) +{ + String name = func_name.substr(0, 3) == "add" ? "date_add" : "date_sub"; + expr->set_tp(tipb::ExprType::ScalarFunc); + expr->set_sig(reverseGetFuncSigByFuncName(name)); + for (size_t i = 0; i < argument_column_number.size(); ++i) + { + auto * argument_expr = expr->add_children(); + columnToTiPBExpr(argument_expr, columns[argument_column_number[i]], i); + } + String unit = date_add_sub_map.find(func_name)->second; + *(expr->add_children()) = constructStringLiteralTiExpr(unit); + /// since we don't know the type, just set a fake one + expr->mutable_field_type()->set_tp(TiDB::TypeLongLong); + if (collator != nullptr) + expr->mutable_field_type()->set_collate(-collator->getCollatorId()); +} +void columnsToTiPBExpr( + tipb::Expr * expr, + const String & func_name, + const ColumnNumbers & argument_column_number, + const ColumnsWithTypeAndName & columns, + const TiDB::TiDBCollatorPtr & collator) +{ + if (func_name == "tidb_cast") + { + columnsToTiPBExprForTiDBCast(expr, func_name, argument_column_number, columns, collator); + } + else if (func_name == "regexp") + { + columnsToTiPBExprForRegExp(expr, func_name, argument_column_number, columns, collator); + } + else if (date_add_sub_map.find(func_name) != date_add_sub_map.end()) + { + columnsToTiPBExprForDateAddSub(expr, func_name, argument_column_number, columns, collator); + } + else + { + expr->set_tp(tipb::ExprType::ScalarFunc); + expr->set_sig(reverseGetFuncSigByFuncName(func_name)); + for (size_t i = 0; i < argument_column_number.size(); ++i) + { + auto * argument_expr = expr->add_children(); + columnToTiPBExpr(argument_expr, columns[argument_column_number[i]], i); + } + /// since we don't know the type, just set a fake one + expr->mutable_field_type()->set_tp(TiDB::TypeLongLong); + if (collator != nullptr) + expr->mutable_field_type()->set_collate(-collator->getCollatorId()); + } +} +} // namespace + +tipb::Expr columnsToTiPBExpr( + const String & func_name, + const ColumnNumbers & argument_column_number, + const ColumnsWithTypeAndName & columns, + const TiDB::TiDBCollatorPtr & collator) +{ + tipb::Expr ret; + columnsToTiPBExpr(&ret, func_name, argument_column_number, columns, collator); + return ret; +} +} // namespace tests +} // namespace DB