diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp b/be/src/olap/calc_delete_bitmap_executor.cpp index 284c03c985d45d..e4c905b32796d7 100644 --- a/be/src/olap/calc_delete_bitmap_executor.cpp +++ b/be/src/olap/calc_delete_bitmap_executor.cpp @@ -56,6 +56,12 @@ Status CalcDeleteBitmapToken::submit(TabletSharedPtr tablet, RowsetSharedPtr cur _status = st; } } + + RowsetIdUnorderedSet rowsetids; + for (const auto& rowset : target_rowsets) { + rowsetids.emplace(rowset->rowset_id()); + } + tablet->add_sentinel_mark_to_delete_bitmap(bitmap, rowsetids); }); } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 9348f78119a779..6944671ee528c3 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3025,6 +3025,11 @@ Status Tablet::calc_delete_bitmap(RowsetSharedPtr rowset, RETURN_IF_ERROR(calc_segment_delete_bitmap(rowset, segments[i], specified_rowsets, seg_delete_bitmap, end_version, rowset_writer)); + RowsetIdUnorderedSet rowsetids; + for (const auto& rowset : specified_rowsets) { + rowsetids.emplace(rowset->rowset_id()); + } + add_sentinel_mark_to_delete_bitmap(seg_delete_bitmap, rowsetids); } } @@ -3317,6 +3322,10 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, << ", cur max_version: " << cur_version << ", transaction_id: " << txn_id << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows; + // check if all the rowset has ROWSET_SENTINEL_MARK + RETURN_IF_ERROR(check_delete_bitmap_correctness(delete_bitmap, cur_version - 1)); + remove_sentinel_mark_from_delete_bitmap(delete_bitmap); + // update version without write lock, compaction and publish_txn // will update delete bitmap, handle compaction with _rowset_update_lock // and publish_txn runs sequential so no need to lock here @@ -3636,4 +3645,43 @@ Status Tablet::calc_delete_bitmap_between_segments( return Status::OK(); } +void Tablet::add_sentinel_mark_to_delete_bitmap(DeleteBitmapPtr delete_bitmap, + const RowsetIdUnorderedSet& rowsetids) { + for (const auto& rowsetid : rowsetids) { + delete_bitmap->add({rowsetid, 0, 0}, DeleteBitmap::ROWSET_SENTINEL_MARK); + } +} + +void Tablet::remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) { + for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) { + bitmap.remove(DeleteBitmap::ROWSET_SENTINEL_MARK); + } +} + +Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, + int64_t max_version) const { + std::map result; + for (const auto& rowsetid : all_rs_id(max_version)) { + result.emplace(rowsetid, false); + } + for (const auto& [key, bitmap] : delete_bitmap->delete_bitmap) { + auto it = result.find(std::get<0>(key)); + if (it == result.end()) { + LOG(WARNING) << "can't find rowsetid when checking delete bitmap correctness"; + return Status::InternalError("check delete bitmap correctness failed"); + } + if (bitmap.contains(DeleteBitmap::ROWSET_SENTINEL_MARK)) { + it->second = true; + } + } + for (const auto& [rowsetid, value] : result) { + if (!value) { + LOG(WARNING) << "check delete bitmap correctness failed, can't find setinel mark in " + "rowset with RowsetId: " + << rowsetid; + return Status::InternalError("check delete bitmap correctness failed"); + } + } + return Status::OK(); +} } // namespace doris diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 3efd2c89ce19f8..c20f24bd41493f 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -545,6 +545,12 @@ class Tablet : public BaseTablet { void set_binlog_config(BinlogConfig binlog_config); + void add_sentinel_mark_to_delete_bitmap(DeleteBitmapPtr delete_bitmap, + const RowsetIdUnorderedSet& rowsetids); + void remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); + Status check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, + int64_t max_version) const; + private: Status _init_once_action(); void _print_missed_versions(const std::vector& missed_versions) const; diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 70830b82de7b7e..c7ffe07fc9d8fc 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -302,6 +303,7 @@ class DeleteBitmap { using Version = uint64_t; using BitmapKey = std::tuple; std::map delete_bitmap; // Ordered map + constexpr static uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits::max() - 1; /** *