update

apache · Jul 27, 2023 · 6e14ac1 · 6e14ac1
1 parent ae5e39a
commit 6e14ac1
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 0 deletions.
diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp b/be/src/olap/calc_delete_bitmap_executor.cpp
@@ -56,6 +56,12 @@ Status CalcDeleteBitmapToken::submit(TabletSharedPtr tablet, RowsetSharedPtr cur
                 _status = st;
             }
         }
+
+        RowsetIdUnorderedSet rowsetids;
+        for (const auto& rowset : target_rowsets) {
+            rowsetids.emplace(rowset->rowset_id());
+        }
+        tablet->add_sentinel_mark_to_delete_bitmap(bitmap, rowsetids);
     });
 }
 

diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
@@ -3025,6 +3025,11 @@ Status Tablet::calc_delete_bitmap(RowsetSharedPtr rowset,
             RETURN_IF_ERROR(calc_segment_delete_bitmap(rowset, segments[i], specified_rowsets,
                                                        seg_delete_bitmap, end_version,
                                                        rowset_writer));
+            RowsetIdUnorderedSet rowsetids;
+            for (const auto& rowset : specified_rowsets) {
+                rowsetids.emplace(rowset->rowset_id());
+            }
+            add_sentinel_mark_to_delete_bitmap(seg_delete_bitmap, rowsetids);
         }
     }
 
@@ -3317,6 +3322,10 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset,
               << ", cur max_version: " << cur_version << ", transaction_id: " << txn_id
               << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows;
 
+    // check if all the rowset has ROWSET_SENTINEL_MARK
+    RETURN_IF_ERROR(check_delete_bitmap_correctness(delete_bitmap, cur_version - 1));
+    remove_sentinel_mark_from_delete_bitmap(delete_bitmap);
+
     // update version without write lock, compaction and publish_txn
     // will update delete bitmap, handle compaction with _rowset_update_lock
     // and publish_txn runs sequential so no need to lock here
@@ -3636,4 +3645,43 @@ Status Tablet::calc_delete_bitmap_between_segments(
     return Status::OK();
 }
 
+void Tablet::add_sentinel_mark_to_delete_bitmap(DeleteBitmapPtr delete_bitmap,
+                                                const RowsetIdUnorderedSet& rowsetids) {
+    for (const auto& rowsetid : rowsetids) {
+        delete_bitmap->add({rowsetid, 0, 0}, DeleteBitmap::ROWSET_SENTINEL_MARK);
+    }
+}
+
+void Tablet::remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap) {
+    for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
+        bitmap.remove(DeleteBitmap::ROWSET_SENTINEL_MARK);
+    }
+}
+
+Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+                                               int64_t max_version) const {
+    std::map<RowsetId, bool> result;
+    for (const auto& rowsetid : all_rs_id(max_version)) {
+        result.emplace(rowsetid, false);
+    }
+    for (const auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
+        auto it = result.find(std::get<0>(key));
+        if (it == result.end()) {
+            LOG(WARNING) << "can't find rowsetid when checking delete bitmap correctness";
+            return Status::InternalError("check delete bitmap correctness failed");
+        }
+        if (bitmap.contains(DeleteBitmap::ROWSET_SENTINEL_MARK)) {
+            it->second = true;
+        }
+    }
+    for (const auto& [rowsetid, value] : result) {
+        if (!value) {
+            LOG(WARNING) << "check delete bitmap correctness failed, can't find setinel mark in "
+                            "rowset with RowsetId: "
+                         << rowsetid;
+            return Status::InternalError("check delete bitmap correctness failed");
+        }
+    }
+    return Status::OK();
+}
 } // namespace doris
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
@@ -545,6 +545,12 @@ class Tablet : public BaseTablet {
 
     void set_binlog_config(BinlogConfig binlog_config);
 
+    void add_sentinel_mark_to_delete_bitmap(DeleteBitmapPtr delete_bitmap,
+                                            const RowsetIdUnorderedSet& rowsetids);
+    void remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap);
+    Status check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
+                                           int64_t max_version) const;
+
 private:
     Status _init_once_action();
     void _print_missed_versions(const std::vector<Version>& missed_versions) const;

diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
@@ -23,6 +23,7 @@
 
 #include <atomic>
 #include <cstddef>
+#include <limits>
 #include <map>
 #include <memory>
 #include <mutex>
@@ -302,6 +303,7 @@ class DeleteBitmap {
     using Version = uint64_t;
     using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
     std::map<BitmapKey, roaring::Roaring> delete_bitmap; // Ordered map
+    constexpr static uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits<uint32_t>::max() - 1;
 
     /**
      *