don't do correctness check when the rowset has 0 written rows

be/src/olap/calc_delete_bitmap_executor.cpp

@@ -56,12 +56,6 @@ Status CalcDeleteBitmapToken::submit(TabletSharedPtr tablet, RowsetSharedPtr cur
                 _status = st;
             }
         }
-
-        RowsetIdUnorderedSet rowsetids;
-        for (const auto& rowset : target_rowsets) {
-            rowsetids.emplace(rowset->rowset_id());
-        }
-        tablet->add_sentinel_mark_to_delete_bitmap(bitmap, rowsetids);
     });
 }
 

be/src/olap/delta_writer.cpp

@@ -489,8 +489,11 @@ Status DeltaWriter::commit_txn(const PSlaveTabletNodes& slave_tablet_nodes,
                                const bool write_single_replica) {
     std::lock_guard<std::mutex> l(_lock);
     SCOPED_TIMER(_close_wait_timer);
+    // only do correctness check if the rowset has at least one row written
+    bool do_correctness_check = (_rowset_writer->num_rows() != 0);
     Status res = _storage_engine->txn_manager()->commit_txn(_req.partition_id, _tablet, _req.txn_id,
-                                                            _req.load_id, _cur_rowset, false);
+                                                            _req.load_id, _cur_rowset, false,
+                                                            do_correctness_check);
 
     if (!res && !res.is<PUSH_TRANSACTION_ALREADY_EXIST>()) {
         LOG(WARNING) << "Failed to commit txn: " << _req.txn_id

be/src/olap/tablet.cpp

@@ -2976,6 +2976,15 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
     }
     DCHECK_EQ(total, row_id) << "segment total rows: " << total << " row_id:" << row_id;
 
+    RowsetIdUnorderedSet rowsetids;
+    for (const auto& rowset : specified_rowsets) {
+        rowsetids.emplace(rowset->rowset_id());
+        LOG(INFO) << "[tabletID:" << tablet_id() << "]"
+                  << "[add_sentinel_mark_to_delete_bitmap][end_version:" << end_version << "]"
+                  << "add:" << rowset->rowset_id();
+    }
+    add_sentinel_mark_to_delete_bitmap(delete_bitmap, rowsetids);
+
     if (pos > 0) {
         RETURN_IF_ERROR(generate_new_block_for_partial_update(
                 rowset_schema, read_plan_ori, read_plan_update, rsid_to_rowset, &block));
@@ -3025,11 +3034,6 @@ Status Tablet::calc_delete_bitmap(RowsetSharedPtr rowset,
             RETURN_IF_ERROR(calc_segment_delete_bitmap(rowset, segments[i], specified_rowsets,
                                                        seg_delete_bitmap, end_version,
                                                        rowset_writer));
-            RowsetIdUnorderedSet rowsetids;
-            for (const auto& rowset : specified_rowsets) {
-                rowsetids.emplace(rowset->rowset_id());
-            }
-            add_sentinel_mark_to_delete_bitmap(seg_delete_bitmap, rowsetids);
         }
     }
 
@@ -3274,7 +3278,7 @@ Status Tablet::commit_phase_update_delete_bitmap(
 Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset,
                                     const RowsetIdUnorderedSet& pre_rowset_ids,
                                     DeleteBitmapPtr delete_bitmap, int64_t txn_id,
-                                    RowsetWriter* rowset_writer) {
+                                    RowsetWriter* rowset_writer, bool do_correctness_check) {
     SCOPED_BVAR_LATENCY(g_tablet_update_delete_bitmap_latency);
     RowsetIdUnorderedSet cur_rowset_ids;
     RowsetIdUnorderedSet rowset_ids_to_add;
@@ -3322,9 +3326,11 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset,
               << ", cur max_version: " << cur_version << ", transaction_id: " << txn_id
               << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows;
 
-    // check if all the rowset has ROWSET_SENTINEL_MARK
-    RETURN_IF_ERROR(check_delete_bitmap_correctness(delete_bitmap, cur_version - 1));
-    remove_sentinel_mark_from_delete_bitmap(delete_bitmap);
+    if (do_correctness_check) {
+        // check if all the rowset has ROWSET_SENTINEL_MARK
+        RETURN_IF_ERROR(check_delete_bitmap_correctness(delete_bitmap, cur_version - 1));
+        remove_sentinel_mark_from_delete_bitmap(delete_bitmap);
+    }
 
     // update version without write lock, compaction and publish_txn
     // will update delete bitmap, handle compaction with _rowset_update_lock
@@ -3658,10 +3664,10 @@ void Tablet::remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitm
     }
 }
 
-Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
-                                               int64_t max_version) const {
+Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version) {
     std::map<RowsetId, bool> result;
-    for (const auto& rowsetid : all_rs_id(max_version)) {
+    RowsetIdUnorderedSet rowsets = all_rs_id(max_version);
+    for (const auto& rowsetid : rowsets) {
         result.emplace(rowsetid, false);
     }
     for (const auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
@@ -3676,12 +3682,15 @@ Status Tablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
     }
     for (const auto& [rowsetid, value] : result) {
         if (!value) {
-            LOG(WARNING) << "check delete bitmap correctness failed, can't find setinel mark in "
-                            "rowset with RowsetId: "
-                         << rowsetid;
+            LOG(WARNING) << "check delete bitmap correctness failed, can't find sentinel mark in "
+                            "rowset with RowsetId:"
+                         << rowsetid << "version:" << get_rowset(rowsetid)->version().to_string()
+                         << "max_version:" << max_version;
             return Status::InternalError("check delete bitmap correctness failed");
         }
     }
+    LOG(INFO) << "[check_delete_bitmap_correctness][max_version:" << max_version << "]"
+              << "rowset size:" << rowsets.size();
     return Status::OK();
 }
 } // namespace doris

be/src/olap/tablet.h

@@ -485,7 +485,8 @@ class Tablet : public BaseTablet {
     Status update_delete_bitmap(const RowsetSharedPtr& rowset,
                                 const RowsetIdUnorderedSet& pre_rowset_ids,
                                 DeleteBitmapPtr delete_bitmap, int64_t txn_id,
-                                RowsetWriter* rowset_writer = nullptr);
+                                RowsetWriter* rowset_writer = nullptr,
+                                bool do_correctness_check = false);
     void calc_compaction_output_rowset_delete_bitmap(
             const std::vector<RowsetSharedPtr>& input_rowsets,
             const RowIdConversion& rowid_conversion, uint64_t start_version, uint64_t end_version,
@@ -548,8 +549,7 @@ class Tablet : public BaseTablet {
     void add_sentinel_mark_to_delete_bitmap(DeleteBitmapPtr delete_bitmap,
                                             const RowsetIdUnorderedSet& rowsetids);
     void remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap);
-    Status check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap,
-                                           int64_t max_version) const;
+    Status check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version);
 
 private:
     Status _init_once_action();

be/src/olap/tablet_meta.h

@@ -303,7 +303,7 @@ class DeleteBitmap {
     using Version = uint64_t;
     using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
     std::map<BitmapKey, roaring::Roaring> delete_bitmap; // Ordered map
-    constexpr static uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits<uint32_t>::max() - 1;
+    constexpr static inline uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits<uint32_t>::max() - 1;
 
     /**
      * 

be/src/olap/txn_manager.cpp

@@ -152,10 +152,11 @@ Status TxnManager::prepare_txn(TPartitionId partition_id, TTransactionId transac
 
 Status TxnManager::commit_txn(TPartitionId partition_id, const TabletSharedPtr& tablet,
                               TTransactionId transaction_id, const PUniqueId& load_id,
-                              const RowsetSharedPtr& rowset_ptr, bool is_recovery) {
+                              const RowsetSharedPtr& rowset_ptr, bool is_recovery,
+                              bool do_correctness_check) {
     return commit_txn(tablet->data_dir()->get_meta(), partition_id, transaction_id,
                       tablet->tablet_id(), tablet->schema_hash(), tablet->tablet_uid(), load_id,
-                      rowset_ptr, is_recovery);
+                      rowset_ptr, is_recovery, do_correctness_check);
 }
 
 Status TxnManager::publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet,
@@ -217,7 +218,7 @@ Status TxnManager::commit_txn(OlapMeta* meta, TPartitionId partition_id,
                               TTransactionId transaction_id, TTabletId tablet_id,
                               SchemaHash schema_hash, TabletUid tablet_uid,
                               const PUniqueId& load_id, const RowsetSharedPtr& rowset_ptr,
-                              bool is_recovery) {
+                              bool is_recovery, bool do_correctness_check) {
     if (partition_id < 1 || transaction_id < 1 || tablet_id < 1) {
         LOG(FATAL) << "invalid commit req "
                    << " partition_id=" << partition_id << " transaction_id=" << transaction_id
@@ -366,9 +367,9 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id,
         tablet->create_transient_rowset_writer(rowset, &rowset_writer);
 
         int64_t t2 = MonotonicMicros();
-        RETURN_IF_ERROR(tablet->update_delete_bitmap(rowset, tablet_txn_info.rowset_ids,
-                                                     tablet_txn_info.delete_bitmap, transaction_id,
-                                                     rowset_writer.get()));
+        RETURN_IF_ERROR(tablet->update_delete_bitmap(
+                rowset, tablet_txn_info.rowset_ids, tablet_txn_info.delete_bitmap, transaction_id,
+                rowset_writer.get(), tablet_txn_info.do_correctness_check));
         int64_t t3 = MonotonicMicros();
         stats->calc_delete_bitmap_time_us = t3 - t2;
         if (rowset->tablet_schema()->is_partial_update()) {

be/src/olap/txn_manager.h

@@ -59,6 +59,7 @@ struct TabletTxnInfo {
     RowsetIdUnorderedSet rowset_ids;
     int64_t creation_time;
     bool ingest {false};
+    bool do_correctness_check {false};
 
     TabletTxnInfo(PUniqueId load_id, RowsetSharedPtr rowset)
             : load_id(load_id), rowset(rowset), creation_time(UnixSeconds()) {}
@@ -123,7 +124,8 @@ class TxnManager {
 
     Status commit_txn(TPartitionId partition_id, const TabletSharedPtr& tablet,
                       TTransactionId transaction_id, const PUniqueId& load_id,
-                      const RowsetSharedPtr& rowset_ptr, bool is_recovery);
+                      const RowsetSharedPtr& rowset_ptr, bool is_recovery,
+                      bool do_correctness_check = false);
 
     Status publish_txn(TPartitionId partition_id, const TabletSharedPtr& tablet,
                        TTransactionId transaction_id, const Version& version,
@@ -138,8 +140,8 @@ class TxnManager {
 
     Status commit_txn(OlapMeta* meta, TPartitionId partition_id, TTransactionId transaction_id,
                       TTabletId tablet_id, SchemaHash schema_hash, TabletUid tablet_uid,
-                      const PUniqueId& load_id, const RowsetSharedPtr& rowset_ptr,
-                      bool is_recovery);
+                      const PUniqueId& load_id, const RowsetSharedPtr& rowset_ptr, bool is_recovery,
+                      bool do_correctness_check = false);
 
     // remove a txn from txn manager
     // not persist rowset meta because