Add DumpOrdinalIndex and VerifyPageChecksum meta tool

be/src/tools/meta_tool.cpp

@@ -93,6 +93,10 @@ using starrocks::ColumnIteratorOptions;
 using starrocks::PageFooterPB;
 using starrocks::DeltaColumnGroupList;
 using starrocks::PrimaryKeyDump;
+using starrocks::ColumnMetaPB;
+using starrocks::OrdinalIndexPB;
+using starrocks::ColumnIndexTypePB;
+using starrocks::OrdinalIndexReader;
 
 DEFINE_string(root_path, "", "storage root path");
 DEFINE_string(operation, "",
@@ -113,6 +117,8 @@ DEFINE_int64(expired_sec, 86400, "expired seconds");
 DEFINE_string(conf_file, "", "conf file path");
 DEFINE_string(audit_file, "", "audit file path");
 DEFINE_bool(do_delete, false, "do delete files");
+DEFINE_uint64(page_offset, -1, "page offset");
+DEFINE_uint32(page_size, -1, "page size");
 
 // flag defined in gflags library
 DECLARE_bool(helpshort);
@@ -144,6 +150,10 @@ std::string get_usage(const std::string& progname) {
       {progname} --operation=ls --root_path=</path/to/storage/path>
     show_meta:
       {progname} --operation=show_meta --pb_meta_path=<path>
+    dump_ordinal_index:
+      {progname} --operation=dump_ordinal_index --file=</path/to/segment/file> --column_index=<column index>
+    verify_page_checksum:
+      {progname} --operation=verify_page_checksum --file=</path/to/segment/file> --page_offset=<page offset> --page_size=<page size>
     show_segment_footer:
       {progname} --operation=show_segment_footer --file=</path/to/segment/file>
     dump_segment_data:
@@ -538,6 +548,84 @@ void show_segment_footer(const std::string& file_name) {
     std::cout << json_footer << std::endl;
 }
 
+void verify_page_checksum(const std::string& file_name, const PagePointer& page_pointer) {
+    auto res = starrocks::FileSystem::Default()->new_random_access_file(file_name);
+    if (!res.ok()) {
+        std::cout << "open file failed: " << res.status() << std::endl;
+        return;
+    }
+    auto input_file = std::move(res).value();
+
+    const uint32_t page_size = page_pointer.size;
+    std::unique_ptr<char[]> page(new char[page_size + starrocks::Column::APPEND_OVERFLOW_MAX_SIZE]);
+    Slice page_slice(page.get(), page_size);
+
+    input_file->read_at_fully(page_pointer.offset, page_slice.data, page_slice.size);
+
+    uint32_t expect = starrocks::decode_fixed32_le((uint8_t*)page_slice.data + page_slice.size - 4);
+    uint32_t actual = starrocks::crc32c::Value(page_slice.data, page_slice.size - 4);
+    if (expect != actual) {
+        std::cout << "Bad page: checksum mismatch (actual=" << actual << " vs expect=" << expect << ")";
+        return;
+    }
+}
+
+void dump_ordinal_index(const ColumnMetaPB& column_meta, RandomAccessFile* input_file) {
+    for (auto& index_meta : column_meta.indexes()) {
+        if (index_meta.type() == ColumnIndexTypePB::ORDINAL_INDEX) {
+            const OrdinalIndexPB& ordinal_index_meta = index_meta.ordinal_index();
+
+            auto reader = std::make_unique<OrdinalIndexReader>();
+            starrocks::IndexReadOptions opts;
+            starrocks::OlapReaderStatistics stats;
+            opts.use_page_cache = false;
+            opts.kept_in_memory = false;
+            opts.read_file = input_file;
+            opts.stats = &stats;
+
+            auto st = reader->load(opts, ordinal_index_meta, column_meta.num_rows());
+            if (!st.ok()) {
+                std::cout << "load ordinal index failed: " << st.status() << std::endl;
+                return;
+            }
+
+            auto iter = reader->begin();
+            while (true) {
+                auto page_index = iter.page_index();
+                auto pp = iter.page();
+                std::cout << "PAGE(" << page_index << "): PagePointer(offset: " << pp.offset << ", size: " << pp.size
+                          << ")" << std::endl;
+                iter.next();
+                if (!iter.valid()) {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+void dump_ordinal_index(const std::string& file_name, const int32_t column_index) {
+    auto res = starrocks::FileSystem::Default()->new_random_access_file(file_name);
+    if (!res.ok()) {
+        std::cout << "open file failed: " << res.status() << std::endl;
+        return;
+    }
+    auto input_file = std::move(res).value();
+    SegmentFooterPB footer;
+    auto status = get_segment_footer(input_file.get(), &footer);
+    if (!status.ok()) {
+        std::cout << "get footer failed: " << status.to_string() << std::endl;
+        return;
+    }
+
+    ColumnMetaPB column_meta = footer.columns(column_index);
+    dump_ordinal_index(column_meta, input_file.get());
+
+    for (const ColumnMetaPB& child_col_meta : column_meta.children_columns()) {
+        dump_ordinal_index(child_col_meta, input_file.get());
+    }
+}
+
 // This function will check the consistency of tablet meta and segment_footer
 // #issue 5415
 void check_meta_consistency(DataDir* data_dir) {
@@ -1047,6 +1135,26 @@ int meta_tool_main(int argc, char** argv) {
         }
 
         batch_delete_meta(tablet_file);
+    } else if (FLAGS_operation == "dump_ordinal_index") {
+        if (FLAGS_file == "") {
+            std::cout << "no file flag for dump ordinal index" << std::endl;
+            return -1;
+        }
+        if (FLAGS_column_index == -1) {
+            std::cout << "no column_index flag for dump ordinal index" << std::endl;
+            return -1;
+        }
+        dump_ordinal_index(FLAGS_file, FLAGS_column_index);
+    } else if (FLAGS_operation == "verify_page_checksum") {
+        if (FLAGS_file == "") {
+            std::cout << "no file flag for verify page checksum" << std::endl;
+            return -1;
+        }
+        if (FLAGS_page_offset == -1 || FLAGS_page_size == -1) {
+            std::cout << "no page offset or page size flag for verify page checksum" << std::endl;
+            return -1;
+        }
+        verify_page_checksum(FLAGS_file, {FLAGS_page_offset, FLAGS_page_size});
     } else if (FLAGS_operation == "show_segment_footer") {
         if (FLAGS_file == "") {
             std::cout << "no file flag for show dict" << std::endl;