diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index ecba51fe50d822..236e5d4ac7bf21 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -708,19 +708,18 @@ Status Compaction::do_inverted_index_compaction() {
         }
 
         std::vector<lucene::store::Directory*> dest_index_dirs(dest_segment_num);
-        std::vector<lucene::store::Directory*> src_index_dirs(src_segment_num);
         try {
+            std::vector<std::unique_ptr<DorisCompoundReader>> src_idx_dirs(src_segment_num);
             for (int src_segment_id = 0; src_segment_id < src_segment_num; src_segment_id++) {
-                auto src_dir =
+                src_idx_dirs[src_segment_id] =
                         DORIS_TRY(inverted_index_file_readers[src_segment_id]->open(index_meta));
-                src_index_dirs[src_segment_id] = src_dir.release();
             }
             for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; dest_segment_id++) {
                 auto* dest_dir =
                         DORIS_TRY(inverted_index_file_writers[dest_segment_id]->open(index_meta));
                 dest_index_dirs[dest_segment_id] = dest_dir;
             }
-            auto st = compact_column(index_meta->index_id(), src_index_dirs, dest_index_dirs,
+            auto st = compact_column(index_meta->index_id(), src_idx_dirs, dest_index_dirs,
                                      index_tmp_path.native(), trans_vec, dest_segment_num_rows);
             if (!st.ok()) {
                 error_handler(index_meta->index_id(), column_uniq_id);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
index e47189f9137ada..c51d63d96ceb42 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
@@ -24,7 +24,8 @@
 #include "util/debug_points.h"
 
 namespace doris::segment_v2 {
-Status compact_column(int64_t index_id, std::vector<lucene::store::Directory*>& src_index_dirs,
+Status compact_column(int64_t index_id,
+                      std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
                       std::vector<lucene::store::Directory*>& dest_index_dirs,
                       std::string_view tmp_path,
                       const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& trans_vec,
@@ -47,7 +48,11 @@ Status compact_column(int64_t index_id, std::vector<lucene::store::Directory*>&
                                                            true /* closeDirOnShutdown */);
 
     DCHECK_EQ(src_index_dirs.size(), trans_vec.size());
-    index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec,
+    std::vector<lucene::store::Directory*> tmp_src_index_dirs(src_index_dirs.size());
+    for (size_t i = 0; i < tmp_src_index_dirs.size(); ++i) {
+        tmp_src_index_dirs[i] = src_index_dirs[i].get();
+    }
+    index_writer->indexCompaction(tmp_src_index_dirs, dest_index_dirs, trans_vec,
                                   dest_segment_num_rows);
 
     index_writer->close();
@@ -56,12 +61,6 @@ Status compact_column(int64_t index_id, std::vector<lucene::store::Directory*>&
     // when index_writer is destroyed, if closeDir is set, dir will be close
     // _CLDECDELETE(dir) will try to ref_cnt--, when it decreases to 1, dir will be destroyed.
     _CLDECDELETE(dir)
-    for (auto* d : src_index_dirs) {
-        if (d != nullptr) {
-            d->close();
-            _CLDELETE(d);
-        }
-    }
     for (auto* d : dest_index_dirs) {
         if (d != nullptr) {
             // NOTE: DO NOT close dest dir here, because it will be closed when dest index writer finalize.
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
index c95a4a7ffae1f8..1a6e4748e033d3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include "common/status.h"
+#include "inverted_index_compound_reader.h"
 
 namespace doris {
 class TabletIndex;
@@ -30,7 +31,8 @@ namespace segment_v2 {
 class InvertedIndexFileWriter;
 class InvertedIndexFileReader;
 
-Status compact_column(int64_t index_id, std::vector<lucene::store::Directory*>& src_index_dirs,
+Status compact_column(int64_t index_id,
+                      std::vector<std::unique_ptr<DorisCompoundReader>>& src_index_dirs,
                       std::vector<lucene::store::Directory*>& dest_index_dirs,
                       std::string_view tmp_path,
                       const std::vector<std::vector<std::pair<uint32_t, uint32_t>>>& trans_vec,