[Improve] Use number of columns to count segment cache

Lchangliang · Jan 9, 2024 · ed5c650 · ed5c650
1 parent 8a1fd83
commit ed5c650
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 3 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -1054,6 +1054,7 @@ DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
 
 // max number of segment cache, default -1 for backward compatibility fd_number*2/5
 DEFINE_mInt32(segment_cache_capacity, "-1");
+DEFINE_mInt32(each_segment_have_columns, "30");
 
 // enable feature binlog, default false
 DEFINE_Bool(enable_feature_binlog, "false");

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -1092,6 +1092,7 @@ DECLARE_mInt32(schema_cache_sweep_time_sec);
 
 // max number of segment cache
 DECLARE_mInt32(segment_cache_capacity);
+DECLARE_mInt32(each_segment_have_columns);
 
 // enable binlog
 DECLARE_Bool(enable_feature_binlog);

diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
@@ -184,6 +184,8 @@ class Segment : public std::enable_shared_from_this<Segment> {
         return safe;
     }
 
+    const TabletSchemaSPtr& tablet_schema() { return _tablet_schema; }
+
 private:
     DISALLOW_COPY_AND_ASSIGN(Segment);
     Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema);

diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
@@ -45,8 +45,9 @@ void SegmentCache::insert(const SegmentCache::CacheKey& key, SegmentCache::Cache
         delete cache_value;
     };
 
-    auto lru_handle = cache()->insert(key.encode(), &value, 1, deleter, CachePriority::NORMAL,
-                                      value.segment->meta_mem_usage());
+    auto lru_handle =
+            cache()->insert(key.encode(), &value, value.segment->tablet_schema()->num_columns(),
+                            deleter, CachePriority::NORMAL, 0);
     handle->push_segment(cache(), lru_handle);
 }
 

diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
@@ -438,7 +438,7 @@ Status ExecEnv::_init_mem_env() {
     }
     LOG(INFO) << "segment_cache_capacity <= fd_number * 2 / 5, fd_number: " << fd_number
               << " segment_cache_capacity: " << segment_cache_capacity;
-    _segment_loader = new SegmentLoader(segment_cache_capacity);
+    _segment_loader = new SegmentLoader(segment_cache_capacity * config::each_segment_have_columns);
 
     _schema_cache = new SchemaCache(config::schema_cache_capacity);