add limits for simultaneous compaction (ydb-platform#1096)

* add limits for simultaneous compaction * fix build
ivanmorozov333 · Jan 18, 2024 · d160c50 · d160c50
1 parent 2d2adb6
commit d160c50
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
@@ -1433,6 +1433,7 @@ message TColumnShardConfig {
     optional bool WritingEnabled = 7 [default = true];
     optional uint32 WritingBufferDurationMs = 8 [default = 0];
     optional bool UseChunkedMergeOnCompaction = 9 [default = true];
+    optional uint64 CompactionMemoryLimit = 10 [default = 536870912];
 }
 
 message TSchemeShardConfig {

diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h
@@ -713,7 +713,8 @@ class TPortionsBucket: public TMoveOnly {
         }
         std::optional<NArrow::TReplaceKey> stopPoint;
         std::optional<TInstant> stopInstant;
-        std::vector<std::shared_ptr<TPortionInfo>> portions = Others.GetOptimizerTaskPortions(512 * 1024 * 1024, stopPoint);
+        const ui64 memLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetCompactionMemoryLimit() : 512 * 1024 * 1024;
+        std::vector<std::shared_ptr<TPortionInfo>> portions = Others.GetOptimizerTaskPortions(memLimit, stopPoint);
         if (nextBorder) {
             if (MainPortion) {
                 portions.emplace_back(MainPortion);