Increase split loader concurrency to 64

sf1000 orc partitioned results concurrency TPCH wall time TPC-DS wall time TPCH CPU time TPC-DS CPU time 0 4 930.229833 1263.213167 96815.4 127909.085000 1 64 904.222000 1164.819667 94923.5 127906.049667 2 32 934.815000 1260.967000 96956.3 128975.193333
trinodb · Nov 17, 2021 · 5697349 · 5697349
1 parent 5f79547
commit 5697349
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java
@@ -72,7 +72,7 @@ public class HiveConfig
     private int minPartitionBatchSize = 10;
     private int maxPartitionBatchSize = 100;
     private int maxInitialSplits = 200;
-    private int splitLoaderConcurrency = 4;
+    private int splitLoaderConcurrency = 64;
     private Integer maxSplitsPerSecond;
     private DataSize maxInitialSplitSize;
     private int domainCompactionThreshold = 100;

diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java
@@ -46,7 +46,7 @@ public void testDefaults()
                 .setMaxPartitionBatchSize(100)
                 .setMaxInitialSplits(200)
                 .setMaxInitialSplitSize(DataSize.of(32, Unit.MEGABYTE))
-                .setSplitLoaderConcurrency(4)
+                .setSplitLoaderConcurrency(64)
                 .setMaxSplitsPerSecond(null)
                 .setDomainCompactionThreshold(100)
                 .setTargetMaxFileSize(DataSize.of(1, Unit.GIGABYTE))