From 56973496998a167125fd8701402f6ea4c844c5d9 Mon Sep 17 00:00:00 2001 From: Karol Sobczak Date: Wed, 10 Nov 2021 15:38:14 +0100 Subject: [PATCH] Increase split loader concurrency to 64 sf1000 orc partitioned results concurrency TPCH wall time TPC-DS wall time TPCH CPU time TPC-DS CPU time 0 4 930.229833 1263.213167 96815.4 127909.085000 1 64 904.222000 1164.819667 94923.5 127906.049667 2 32 934.815000 1260.967000 96956.3 128975.193333 --- .../src/main/java/io/trino/plugin/hive/HiveConfig.java | 2 +- .../src/test/java/io/trino/plugin/hive/TestHiveConfig.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java index 959ac033fcba..634ff350ad84 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java @@ -72,7 +72,7 @@ public class HiveConfig private int minPartitionBatchSize = 10; private int maxPartitionBatchSize = 100; private int maxInitialSplits = 200; - private int splitLoaderConcurrency = 4; + private int splitLoaderConcurrency = 64; private Integer maxSplitsPerSecond; private DataSize maxInitialSplitSize; private int domainCompactionThreshold = 100; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java index b3d503ab24fd..b99c79533c01 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java @@ -46,7 +46,7 @@ public void testDefaults() .setMaxPartitionBatchSize(100) .setMaxInitialSplits(200) .setMaxInitialSplitSize(DataSize.of(32, Unit.MEGABYTE)) - .setSplitLoaderConcurrency(4) + .setSplitLoaderConcurrency(64) .setMaxSplitsPerSecond(null) .setDomainCompactionThreshold(100) .setTargetMaxFileSize(DataSize.of(1, Unit.GIGABYTE))