Get partitions by batch of size 5k per RPC

Signed-off-by: Letian Jiang <[email protected]>
StarRocks · Mar 9, 2023 · 73be4ec · 73be4ec
1 parent ee640ef
commit 73be4ec
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -1618,7 +1618,7 @@ public class Config extends ConfigBase {
      * The maximum number of partitions to fetch from the metastore in one RPC.
      */
     @ConfField
-    public static int max_hive_partitions_per_rpc = 1000;
+    public static int max_hive_partitions_per_rpc = 5000;
 
     /**
      * The interval of lazy refreshing remote file's metadata cache

diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveMetaClient.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveMetaClient.java
@@ -38,6 +38,7 @@
 import org.apache.thrift.transport.TTransportException;
 
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -219,7 +220,7 @@ public Partition getPartition(String dbName, String tableName, List<String> part
      */
     public List<Partition> getPartitionsByNames(String dbName, String tblName, List<String> partitionNames) {
         int size = partitionNames.size();
-        List<Partition> partitions;
+        List<Partition> partitions = new ArrayList<>();
         PlannerProfile.addCustomProperties("HMS.PARTITIONS.getPartitionsByNames." + tblName,
                 String.format("%s partitions", size));
 
@@ -228,7 +229,12 @@ public List<Partition> getPartitionsByNames(String dbName, String tblName, List<
             StarRocksConnectorException connectionException = null;
             try {
                 client = getClient();
-                partitions = client.hiveClient.getPartitionsByNames(dbName, tblName, partitionNames);
+                for (int start = 0; start < partitionNames.size(); start += Config.max_hive_partitions_per_rpc) {
+                    int end = Math.min(start + Config.max_hive_partitions_per_rpc, partitionNames.size());
+                    List<String> namesPerRPC = partitionNames.subList(start, end);
+                    List<Partition> partsPerRPC = client.hiveClient.getPartitionsByNames(dbName, tblName, namesPerRPC);
+                    partitions.addAll(partsPerRPC);
+                }
                 if (partitions.size() != partitionNames.size()) {
                     LOG.warn("Expect to fetch {} partition on [{}.{}], but actually fetched {} partition",
                             partitionNames.size(), dbName, tblName, partitions.size());