From 464aebcf539cc82d94f335e081660d0df3bf82d1 Mon Sep 17 00:00:00 2001 From: before-Sunrise <71162020+before-Sunrise@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:50:17 +0800 Subject: [PATCH] [BugFix] fix list partition multi values' probelm (#51036) Signed-off-by: before-Sunrise (cherry picked from commit f3ba5b878767616afc7c6af15dd41702223739f0) --- .../com/starrocks/catalog/ListPartitionInfo.java | 2 +- .../rule/transformation/ListPartitionPruner.java | 15 +++++++++++++-- .../starrocks/sql/plan/PartitionPruneTest.java | 6 ++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java index 1a309625632bd..261dcb823cee6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/ListPartitionInfo.java @@ -328,7 +328,7 @@ public Map> getIdToValues() { * @param id * @return true if the partition can be pruned */ - public boolean pruneById(long id) { + public boolean isSingleValuePartition(long id) { List values = getIdToValues().get(id); if (values != null && values.size() == 1) { return true; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java index 03571678c12a7..54d719d2182aa 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ListPartitionPruner.java @@ -589,7 +589,7 @@ private Set evalBinaryPredicate(BinaryPredicateOperator binaryPredicate) { } else { Set partitionIds = partitionValueMap.get(literal); for (Long id : partitionIds) { - if (listPartitionInfo.pruneById(id)) { + if (listPartitionInfo.isSingleValuePartition(id)) { matches.remove(id); } } @@ -706,7 +706,18 @@ private Set evalInPredicate(InPredicateOperator inPredicate) { Set partitions = partitionValueMap.get(literal); if (partitions != null) { if (inPredicate.isNotIn()) { - matches.removeAll(partitions); + // external table, one partition column for one partition can only have one value + if (listPartitionInfo == null) { + matches.removeAll(partitions); + } else { + // for olap table, if one partition is multi value partition like PARTITION pCalifornia VALUES IN ("Los Angeles","San Francisco","San Diego") + // and we have a not in predicate like city not in ("Los Angeles"), it's not safe to remove this partition + for (Long id : partitions) { + if (listPartitionInfo.isSingleValuePartition(id)) { + matches.remove(id); + } + } + } } else { matches.addAll(partitions); } diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java index 5ff658424dcee..3ada493599a0a 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java @@ -394,6 +394,12 @@ public void testMinMaxPrune_ListValues() throws Exception { starRocksAssert.query("select min(c1-1)+1, max(c1-1)-1 from t1_list_multi_values") .explainContains("OlapScanNode"); + // multi-value partition doesn't support partition prune + starRocksAssert.query("select * from t1_list_multi_values where c1 not in (10, 2, 8, 5)") + .explainContains("partitions=4/4"); + + starRocksAssert.query("select * from t1_list_multi_values where c1 != 10") + .explainContains("partitions=4/4"); // TODO: not supported // multi-item list partition