Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

branch-3.0: [fix](nereids)Solve the problem of pruning wrong partitions in multi-column partition pruning #43657

Closed
wants to merge 1 commit into from
Closed

branch-3.0: [fix](nereids)Solve the problem of pruning wrong partitions in multi-column partition pruning #43657

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions ...main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator
private final List<List<Expression>> inputs;
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
private final Map<Expression, ColumnRange> rangeMap = new HashMap<>();

/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots,
Expand Down Expand Up @@ -175,8 +174,8 @@ public List<Map<Slot, PartitionSlotInput>> getOnePartitionInputs() {
@Override
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
Map<Expression, ColumnRange> defaultColumnRanges = currentInputs.values().iterator().next().columnRanges;
rangeMap.putAll(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs));
Map<Expression, ColumnRange> rangeMap = new HashMap<>(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs, rangeMap));
return result.result;
}

Expand Down Expand Up @@ -397,6 +396,7 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.intersect(rightRange));

result = returnFalseIfExistEmptyRange(result);
Expand Down Expand Up @@ -425,6 +425,7 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) {
result.childrenResult);
}
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.union(rightRange));
return returnFalseIfExistEmptyRange(result);
}
Expand All @@ -437,8 +438,8 @@ public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
for (Map.Entry<Expression, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
Expression expr = entry.getKey();
ColumnRange childRange = entry.getValue();
ColumnRange partitionRange = rangeMap.containsKey(expr)
? rangeMap.get(expr) : ColumnRange.all();
ColumnRange partitionRange = context.rangeMap.containsKey(expr)
? context.rangeMap.get(expr) : ColumnRange.all();
newRanges.put(expr, partitionRange.intersect(childRange.complete()));
}
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
Expand Down Expand Up @@ -562,6 +563,7 @@ private Map<Expression, ColumnRange> replaceExprRange(Map<Expression, ColumnRang

private EvaluateRangeResult mergeRanges(
Expression originResult, EvaluateRangeResult left, EvaluateRangeResult right,
Map<Expression, ColumnRange> rangeMap,
BiFunction<ColumnRange, ColumnRange, ColumnRange> mergeFunction) {

Map<Expression, ColumnRange> leftRanges = left.columnRanges;
Expand Down Expand Up @@ -623,7 +625,7 @@ public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInpu
if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
partitionSlotContainsNull.put(dateTrunc, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

@Override
Expand All @@ -636,7 +638,7 @@ public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) {
if (partitionSlotContainsNull.containsKey(dateChild)) {
partitionSlotContainsNull.put(date, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

@Override
Expand All @@ -649,7 +651,7 @@ public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInpu
if (partitionSlotContainsNull.containsKey(converTzChild)) {
partitionSlotContainsNull.put(convertTz, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}

private boolean isPartitionSlot(Slot slot) {
Expand Down Expand Up @@ -681,10 +683,12 @@ private Map<Slot, PartitionSlotInput> fillSlotRangesToInputs(

/** EvaluateRangeInput */
public static class EvaluateRangeInput {
private Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Expression, ColumnRange> rangeMap;

public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput) {
public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput, Map<Expression, ColumnRange> rangeMap) {
this.slotToInput = slotToInput;
this.rangeMap = rangeMap;
}
}

Expand Down Expand Up @@ -816,7 +820,8 @@ private List<Map<Slot, PartitionSlotInput>> commonComputeOnePartitionInputs() {
return onePartitionInputs;
}

private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) {
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result,
Map<Expression, ColumnRange> rangeMap) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
Expand Down
25 changes: 25 additions & 0 deletions regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,4 +334,29 @@ suite("test_date_trunc_prune") {
( NOT ( ( table1 . `col_int_undef_signed` != table1 . `col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' ) AND
table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1 ORDER BY field1 LIMIT 1000;
"""

// test multi column partition table and predicate with date_trunc
sql "drop table if exists t_multi_column_partition"
sql """
create table t_multi_column_partition(a int, dt datetime, v int) partition by range(a, dt)
(
partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')),
partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')),
partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')),
partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')),
partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00'))
)
distributed by hash(a) properties("replication_num"="1");
"""
sql """
insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),
(10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2),
(25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2),
(40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000),
(2,'2023-01-10 01:00:00',1000)
"""
explain {
sql """select * from t_multi_column_partition where a=2 and date_trunc(dt,'day') <'2024-01-1 00:00:00';"""
contains ("partitions=1/5 (p0)")
}
}