From e52e56106360db709aff8a24ec5d81dbed07202d Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 10 Dec 2024 14:53:04 +0800 Subject: [PATCH] fix some code --- .../mv/AbstractMaterializedViewRule.java | 77 ++----- .../rules/exploration/mv/Predicates.java | 214 ++++++++++-------- .../exploration/mv/PredicatesSplitter.java | 33 ++- .../rules/exploration/mv/StructInfo.java | 2 +- .../expression/PredicatesSplitterTest.java | 8 +- .../mv/date_trunc/mv_with_date_trunc.groovy | 134 +++++++++++ 6 files changed, 297 insertions(+), 171 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 9ab33416f7d9c0..c7049edfe57ecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.ExpressionInfo; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PartitionRemover; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; @@ -46,12 +47,10 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; -import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.JoinType; @@ -566,7 +565,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf */ protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, BitSet sourcePlanBitSet, - Map shuttledQueryMap, CascadesContext cascadesContext) { + Map queryExprToInfoMap, CascadesContext cascadesContext) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions // can not be represented by target expressions, return null. @@ -580,7 +579,8 @@ protected List rewriteExpression(List sourceEx flattenExpressionMap.get(0); List rewrittenExpressions = new ArrayList<>(); - for (Expression expressionShuttledToRewrite : sourceShuttledExpressions) { + for (int exprIndex = 0; exprIndex < sourceShuttledExpressions.size(); exprIndex++) { + Expression expressionShuttledToRewrite = sourceShuttledExpressions.get(exprIndex); if (expressionShuttledToRewrite instanceof Literal) { rewrittenExpressions.add(expressionShuttledToRewrite); continue; @@ -599,44 +599,35 @@ protected List rewriteExpression(List sourceEx // if contains any slot to rewrite, which means can not be rewritten by target, // expressionShuttledToRewrite is slot#0 > '2024-01-01' but mv plan output is date_trunc(slot#0, 'day') // which would try to rewrite - // paramExpressionToDateTruncMap is {slot#0 : date_trunc(slot#0, 'day')} - Map paramExpressionToDateTruncMap = new HashMap<>(); + // viewExpressionParamToDateTruncMap is {slot#0 : date_trunc(slot#0, 'day')} + Map viewExpressionParamToDateTruncMap = new HashMap<>(); targetToTargetReplacementMappingQueryBased.keySet().forEach(expr -> { if (expr instanceof DateTrunc) { - paramExpressionToDateTruncMap.put(expr.child(0), (DateTrunc) expr); + viewExpressionParamToDateTruncMap.put(expr.child(0), (DateTrunc) expr); } }); - Expression queryExpr = expressionShuttledToRewrite.child(0); - Map shuttledQueryParamToExpressionMap = new HashMap<>(); - // TODO: 2024/12/5 optimize performance - for (Map.Entry expressionEntry : shuttledQueryMap.entrySet()) { - Expression shuttledQueryParamExpression = ExpressionUtils.shuttleExpressionWithLineage( - expressionEntry.getKey(), sourcePlan, sourcePlanBitSet); - shuttledQueryParamToExpressionMap.put(shuttledQueryParamExpression.child(0) instanceof Literal - ? shuttledQueryParamExpression.child(1) : shuttledQueryParamExpression.child(0), - expressionEntry.getValue()); - } + Expression queryUsedExpr = expressionShuttledToRewrite.child(0); - if (paramExpressionToDateTruncMap.isEmpty() || shuttledQueryMap.isEmpty() - || !shuttledQueryMap.containsKey(expressionShuttledToRewrite) - || !paramExpressionToDateTruncMap.containsKey(queryExpr)) { + if (!queryExprToInfoMap.containsKey(sourceExpressionsToWrite.get(exprIndex)) + || !viewExpressionParamToDateTruncMap.containsKey(queryUsedExpr)) { // mv date_trunc expression can not offer expression for query, // can not try to rewrite by date_trunc, bail out return ImmutableList.of(); } Map datetruncMap = new HashMap<>(); - Literal queryLiteral = shuttledQueryMap.get(expressionShuttledToRewrite); - datetruncMap.put(queryExpr, queryLiteral); + Literal queryLiteral = queryExprToInfoMap.get(expressionShuttledToRewrite) == null + ? null : queryExprToInfoMap.get(expressionShuttledToRewrite).literal; + datetruncMap.put(queryUsedExpr, queryLiteral); Expression replacedWithLiteral = ExpressionUtils.replace( - paramExpressionToDateTruncMap.get(queryExpr), datetruncMap); + viewExpressionParamToDateTruncMap.get(queryUsedExpr), datetruncMap); Expression foldedExpressionWithLiteral = FoldConstantRuleOnFE.evaluate(replacedWithLiteral, new ExpressionRewriteContext(cascadesContext)); if (foldedExpressionWithLiteral.equals(queryLiteral)) { // after date_trunc simplify if equals to original expression, could rewritten by mv replacedExpression = ExpressionUtils.replace(expressionShuttledToRewrite, targetToTargetReplacementMappingQueryBased, - paramExpressionToDateTruncMap); + viewExpressionParamToDateTruncMap); } if (replacedExpression.anyMatch(slotsToRewrite::contains)) { return ImmutableList.of(); @@ -802,42 +793,20 @@ protected SplitPredicate predicatesCompensate( } // viewEquivalenceClass to query based // equal predicate compensate - final Set equalCompensateConjunctions = Predicates.compensateEquivalence( - queryStructInfo, - viewStructInfo, - viewToQuerySlotMapping, - comparisonResult); + final Map equalCompensateConjunctions = Predicates.compensateEquivalence( + queryStructInfo, viewStructInfo, viewToQuerySlotMapping, comparisonResult); // range compensate - final Map rangeCompensatePredicates = Predicates.compensateRangePredicate( - queryStructInfo, - viewStructInfo, - viewToQuerySlotMapping, - comparisonResult, - cascadesContext); + final Map rangeCompensatePredicates = + Predicates.compensateRangePredicate(queryStructInfo, viewStructInfo, viewToQuerySlotMapping, + comparisonResult, cascadesContext); // residual compensate - final Set residualCompensatePredicates = Predicates.compensateResidualPredicate( - queryStructInfo, - viewStructInfo, - viewToQuerySlotMapping, - comparisonResult); + final Map residualCompensatePredicates = Predicates.compensateResidualPredicate( + queryStructInfo, viewStructInfo, viewToQuerySlotMapping, comparisonResult); if (equalCompensateConjunctions == null || rangeCompensatePredicates == null || residualCompensatePredicates == null) { return SplitPredicate.INVALID_INSTANCE; } - if (equalCompensateConjunctions.stream().anyMatch(expr -> expr.containsType(AggregateFunction.class)) - || rangeCompensatePredicates.keySet().stream() - .anyMatch(expr -> expr.containsType(AggregateFunction.class)) - || residualCompensatePredicates.stream().anyMatch(expr -> - expr.containsType(AggregateFunction.class))) { - return SplitPredicate.INVALID_INSTANCE; - } - return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE - : ExpressionUtils.and(equalCompensateConjunctions), - rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE - : ExpressionUtils.and(rangeCompensatePredicates.keySet()), - rangeCompensatePredicates.isEmpty() ? ImmutableMap.of() : rangeCompensatePredicates, - residualCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE - : ExpressionUtils.and(residualCompensatePredicates)); + return SplitPredicate.of(equalCompensateConjunctions, rangeCompensatePredicates, residualCompensatePredicates); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 24f786f94486e1..b3dc2d26835220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -27,8 +27,9 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.util.ExpressionUtils; @@ -46,7 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; /** @@ -87,7 +87,7 @@ public static SplitPredicate splitPredicates(Expression expression) { /** * compensate equivalence predicates */ - public static Set compensateEquivalence(StructInfo queryStructInfo, + public static Map compensateEquivalence(StructInfo queryStructInfo, StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, ComparisonResult comparisonResult) { @@ -98,9 +98,9 @@ public static Set compensateEquivalence(StructInfo queryStructInfo, if (viewEquivalenceClassQueryBased == null) { return null; } - final Set equalCompensateConjunctions = new HashSet<>(); + final Map equalCompensateConjunctions = new HashMap<>(); if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { - equalCompensateConjunctions.add(BooleanLiteral.TRUE); + return ImmutableMap.of(); } if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { return null; @@ -115,37 +115,42 @@ public static Set compensateEquivalence(StructInfo queryStructInfo, // do equal compensate Set> mappedQueryEquivalenceSet = queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet(); - queryEquivalenceClass.getEquivalenceSetList().forEach( - queryEquivalenceSet -> { - // compensate the equivalence in query but not in view - if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) { - Iterator iterator = queryEquivalenceSet.iterator(); - SlotReference first = iterator.next(); - while (iterator.hasNext()) { - Expression equals = new EqualTo(first, iterator.next()); - equalCompensateConjunctions.add(equals); - } - } else { - // compensate the equivalence both in query and view, but query has more equivalence - List viewEquivalenceSet = - queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); - List copiedQueryEquivalenceSet = new ArrayList<>(queryEquivalenceSet); - copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); - SlotReference first = viewEquivalenceSet.iterator().next(); - for (SlotReference slotReference : copiedQueryEquivalenceSet) { - Expression equals = new EqualTo(first, slotReference); - equalCompensateConjunctions.add(equals); - } + + for (List queryEquivalenceSet : queryEquivalenceClass.getEquivalenceSetList()) { + // compensate the equivalence in query but not in view + if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) { + Iterator iterator = queryEquivalenceSet.iterator(); + SlotReference first = iterator.next(); + while (iterator.hasNext()) { + Expression equals = new EqualTo(first, iterator.next()); + if (equals.anyMatch(AggregateFunction.class::isInstance)) { + return null; + } + equalCompensateConjunctions.put(equals, ExpressionInfo.EMPTY); + } + } else { + // compensate the equivalence both in query and view, but query has more equivalence + List viewEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); + List copiedQueryEquivalenceSet = new ArrayList<>(queryEquivalenceSet); + copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); + SlotReference first = viewEquivalenceSet.iterator().next(); + for (SlotReference slotReference : copiedQueryEquivalenceSet) { + Expression equals = new EqualTo(first, slotReference); + if (equals.anyMatch(AggregateFunction.class::isInstance)) { + return null; } + equalCompensateConjunctions.put(equals, ExpressionInfo.EMPTY); } - ); + } + } return equalCompensateConjunctions; } /** * compensate range predicates */ - public static Map compensateRangePredicate(StructInfo queryStructInfo, + public static Map compensateRangePredicate(StructInfo queryStructInfo, StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, ComparisonResult comparisonResult, @@ -153,13 +158,16 @@ public static Map compensateRangePredicate(StructInfo query SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); - Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); - Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); - Expression viewRangePredicateQueryBased = - ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap()); + Set viewRangeQueryBasedSet = new HashSet<>(); + for (Expression viewExpression : viewSplitPredicate.getRangePredicateMap().keySet()) { + viewRangeQueryBasedSet.add( + ExpressionUtils.replace(viewExpression, viewToQuerySlotMapping.toSlotReferenceMap())); + } + viewRangeQueryBasedSet.remove(BooleanLiteral.TRUE); + + Set queryRangeSet = querySplitPredicate.getRangePredicateMap().keySet(); + queryRangeSet.remove(BooleanLiteral.TRUE); - Set queryRangeSet = ExpressionUtils.extractConjunctionToSet(queryRangePredicate); - Set viewRangeQueryBasedSet = ExpressionUtils.extractConjunctionToSet(viewRangePredicateQueryBased); Set differentExpressions = new HashSet<>(); Sets.difference(queryRangeSet, viewRangeQueryBasedSet).copyInto(differentExpressions); Sets.difference(viewRangeQueryBasedSet, queryRangeSet).copyInto(differentExpressions); @@ -174,16 +182,22 @@ public static Map compensateRangePredicate(StructInfo query // normalized expressions is not in query, can not compensate return null; } - Map normalizedExpressionsWithLiteral = new HashMap<>(); + Map normalizedExpressionsWithLiteral = new HashMap<>(); for (Expression expression : normalizedExpressions) { Set literalSet = expression.collect(expressionTreeNode -> expressionTreeNode instanceof Literal); if (!(expression instanceof ComparisonPredicate) - || (expression instanceof GreaterThan || expression instanceof LessThan) + || (expression instanceof GreaterThan || expression instanceof LessThanEqual) || literalSet.size() != 1) { - normalizedExpressionsWithLiteral.put(expression, null); + if (expression.anyMatch(AggregateFunction.class::isInstance)) { + return null; + } + normalizedExpressionsWithLiteral.put(expression, ExpressionInfo.EMPTY); continue; } - normalizedExpressionsWithLiteral.put(expression, literalSet.iterator().next()); + if (expression.anyMatch(AggregateFunction.class::isInstance)) { + return null; + } + normalizedExpressionsWithLiteral.put(expression, new ExpressionInfo(literalSet.iterator().next())); } return normalizedExpressionsWithLiteral; } @@ -200,31 +214,38 @@ private static Set normalizeExpression(Expression expression, Cascad /** * compensate residual predicates */ - public static Set compensateResidualPredicate(StructInfo queryStructInfo, + public static Map compensateResidualPredicate(StructInfo queryStructInfo, StructInfo viewStructInfo, SlotMapping viewToQuerySlotMapping, ComparisonResult comparisonResult) { // TODO Residual predicates compensate, simplify implementation currently. SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); - Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); - Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); - Expression viewResidualPredicateQueryBased = - ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap()); - Set queryResidualSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); - Set viewResidualQueryBasedSet = - Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); + + Set viewResidualQueryBasedSet = new HashSet<>(); + for (Expression viewExpression : viewSplitPredicate.getResidualPredicateMap().keySet()) { + viewResidualQueryBasedSet.add( + ExpressionUtils.replace(viewExpression, viewToQuerySlotMapping.toSlotReferenceMap())); + } + viewResidualQueryBasedSet.remove(BooleanLiteral.TRUE); + + Set queryResidualSet = querySplitPredicate.getResidualPredicateMap().keySet(); // remove unnecessary literal BooleanLiteral.TRUE queryResidualSet.remove(BooleanLiteral.TRUE); - viewResidualQueryBasedSet.remove(BooleanLiteral.TRUE); // query residual predicate can not contain all view residual predicate when view have residual predicate, // bail out if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) { return null; } queryResidualSet.removeAll(viewResidualQueryBasedSet); - return queryResidualSet; + Map expressionExpressionInfoMap = new HashMap<>(); + for (Expression needCompensate : queryResidualSet) { + if (needCompensate.anyMatch(AggregateFunction.class::isInstance)) { + return null; + } + expressionExpressionInfoMap.put(needCompensate, ExpressionInfo.EMPTY); + } + return expressionExpressionInfoMap; } @Override @@ -232,51 +253,57 @@ public String toString() { return Utils.toSqlString("Predicates", "pulledUpPredicates", pulledUpPredicates); } + /** + * The struct info for expression, such as the constant that it used + */ + public static final class ExpressionInfo { + + public static final ExpressionInfo EMPTY = new ExpressionInfo(null); + + public final Literal literal; + + public ExpressionInfo(Literal literal) { + this.literal = literal; + } + } + /** * The split different representation for predicate expression, such as equal, range and residual predicate. */ public static final class SplitPredicate { public static final SplitPredicate INVALID_INSTANCE = - SplitPredicate.of(null, null, null, null); - private final Optional equalPredicate; - private final Optional rangePredicate; - private final Optional> rangePredicateMap; - private final Optional residualPredicate; - - public SplitPredicate(Expression equalPredicate, - Expression rangePredicate, - Map rangePredicateMap, - Expression residualPredicate) { - this.equalPredicate = Optional.ofNullable(equalPredicate); - this.rangePredicate = Optional.ofNullable(rangePredicate); - this.rangePredicateMap = Optional.ofNullable(rangePredicateMap); - this.residualPredicate = Optional.ofNullable(residualPredicate); - } + SplitPredicate.of(null, null, null); + private final Map equalPredicateMap; + private final Map rangePredicateMap; + private final Map residualPredicateMap; - public Expression getEqualPredicate() { - return equalPredicate.orElse(BooleanLiteral.TRUE); + public SplitPredicate(Map equalPredicateMap, + Map rangePredicateMap, + Map residualPredicateMap) { + this.equalPredicateMap = equalPredicateMap; + this.rangePredicateMap = rangePredicateMap; + this.residualPredicateMap = residualPredicateMap; } - public Expression getRangePredicate() { - return rangePredicate.orElse(BooleanLiteral.TRUE); + public Map getEqualPredicateMap() { + return equalPredicateMap; } - public Map getRangePredicateMap() { - return rangePredicateMap.orElse(ImmutableMap.of()); + public Map getRangePredicateMap() { + return rangePredicateMap; } - public Expression getResidualPredicate() { - return residualPredicate.orElse(BooleanLiteral.TRUE); + public Map getResidualPredicateMap() { + return residualPredicateMap; } /** * SplitPredicate construct */ - public static SplitPredicate of(Expression equalPredicates, - Expression rangePredicates, - Map rangePredicateSet, - Expression residualPredicates) { - return new SplitPredicate(equalPredicates, rangePredicates, rangePredicateSet, residualPredicates); + public static SplitPredicate of(Map equalPredicateMap, + Map rangePredicateMap, + Map residualPredicateMap) { + return new SplitPredicate(equalPredicateMap, rangePredicateMap, residualPredicateMap); } /** @@ -287,22 +314,21 @@ public boolean isInvalid() { } public List toList() { - return ImmutableList.of(getEqualPredicate(), getRangePredicate(), getResidualPredicate()); + if (isInvalid()) { + return ImmutableList.of(); + } + List flattenExpressions = new ArrayList<>(getEqualPredicateMap().keySet()); + flattenExpressions.addAll(getRangePredicateMap().keySet()); + flattenExpressions.addAll(getResidualPredicateMap().keySet()); + return flattenExpressions; } /** * Check the predicates in SplitPredicate is whether all true or not */ public boolean isAlwaysTrue() { - Expression equalExpr = getEqualPredicate(); - Expression rangeExpr = getRangePredicate(); - Expression residualExpr = getResidualPredicate(); - return equalExpr instanceof BooleanLiteral - && rangeExpr instanceof BooleanLiteral - && residualExpr instanceof BooleanLiteral - && ((BooleanLiteral) equalExpr).getValue() - && ((BooleanLiteral) rangeExpr).getValue() - && ((BooleanLiteral) residualExpr).getValue(); + return getEqualPredicateMap().isEmpty() && getRangePredicateMap().isEmpty() + && getResidualPredicateMap().isEmpty(); } @Override @@ -314,22 +340,22 @@ public boolean equals(Object o) { return false; } SplitPredicate that = (SplitPredicate) o; - return Objects.equals(equalPredicate, that.equalPredicate) - && Objects.equals(rangePredicate, that.rangePredicate) - && Objects.equals(residualPredicate, that.residualPredicate); + return Objects.equals(equalPredicateMap, that.equalPredicateMap) + && Objects.equals(rangePredicateMap, that.residualPredicateMap) + && Objects.equals(residualPredicateMap, that.residualPredicateMap); } @Override public int hashCode() { - return Objects.hash(equalPredicate, rangePredicate, residualPredicate); + return Objects.hash(equalPredicateMap, rangePredicateMap, residualPredicateMap); } @Override public String toString() { return Utils.toSqlString("SplitPredicate", - "equalPredicate", equalPredicate, - "rangePredicate", rangePredicate, - "residualPredicate", residualPredicate); + "equalPredicate", equalPredicateMap, + "rangePredicate", rangePredicateMap, + "residualPredicate", residualPredicateMap); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index e66a847cae7f01..a7a5af74d009ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.nereids.rules.exploration.mv.Predicates.ExpressionInfo; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; @@ -28,9 +29,9 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; import org.apache.doris.nereids.util.ExpressionUtils; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; -import java.util.Set; +import java.util.Map; /** * Split the expression to equal, range and residual predicate. @@ -39,9 +40,9 @@ */ public class PredicatesSplitter { - private final Set equalPredicates = new LinkedHashSet<>(); - private final Set rangePredicates = new LinkedHashSet<>(); - private final Set residualPredicates = new LinkedHashSet<>(); + private final Map equalPredicates = new HashMap<>(); + private final Map rangePredicates = new HashMap<>(); + private final Map residualPredicates = new HashMap<>(); private final List conjunctExpressions; public PredicatesSplitter(Expression target) { @@ -65,19 +66,19 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Vo boolean rightArgOnlyContainsColumnRef = containOnlyColumnRef(rightArg, true); if (comparisonPredicate instanceof EqualPredicate) { if (leftArgOnlyContainsColumnRef && rightArgOnlyContainsColumnRef) { - equalPredicates.add(comparisonPredicate); + equalPredicates.put(comparisonPredicate, ExpressionInfo.EMPTY); return null; } else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal) || (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) { - rangePredicates.add(comparisonPredicate); + rangePredicates.put(comparisonPredicate, ExpressionInfo.EMPTY); } else { - residualPredicates.add(comparisonPredicate); + residualPredicates.put(comparisonPredicate, ExpressionInfo.EMPTY); } } else if ((leftArgOnlyContainsColumnRef && rightArg instanceof Literal) || (rightArgOnlyContainsColumnRef && leftArg instanceof Literal)) { - rangePredicates.add(comparisonPredicate); + rangePredicates.put(comparisonPredicate, ExpressionInfo.EMPTY); } else { - residualPredicates.add(comparisonPredicate); + residualPredicates.put(comparisonPredicate, ExpressionInfo.EMPTY); } return null; } @@ -86,26 +87,22 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Vo public Void visitInPredicate(InPredicate inPredicate, Void context) { if (containOnlyColumnRef(inPredicate.getCompareExpr(), true) && (ExpressionUtils.isAllLiteral(inPredicate.getOptions()))) { - rangePredicates.add(inPredicate); + rangePredicates.put(inPredicate, ExpressionInfo.EMPTY); } else { - residualPredicates.add(inPredicate); + residualPredicates.put(inPredicate, ExpressionInfo.EMPTY); } return null; } @Override public Void visit(Expression expr, Void context) { - residualPredicates.add(expr); + residualPredicates.put(expr, ExpressionInfo.EMPTY); return null; } } public Predicates.SplitPredicate getSplitPredicate() { - return Predicates.SplitPredicate.of( - equalPredicates.isEmpty() ? null : ExpressionUtils.and(equalPredicates), - rangePredicates.isEmpty() ? null : ExpressionUtils.and(rangePredicates), - null, - residualPredicates.isEmpty() ? null : ExpressionUtils.and(residualPredicates)); + return Predicates.SplitPredicate.of(equalPredicates, rangePredicates, residualPredicates); } private static boolean containOnlyColumnRef(Expression expression, boolean allowCast) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 365360e06b096c..3f0397dc4115ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -286,7 +286,7 @@ private static Pair predicatesDerive(Predicate .collect(Collectors.toList()); SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); EquivalenceClass equivalenceClass = new EquivalenceClass(); - for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) { + for (Expression expression : splitPredicate.getEqualPredicateMap().keySet()) { if (expression instanceof Literal) { continue; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java index 0374244ce56474..a0719f2c86927d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java @@ -67,9 +67,9 @@ private void assetEquals(String expression, if (!StringUtils.isEmpty(expectedEqualExpr)) { Expression equalExpression = replaceUnboundSlot(PARSER.parseExpression(expectedEqualExpr), mem); - Assertions.assertEquals(equalExpression, splitPredicate.getEqualPredicate()); + Assertions.assertEquals(equalExpression, splitPredicate.getEqualPredicateMap()); } else { - Assertions.assertEquals(splitPredicate.getEqualPredicate(), BooleanLiteral.TRUE); + Assertions.assertEquals(splitPredicate.getEqualPredicateMap(), BooleanLiteral.TRUE); } if (!StringUtils.isEmpty(expectedRangeExpr)) { @@ -81,9 +81,9 @@ private void assetEquals(String expression, if (!StringUtils.isEmpty(expectedResidualExpr)) { Expression residualExpression = replaceUnboundSlot(PARSER.parseExpression(expectedResidualExpr), mem); - Assertions.assertEquals(residualExpression, splitPredicate.getResidualPredicate()); + Assertions.assertEquals(residualExpression, splitPredicate.getResidualPredicateMap()); } else { - Assertions.assertEquals(splitPredicate.getResidualPredicate(), BooleanLiteral.TRUE); + Assertions.assertEquals(splitPredicate.getResidualPredicateMap(), BooleanLiteral.TRUE); } } diff --git a/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy new file mode 100644 index 00000000000000..59db8204406510 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/date_trunc/mv_with_date_trunc.groovy @@ -0,0 +1,134 @@ +package mv.date_trunc +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("mv_with_date_trunc") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists lineitem + """ + + sql """ + drop table if exists lineitem; + """ + + sql """ + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey integer not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimalv3(15, 2) not null, + l_extendedprice decimalv3(15, 2) not null, + l_discount decimalv3(15, 2) not null, + l_tax decimalv3(15, 2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate DATETIME not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null + ) DUPLICATE KEY( + l_orderkey, l_partkey, l_suppkey, + l_linenumber + ) PARTITION BY RANGE(l_shipdate) ( + FROM + ('2023-10-16') TO ('2023-11-30') INTERVAL 1 DAY + ) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ("replication_num" = "1"); + """ + + + sql """ + insert into lineitem + values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', + '2023-10-17', '2023-10-17 00:10:00', + '2023-10-17', 'a', 'b', 'yyyyyyyyy' + ), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', + '2023-10-18', '2023-10-18 08:10:00', + '2023-10-18', 'a', 'b', 'yyyyyyyyy' + ), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', + '2023-10-19', '2023-10-19 12:10:00', + '2023-10-19', 'c', 'd', 'xxxxxxxxx' + ); + """ + + def mv1_0 = """ + select + l_shipmode, + date_trunc(l_shipdate, 'day') as day_trunc, + count(*) + from + lineitem + group by + l_shipmode, + date_trunc(l_shipdate, 'day'); + """ + def query1_0 = """ + select + l_shipmode, + count(*) + from + lineitem + where + l_shipdate >= '2023-10-17' and l_shipdate < '2023-10-18' + group by + l_shipmode; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + +// def mv1_0 = """ +// select +// l_shipmode, +// date_trunc(l_shipdate, 'day') as day_trunc, +// count(*) +// from +// lineitem +// group by +// l_shipmode, +// date_trunc(l_shipdate, 'day'); +// """ +// def query1_0 = """ +// select +// l_shipmode, +// count(*) +// from +// lineitem +// where +// l_shipdate >= '2023-10-17' and l_shipdate < '2023-10-18' +// group by +// l_shipmode; +// """ +// order_qt_query1_0_before "${query1_0}" +// async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") +// order_qt_query1_0_after "${query1_0}" +// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + +}