Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (backport #51304) #51326

Closed
wants to merge 1 commit into from
Closed

[BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (backport #51304) #51326

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/ReorderJoinRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

package com.starrocks.sql.optimizer.rule.join;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.starrocks.common.FeConstants;
Expand Down Expand Up @@ -267,6 +268,14 @@ private OptExpression deriveNewOptExpression(OptExpression optExpression,
LogicalProperty newProperty = new LogicalProperty(optExpression.getLogicalProperty());
newProperty.setOutputColumns(newCols);

if (!Optional.ofNullable(optExpression.getStatistics()).isPresent()) {
ExpressionContext expressionContext = new ExpressionContext(optExpression);
StatisticsCalculator statisticsCalculator = new StatisticsCalculator(
expressionContext, optimizerContext.getColumnRefFactory(), optimizerContext);
statisticsCalculator.estimatorStats();
optExpression.setStatistics(expressionContext.getStatistics());
}
Preconditions.checkState(optExpression.getStatistics() != null);
Statistics newStats = Statistics.buildFrom(optExpression.getStatistics()).build();
Iterator<Map.Entry<ColumnRefOperator, ColumnStatistic>>
iterator = newStats.getColumnStatistics().entrySet().iterator();
Expand Down
160 changes: 160 additions & 0 deletions fe/fe-core/src/test/java/com/starrocks/sql/plan/ReplayFromDumpTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.junit.Ignore;
import org.junit.Test;

<<<<<<< HEAD
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
Expand Down Expand Up @@ -160,6 +161,19 @@ protected Pair<QueryDumpInfo, String> getPlanFragment(String dumpJsonStr, Sessio
return new Pair<>(queryDumpInfo,
UtFrameUtils.getNewPlanAndFragmentFromDump(connectContext, queryDumpInfo).second.
getExplainString(level));
=======
import java.util.stream.Stream;

public class ReplayFromDumpTest extends ReplayFromDumpTestBase {
@Test
public void testForceRuleBasedRewrite() throws Exception {
QueryDumpInfo queryDumpInfo = getDumpInfoFromJson(getDumpInfoFromFile("query_dump/force_rule_based_mv_rewrite"));
SessionVariable sessionVariable = queryDumpInfo.getSessionVariable();
sessionVariable.setEnableForceRuleBasedMvRewrite(true);
Pair<QueryDumpInfo, String> replayPair =
getCostPlanFragment(getDumpInfoFromFile("query_dump/force_rule_based_mv_rewrite"), sessionVariable);
Assert.assertTrue(replayPair.second, replayPair.second.contains("partition_flat_consumptions_partition_drinks_dates"));
>>>>>>> 61ebfc79f8 ([BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (#51304))
}

@Test
Expand Down Expand Up @@ -872,4 +886,150 @@ public void testNormalizeNonTrivialProject() throws Exception {
FeConstants.USE_MOCK_DICT_MANAGER = false;
}
}
<<<<<<< HEAD
=======

@Test
public void testListPartitionPrunerWithNEExpr() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getCostPlanFragment(getDumpInfoFromFile("query_dump/list_partition_prune_dump"));
// partitions should not be pruned
Assert.assertTrue(replayPair.second, !replayPair.second.contains("partitionsRatio=2/3, tabletsRatio=20/20"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("0:OlapScanNode\n" +
" table: partitions2_keys311, rollup: partitions2_keys311\n" +
" preAggregation: on\n" +
" Predicates: [7: undef_signed_not_null, VARCHAR, false] != 'j'\n" +
" partitionsRatio=3/3, tabletsRatio=30/30"));
}

@Test
public void testTopNPushDownBelowUnionAll() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/topn_push_down_union"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);

// Topn should be pushed down below union all and contains no duplicated ording columns
PlanTestBase.assertContains(replayPair.second, " 26:TOP-N\n" +
" | order by: <slot 240> 240: expr ASC, <slot 241> 241: cast DESC, <slot 206> 206: mock_025 DESC\n" +
" | offset: 0\n" +
" | limit: 200");
PlanTestBase.assertContains(replayPair.second, "17:TOP-N\n" +
" | order by: <slot 165> 165: cast ASC, <slot 153> 153: cast DESC, <slot 166> 166: expr ASC, " +
"<slot 167> 167: cast DESC\n" +
" | offset: 0\n" +
" | limit: 200");
}

@Test
public void testNoCTEOperatorPropertyDerived() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/no_cte_operator_test"),
null, TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("23:Project\n" +
" | <slot 193> : 193: mock_081\n" +
" | <slot 194> : 194: mock_089\n" +
" | <slot 391> : 391: case\n" +
" | <slot 396> : 396: rank()"));
Assert.assertTrue(replayPair.second, replayPair.second.contains(" 20:SORT\n" +
" | order by: <slot 194> 194: mock_089 ASC," +
" <slot 395> 395: case ASC, <slot 193> 193: mock_081 ASC, " +
"<slot 233> 233: mock_065 ASC\n" +
" | analytic partition by: 194: mock_089, 395: case, 193: mock_081\n" +
" | offset: 0\n" +
" | \n" +
" 19:EXCHANGE"));
}

@Test
public void testTimeoutDeepJoinCostPrune() throws Exception {
Tracers.register(connectContext);
Tracers.init(connectContext, Tracers.Mode.TIMER, "optimizer");
connectContext.getSessionVariable().setOptimizerExecuteTimeout(-1);

Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/deep_join_cost"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
String ss = Tracers.printScopeTimer();
int start = ss.indexOf("EnforceAndCostTask[") + "EnforceAndCostTask[".length();
int end = ss.indexOf("]", start);
long count = Long.parseLong(ss.substring(start, end));
Assert.assertTrue(ss, count < 10000);
}

@Test
public void testDistinctConstantRewrite() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/distinct_constant"),
connectContext.getSessionVariable(), TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("4:AGGREGATE (update serialize)\n" +
" | output: multi_distinct_count(1)"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("9:AGGREGATE (update serialize)\n" +
" | output: multi_distinct_count(NULL)"));
}

@Test
public void testSplitOrderBy() throws Exception {
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/split_order_by"),
null, TExplainLevel.NORMAL);
Assert.assertTrue(replayPair.second, replayPair.second.contains("21:MERGING-EXCHANGE"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("20:TOP-N"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("15:MERGING-EXCHANGE"));
Assert.assertTrue(replayPair.second, replayPair.second.contains("14:TOP-N"));

}

@Test
public void testQueryCacheSetOperator() throws Exception {

String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableQueryCache(true);
QueryDumpInfo dumpInfo = getDumpInfoFromJson(getDumpInfoFromFile("query_dump/query_cache_set_operator"));
ExecPlan execPlan = UtFrameUtils.getPlanFragmentFromQueryDump(connectContext, dumpInfo);
Assert.assertTrue(execPlan.getFragments().stream().anyMatch(frag -> frag.getCacheParam() != null));
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}

@Test
public void testQueryTimeout() {
Assert.assertThrows(StarRocksPlannerException.class,
() -> getPlanFragment(getDumpInfoFromFile("query_dump/query_timeout"), null, TExplainLevel.NORMAL));
}

@Test
public void testQueryCacheMisuseExogenousRuntimeFilter() throws Exception {
String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableQueryCache(true);
QueryDumpInfo dumpInfo =
getDumpInfoFromJson(getDumpInfoFromFile("query_dump/query_cache_misuse_exogenous_runtime_filter"));
ExecPlan execPlan = UtFrameUtils.getPlanFragmentFromQueryDump(connectContext, dumpInfo);
Assert.assertTrue(execPlan.getFragments().stream().noneMatch(frag -> frag.getCacheParam() != null));
Assert.assertTrue(
execPlan.getFragments().stream().anyMatch(frag -> !frag.getProbeRuntimeFilters().isEmpty()));
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}

@Test
public void testPruneTableNPE() throws Exception {
String savedSv = connectContext.getSessionVariable().getJsonString();
try {
connectContext.getSessionVariable().setEnableCboTablePrune(true);
connectContext.getSessionVariable().setEnableRboTablePrune(true);
Pair<QueryDumpInfo, String> replayPair =
getPlanFragment(getDumpInfoFromFile("query_dump/prune_table_npe"),
null, TExplainLevel.NORMAL);
long numHashJoins = Stream.of(replayPair.second.split("\n"))
.filter(ln -> ln.contains("HASH JOIN")).count();
Assert.assertEquals(numHashJoins, 2);
} finally {
connectContext.getSessionVariable().replayFromJson(savedSv);
}
}
>>>>>>> 61ebfc79f8 ([BugFix] Operator's statistics not be estimated during JoinReorder of table-pruning phase (#51304))
}
Loading
Loading