Skip to content

Commit

Permalink
[improvment](planner) unset common fields to reduce plan thrift size (a…
Browse files Browse the repository at this point in the history
…pache#12495)

1. For query with 1656 union, the plan thrift size will be reduced from 400MB+ to 2MB.
This optimization is introduced from apache#4904, but lost after apache#9720

2. Disable ExprSubstitutionMap.verify when debug is disable.
So that the plan time of query with 1656 union will be reduced from 20s to 2s
  • Loading branch information
morningman authored and Henry2SS committed Sep 9, 2022
1 parent 6bd4272 commit 6cac744
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 23 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -279,17 +279,21 @@ public String debugString() {
* and that all rhs exprs are analyzed.
*/
private void verify() {
for (int i = 0; i < lhs_.size(); ++i) {
for (int j = i + 1; j < lhs_.size(); ++j) {
if (lhs_.get(i).equals(lhs_.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
// This method is very very time consuming, especially when planning large complex query.
// So disable it by default.
if (LOG.isDebugEnabled()) {
for (int i = 0; i < lhs_.size(); ++i) {
for (int j = i + 1; j < lhs_.size(); ++j) {
if (lhs_.get(i).equals(lhs_.get(j))) {
if (LOG.isTraceEnabled()) {
LOG.trace("verify: smap=" + this.debugString());
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
// TODO(zc): partition by k1, order by k1, there is failed.
// Preconditions.checkState(false);
}
Preconditions.checkState(!checkAnalyzed_ || rhs_.get(i).isAnalyzed());
}
Preconditions.checkState(!checkAnalyzed_ || rhs_.get(i).isAnalyzed());
}
}

Expand Down
20 changes: 5 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,6 @@ public class Coordinator {
// copied from TQueryExecRequest; constant across all fragments
private TDescriptorTable descTable;

private Set<Long> alreadySentBackendIds = Sets.newHashSet();

// Why do we use query global?
// When `NOW()` function is in sql, we need only one now(),
// but, we execute `NOW()` distributed.
Expand Down Expand Up @@ -390,7 +388,6 @@ public void clearExportStatus() {
}
this.exportFiles.clear();
this.needCheckBackendExecStates.clear();
this.alreadySentBackendIds.clear();
} finally {
lock.unlock();
}
Expand Down Expand Up @@ -716,9 +713,6 @@ private void waitRpc(List<Pair<BackendExecStates, Future<PExecPlanFragmentResult
throw new UserException(errMsg, exception);
}
}

// succeed to send the plan fragment, update the "alreadySentBackendIds"
alreadySentBackendIds.add(pair.first.beId);
}
}

Expand Down Expand Up @@ -2002,15 +1996,11 @@ public BackendExecState(PlanFragmentId fragmentId, int instanceId, int profileFr
* This information can be obtained from the cache of BE.
*/
public void unsetFields() {
if (alreadySentBackendIds.contains(backend.getId())) {
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
} else {
this.rpcParams.setIsSimplifiedParam(false);
}
this.rpcParams.unsetDescTbl();
this.rpcParams.unsetCoord();
this.rpcParams.unsetQueryGlobals();
this.rpcParams.unsetResourceInfo();
this.rpcParams.setIsSimplifiedParam(true);
}

// update profile.
Expand Down

0 comments on commit 6cac744

Please sign in to comment.