[chore](variable) remove enable_nereids_planner flag (apache#40743)

amorynan · Sep 18, 2024 · 4c212f1 · 4c212f1
1 parent 64463f5
commit 4c212f1
Show file tree

Hide file tree

Showing 115 changed files with 483 additions and 10,513 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
@@ -262,7 +262,6 @@ public String toSqlImpl() {
                     && ConnectContext.get().getState().isNereids()
                     && !ConnectContext.get().getState().isQuery()
                     && ConnectContext.get().getSessionVariable() != null
-                    && ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()
                     && desc != null) {
                 return label + "[#" + desc.getId().asInt() + "]";
             } else {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java
@@ -231,7 +231,6 @@ public static AutoCloseConnectContext buildConnectContext() {
         sessionVariable.setEnableInsertStrict(true);
         sessionVariable.setInsertMaxFilterRatio(1);
         // sessionVariable.parallelExecInstanceNum = StatisticConstants.STATISTIC_PARALLEL_EXEC_INSTANCE_NUM;
-        sessionVariable.setEnableNereidsPlanner(false);
         sessionVariable.enableProfile = false;
         connectContext.setEnv(Env.getCurrentEnv());
         connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java
@@ -18,7 +18,6 @@
 package org.apache.doris.load;
 
 import org.apache.doris.analysis.OutFileClause;
-import org.apache.doris.analysis.SelectStmt;
 import org.apache.doris.analysis.StatementBase;
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
@@ -95,15 +94,10 @@ public void execute() throws JobException {
                     table.readLock();
                     try {
                         List<Long> tabletIds;
-                        if (exportJob.getSessionVariables().isEnableNereidsPlanner()) {
-                            LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) selectStmtLists.get(idx);
-                            Optional<UnboundRelation> unboundRelation = findUnboundRelation(
-                                    logicalPlanAdapter.getLogicalPlan());
-                            tabletIds = unboundRelation.get().getTabletIds();
-                        } else {
-                            SelectStmt selectStmt = (SelectStmt) selectStmtLists.get(idx);
-                            tabletIds = selectStmt.getTableRefs().get(0).getSampleTabletIds();
-                        }
+                        LogicalPlanAdapter logicalPlanAdapter = (LogicalPlanAdapter) selectStmtLists.get(idx);
+                        Optional<UnboundRelation> unboundRelation = findUnboundRelation(
+                                logicalPlanAdapter.getLogicalPlan());
+                        tabletIds = unboundRelation.get().getTabletIds();
 
                         for (Long tabletId : tabletIds) {
                             TabletMeta tabletMeta = Env.getCurrentEnv().getTabletInvertedIndex().getTabletMeta(

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/SimpleJobScheduler.java
@@ -18,7 +18,7 @@
 package org.apache.doris.nereids.jobs.scheduler;
 
 import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.exceptions.DoNotFallbackException;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.jobs.Job;
 import org.apache.doris.qe.SessionVariable;
 
@@ -37,7 +37,7 @@ public void executeJobPool(ScheduleContext scheduleContext) {
             if (sessionVariable.enableNereidsTimeout
                     && context.getStatementContext().getStopwatch().elapsed(TimeUnit.MILLISECONDS)
                     > sessionVariable.nereidsTimeoutSecond * 1000L) {
-                throw new DoNotFallbackException(
+                throw new AnalysisException(
                         "Nereids cost too much time ( > " + sessionVariable.nereidsTimeoutSecond + "s )");
             }
             Job job = pool.pop();

diff --git a/...ore/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java b/...ore/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
@@ -23,7 +23,6 @@
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.exceptions.AnalysisException;
-import org.apache.doris.nereids.exceptions.MustFallbackException;
 import org.apache.doris.nereids.hint.Hint;
 import org.apache.doris.nereids.hint.LeadingHint;
 import org.apache.doris.nereids.hint.OrderedHint;
@@ -108,12 +107,6 @@ private void setVar(SelectHintSetVar selectHint, StatementContext context) {
                 }
             }
         }
-        // if sv set enable_nereids_planner=true and hint set enable_nereids_planner=false, we should set
-        // enable_fallback_to_original_planner=true and revert it after executing.
-        // throw exception to fall back to original planner
-        if (!sessionVariable.isEnableNereidsPlanner()) {
-            throw new MustFallbackException("The nereids is disabled in this sql, fallback to original planner");
-        }
     }
 
     private void extractLeading(SelectHintLeading selectHint, CascadesContext context,

diff --git a/...src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java b/...src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
@@ -73,7 +73,7 @@ public void checkLegalityBeforeTypeCoercion() {
         DataType argType = ((ArrayType) child(0).getDataType()).getItemType();
         if (!(argType.isIntegralType() || argType.isFloatLikeType() || argType.isDecimalLikeType()
                 || argType.isDateLikeType() || argType.isBooleanType())) {
-            throw new AnalysisException("array_apply does not support type: " + toSql());
+            throw new AnalysisException("array_apply does not support type " + argType + ", expression is " + toSql());
         }
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterMTMVCommand.java
@@ -32,7 +32,7 @@
 /**
  * alter multi table materialized view
  */
-public class AlterMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class AlterMTMVCommand extends Command implements ForwardWithSync {
 
     public static final Logger LOG = LogManager.getLogger(AlterMTMVCommand.class);
     private final AlterMTMVInfo alterMTMVInfo;

diff --git a/...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java b/...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
@@ -30,7 +30,7 @@
 /**
  * cancel mtmv task
  */
-public class CancelMTMVTaskCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class CancelMTMVTaskCommand extends Command implements ForwardWithSync {
     private final CancelMTMVTaskInfo cancelMTMVTaskInfo;
 
     public CancelMTMVTaskCommand(CancelMTMVTaskInfo cancelMTMVTaskInfo) {

diff --git a/...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java b/...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateMTMVCommand.java
@@ -33,7 +33,7 @@
 /**
  * create multi table materialized view
  */
-public class CreateMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class CreateMTMVCommand extends Command implements ForwardWithSync {
 
     public static final Logger LOG = LogManager.getLogger(CreateMTMVCommand.class);
     private final CreateMTMVInfo createMTMVInfo;

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropMTMVCommand.java
@@ -30,7 +30,7 @@
 /**
  * refresh mtmv
  */
-public class DropMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class DropMTMVCommand extends Command implements ForwardWithSync {
     private final DropMTMVInfo dropMTMVInfo;
 
     public DropMTMVCommand(DropMTMVInfo dropMTMVInfo) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NotAllowFallback.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NotAllowFallback.java
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PauseMTMVCommand.java
@@ -30,7 +30,7 @@
 /**
  * pause mtmv
  */
-public class PauseMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class PauseMTMVCommand extends Command implements ForwardWithSync {
     private final PauseMTMVInfo pauseMTMVInfo;
 
     public PauseMTMVCommand(PauseMTMVInfo pauseMTMVInfo) {

diff --git a/...-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java b/...-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/RefreshMTMVCommand.java
@@ -30,7 +30,7 @@
 /**
  * refresh mtmv
  */
-public class RefreshMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class RefreshMTMVCommand extends Command implements ForwardWithSync {
     private final RefreshMTMVInfo refreshMTMVInfo;
 
     public RefreshMTMVCommand(RefreshMTMVInfo refreshMTMVInfo) {

diff --git a/...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java b/...e-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeMTMVCommand.java
@@ -30,7 +30,7 @@
 /**
  * resume mtmv
  */
-public class ResumeMTMVCommand extends Command implements ForwardWithSync, NotAllowFallback {
+public class ResumeMTMVCommand extends Command implements ForwardWithSync {
     private final ResumeMTMVInfo resumeMTMVInfo;
 
     public ResumeMTMVCommand(ResumeMTMVInfo resumeMTMVInfo) {

diff --git a/...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateMTMVCommand.java b/...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateMTMVCommand.java
@@ -29,7 +29,7 @@
 /**
  * resume mtmv
  */
-public class ShowCreateMTMVCommand extends Command implements NoForward, NotAllowFallback {
+public class ShowCreateMTMVCommand extends Command implements NoForward {
     private final ShowCreateMTMVInfo showCreateMTMVInfo;
 
     public ShowCreateMTMVCommand(ShowCreateMTMVInfo showCreateMTMVInfo) {

diff --git a/...java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java b/...java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
@@ -30,8 +30,6 @@
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.ScanNode;
 import org.apache.doris.planner.SchemaScanNode;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.thrift.TExplainLevel;
 
 import com.google.common.collect.ArrayListMultimap;
@@ -255,12 +253,6 @@ private static boolean shouldAssignByBucket(PlanFragment fragment) {
     }
 
     private static boolean enableBucketShuffleJoin() {
-        if (ConnectContext.get() != null) {
-            SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
-            if (!sessionVariable.isEnableBucketShuffleJoin() && !sessionVariable.isEnableNereidsPlanner()) {
-                return false;
-            }
-        }
         return true;
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
@@ -484,11 +484,7 @@ public boolean isTransferQueryStatisticsWithEveryBatch() {
     }
 
     public int getFragmentSequenceNum() {
-        if (ConnectContext.get().getSessionVariable().isEnableNereidsPlanner()) {
-            return fragmentSequenceNum;
-        } else {
-            return fragmentId.asInt();
-        }
+        return fragmentSequenceNum;
     }
 
     public void setFragmentSequenceNum(int seq) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlQueryExecutor.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.MysqlColType;
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.Type;
-import org.apache.doris.mysql.MysqlCommand;
 import org.apache.doris.plsql.exception.QueryException;
 import org.apache.doris.qe.AutoCloseConnectContext;
 import org.apache.doris.qe.ConnectContext;
@@ -47,7 +46,7 @@ public QueryResult executeQuery(String sql, ParserRuleContext ctx) {
             autoCloseCtx.call();
             context.setRunProcedure(true);
             ConnectProcessor processor = new MysqlConnectProcessor(context);
-            processor.executeQuery(MysqlCommand.COM_QUERY, sql);
+            processor.executeQuery(sql);
             StmtExecutor executor = context.getExecutor();
             if (executor.getParsedStmt().getResultExprs() != null) {
                 return new QueryResult(new DorisRowResult(executor.getCoord(), executor.getColumns(),

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -48,15 +48,13 @@
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.metric.MetricRepo;
 import org.apache.doris.mysql.MysqlChannel;
-import org.apache.doris.mysql.MysqlCommand;
 import org.apache.doris.mysql.MysqlPacket;
 import org.apache.doris.mysql.MysqlSerializer;
 import org.apache.doris.mysql.MysqlServerStatusFlag;
 import org.apache.doris.nereids.SqlCacheContext;
 import org.apache.doris.nereids.SqlCacheContext.CacheKeyType;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.exceptions.NotSupportedException;
-import org.apache.doris.nereids.exceptions.ParseException;
 import org.apache.doris.nereids.glue.LogicalPlanAdapter;
 import org.apache.doris.nereids.minidump.MinidumpUtils;
 import org.apache.doris.nereids.parser.Dialect;
@@ -218,7 +216,7 @@ protected void auditAfterExec(String origStmt, StatementBase parsedStmt,
     }
 
     // only throw an exception when there is a problem interacting with the requesting client
-    protected void handleQuery(MysqlCommand mysqlCommand, String originStmt) throws ConnectionException {
+    protected void handleQuery(String originStmt) throws ConnectionException {
         if (Config.isCloudMode()) {
             if (!ctx.getCurrentUserIdentity().isRootUser()
                     && ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getInstanceStatus()
@@ -233,15 +231,15 @@ protected void handleQuery(MysqlCommand mysqlCommand, String originStmt) throws
             }
         }
         try {
-            executeQuery(mysqlCommand, originStmt);
+            executeQuery(originStmt);
         } catch (ConnectionException exception) {
             throw exception;
         } catch (Exception ignored) {
             // saved use handleQueryException
         }
     }
 
-    public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Exception {
+    public void executeQuery(String originStmt) throws Exception {
         if (MetricRepo.isInit) {
             MetricRepo.COUNTER_REQUEST_ALL.increase(1L);
             MetricRepo.increaseClusterRequestAll(ctx.getCloudCluster(false));
@@ -252,89 +250,37 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex
         ctx.setSqlHash(sqlHash);
 
         SessionVariable sessionVariable = ctx.getSessionVariable();
-        boolean wantToParseSqlFromSqlCache = sessionVariable.isEnableNereidsPlanner()
-                && CacheAnalyzer.canUseSqlCache(sessionVariable);
+        boolean wantToParseSqlFromSqlCache = CacheAnalyzer.canUseSqlCache(sessionVariable);
         List<StatementBase> stmts = null;
-        Exception nereidsParseException = null;
-        Exception nereidsSyntaxException = null;
         long parseSqlStartTime = System.currentTimeMillis();
         List<StatementBase> cachedStmts = null;
         CacheKeyType cacheKeyType = null;
-        if (sessionVariable.isEnableNereidsPlanner()) {
-            if (wantToParseSqlFromSqlCache) {
-                cachedStmts = parseFromSqlCache(originStmt);
-                Optional<SqlCacheContext> sqlCacheContext = ConnectContext.get()
-                        .getStatementContext().getSqlCacheContext();
-                if (sqlCacheContext.isPresent()) {
-                    cacheKeyType = sqlCacheContext.get().getCacheKeyType();
-                }
-                if (cachedStmts != null) {
-                    stmts = cachedStmts;
-                }
-            }
-
-            if (cachedStmts == null) {
-                try {
-                    stmts = new NereidsParser().parseSQL(convertedStmt, sessionVariable);
-                } catch (NotSupportedException e) {
-                    // Parse sql failed, audit it and return
-                    handleQueryException(e, convertedStmt, null, null);
-                    return;
-                } catch (ParseException e) {
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("Nereids parse sql failed. Reason: {}. Statement: \"{}\".",
-                                e.getMessage(), convertedStmt);
-                    }
-                    // ATTN: Do not set nereidsParseException in this case.
-                    // Because ParseException means the sql is not supported by Nereids.
-                    // It should be parsed by old parser, so not setting nereidsParseException to avoid
-                    // suppressing the exception thrown by old parser.
-                    nereidsParseException = e;
-                } catch (Exception e) {
-                    // TODO: We should catch all exception here until we support all query syntax.
-                    if (LOG.isDebugEnabled()) {
-                        LOG.debug("Nereids parse sql failed with other exception. Reason: {}. Statement: \"{}\".",
-                                e.getMessage(), convertedStmt);
-                    }
-                    nereidsSyntaxException = e;
-                }
+        if (wantToParseSqlFromSqlCache) {
+            cachedStmts = parseFromSqlCache(originStmt);
+            Optional<SqlCacheContext> sqlCacheContext = ConnectContext.get()
+                    .getStatementContext().getSqlCacheContext();
+            if (sqlCacheContext.isPresent()) {
+                cacheKeyType = sqlCacheContext.get().getCacheKeyType();
             }
-
-            if (stmts == null) {
-                String errMsg;
-                Throwable exception = null;
-                if (nereidsParseException != null) {
-                    errMsg = nereidsParseException.getMessage();
-                    exception = nereidsParseException;
-                } else if (nereidsSyntaxException != null) {
-                    errMsg = nereidsSyntaxException.getMessage();
-                    exception = nereidsSyntaxException;
-                } else {
-                    errMsg = "Nereids parse statements failed. " + originStmt;
-                }
-                if (exception == null) {
-                    exception = new AnalysisException(errMsg);
-                } else {
-                    exception = new AnalysisException(errMsg, exception);
-                }
-                handleQueryException(exception, originStmt, null, null);
-                return;
+            if (cachedStmts != null) {
+                stmts = cachedStmts;
             }
         }
 
-        // stmts == null when Nereids cannot planner this query or Nereids is disabled.
         if (stmts == null) {
-            if (mysqlCommand == MysqlCommand.COM_STMT_PREPARE) {
-                // avoid fall back to legacy planner
-                ctx.getState().setError(ErrorCode.ERR_UNSUPPORTED_PS, "Not supported such prepared statement");
-                ctx.getState().setErrType(QueryState.ErrType.OTHER_ERR);
-                return;
-            }
             try {
-                stmts = parse(convertedStmt);
-            } catch (Throwable throwable) {
+                stmts = new NereidsParser().parseSQL(convertedStmt, sessionVariable);
+            } catch (NotSupportedException e) {
                 // Parse sql failed, audit it and return
-                handleQueryException(throwable, convertedStmt, null, null);
+                handleQueryException(e, convertedStmt, null, null);
+                return;
+            } catch (Exception e) {
+                if (LOG.isDebugEnabled()) {
+                    LOG.debug("Nereids parse sql failed. Reason: {}. Statement: \"{}\".",
+                            e.getMessage(), convertedStmt, e);
+                }
+                Throwable exception = new AnalysisException(e.getMessage(), e);
+                handleQueryException(exception, originStmt, null, null);
                 return;
             }
         }