[trinodb#37] https://git.bilibili.co/datacenter/bili-trino/-/issues/37,…

… calculate some agg function from metadata

core/trino-main/src/main/java/io/trino/FeaturesConfig.java

@@ -101,6 +101,7 @@ public class FeaturesConfig
 
     private boolean hideInaccessibleColumns;
     private boolean forceSpillingJoin;
+    private boolean optimizeQueryWithMetadata = true;
 
     public enum DataIntegrityVerification
     {
@@ -412,6 +413,18 @@ public FeaturesConfig setLateMaterializationEnabled(boolean lateMaterializationE
         return this;
     }
 
+    public boolean isOptimizeQueryWithMetadata()
+    {
+        return optimizeQueryWithMetadata;
+    }
+
+    @Config("optimizer.optimize-query-with-metadata")
+    public FeaturesConfig setOptimizeQueryWithMetadata(boolean optimizeQueryWithMetadata)
+    {
+        this.optimizeQueryWithMetadata = optimizeQueryWithMetadata;
+        return this;
+    }
+
     public boolean isLegacyCatalogRoles()
     {
         return legacyCatalogRoles;

core/trino-main/src/main/java/io/trino/SystemSessionProperties.java

@@ -157,6 +157,7 @@ public final class SystemSessionProperties
     public static final String RETRY_MAX_DELAY = "retry_max_delay";
     public static final String RETRY_DELAY_SCALE_FACTOR = "retry_delay_scale_factor";
     public static final String HIDE_INACCESSIBLE_COLUMNS = "hide_inaccessible_columns";
+
     public static final String FAULT_TOLERANT_EXECUTION_TARGET_TASK_INPUT_SIZE = "fault_tolerant_execution_target_task_input_size";
     public static final String FAULT_TOLERANT_EXECUTION_MIN_TASK_SPLIT_COUNT = "fault_tolerant_execution_min_task_split_count";
     public static final String FAULT_TOLERANT_EXECUTION_TARGET_TASK_SPLIT_COUNT = "fault_tolerant_execution_target_task_split_count";
@@ -173,7 +174,7 @@ public final class SystemSessionProperties
     public static final String JOIN_PARTITIONED_BUILD_MIN_ROW_COUNT = "join_partitioned_build_min_row_count";
     public static final String USE_EXACT_PARTITIONING = "use_exact_partitioning";
     public static final String FORCE_SPILLING_JOIN = "force_spilling_join";
-
+    public static final String QUERY_OPTIMIZE_WITH_METADATA_ENABLED = "query_optimize_with_metadata_enabled";
     private final List<PropertyMetadata<?>> sessionProperties;
 
     public SystemSessionProperties()
@@ -853,6 +854,11 @@ public SystemSessionProperties(
                         FORCE_SPILLING_JOIN,
                         "Force the usage of spliing join operator in favor of the non-spilling one, even if spill is not enabled",
                         featuresConfig.isForceSpillingJoin(),
+                        false),
+                booleanProperty(
+                        QUERY_OPTIMIZE_WITH_METADATA_ENABLED,
+                        "Optimizing queries by leveraging metadata",
+                        featuresConfig.isOptimizeQueryWithMetadata(),
                         false));
     }
 
@@ -1526,4 +1532,9 @@ public static boolean isForceSpillingOperator(Session session)
     {
         return session.getSystemProperty(FORCE_SPILLING_JOIN, Boolean.class);
     }
+
+    public static boolean isQueryOptimizeWithMetadataEnabled(Session session)
+    {
+        return session.getSystemProperty(QUERY_OPTIMIZE_WITH_METADATA_ENABLED, Boolean.class);
+    }
 }

core/trino-main/src/main/java/io/trino/cost/SymbolStatsEstimate.java

@@ -39,6 +39,7 @@ public class SymbolStatsEstimate
     private final double nullsFraction;
     private final double averageRowSize;
     private final double distinctValuesCount;
+    private final double accurateNullsCount;
 
     public static SymbolStatsEstimate unknown()
     {
@@ -57,6 +58,17 @@ public SymbolStatsEstimate(
             @JsonProperty("nullsFraction") double nullsFraction,
             @JsonProperty("averageRowSize") double averageRowSize,
             @JsonProperty("distinctValuesCount") double distinctValuesCount)
+    {
+        this(lowValue, highValue, nullsFraction, averageRowSize, distinctValuesCount, NaN);
+    }
+
+    public SymbolStatsEstimate(
+            double lowValue,
+            double highValue,
+            double nullsFraction,
+            double averageRowSize,
+            double distinctValuesCount,
+            double accurateNullsCount)
     {
         checkArgument(
                 lowValue <= highValue || (isNaN(lowValue) && isNaN(highValue)),
@@ -79,6 +91,7 @@ public SymbolStatsEstimate(
         checkArgument(distinctValuesCount >= 0 || isNaN(distinctValuesCount), "Distinct values count should be non-negative, got: %s", distinctValuesCount);
         // TODO normalize distinctValuesCount for an empty range (or validate it is already normalized)
         this.distinctValuesCount = distinctValuesCount;
+        this.accurateNullsCount = accurateNullsCount;
     }
 
     @JsonProperty
@@ -131,6 +144,11 @@ public SymbolStatsEstimate mapDistinctValuesCount(Function<Double, Double> mappi
         return buildFrom(this).setDistinctValuesCount(mappingFunction.apply(distinctValuesCount)).build();
     }
 
+    public double getAccurateNullsCount()
+    {
+        return accurateNullsCount;
+    }
+
     public boolean isUnknown()
     {
         return this.equals(UNKNOWN);
@@ -199,6 +217,7 @@ public static final class Builder
         private double nullsFraction = NaN;
         private double averageRowSize = NaN;
         private double distinctValuesCount = NaN;
+        private double accurateNullsCount = NaN;
 
         public Builder setStatisticsRange(StatisticRange range)
         {
@@ -237,9 +256,15 @@ public Builder setDistinctValuesCount(double distinctValuesCount)
             return this;
         }
 
+        public Builder setAccurateNullsCount(double accurateNullsCount)
+        {
+            this.accurateNullsCount = accurateNullsCount;
+            return this;
+        }
+
         public SymbolStatsEstimate build()
         {
-            return new SymbolStatsEstimate(lowValue, highValue, nullsFraction, averageRowSize, distinctValuesCount);
+            return new SymbolStatsEstimate(lowValue, highValue, nullsFraction, averageRowSize, distinctValuesCount, accurateNullsCount);
         }
     }
 }

core/trino-main/src/main/java/io/trino/cost/TableScanStatsRule.java

@@ -103,6 +103,7 @@ else if (type instanceof FixedWidthType) {
             result.setLowValue(range.getMin());
             result.setHighValue(range.getMax());
         });
+        result.setAccurateNullsCount(columnStatistics.getAccurateNullsCount().getValue());
         return result.build();
     }
 }

core/trino-main/src/main/java/io/trino/sql/analyzer/QueryExplainer.java

@@ -151,10 +151,19 @@ public String getJsonPlan(Session session, Statement statement, Type planType, L
     }
 
     public Plan getLogicalPlan(Session session, Statement statement, List<Expression> parameters, WarningCollector warningCollector)
+    {
+        return getLogicalPlan(session, statement, OPTIMIZED_AND_VALIDATED, parameters, warningCollector, true);
+    }
+
+    public Plan getLogicalPlan(Session session, Statement statement, LogicalPlanner.Stage stage, List<Expression> parameters, WarningCollector warningCollector, boolean collectPlanStatistics)
     {
         // analyze statement
         Analysis analysis = analyze(session, statement, parameters, warningCollector);
+        return getLogicalPlan(session, analysis, stage, warningCollector, collectPlanStatistics);
+    }
 
+    public Plan getLogicalPlan(Session session, Analysis analysis, LogicalPlanner.Stage stage, WarningCollector warningCollector, boolean collectPlanStatistics)
+    {
         PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
 
         // plan statement

core/trino-main/src/main/java/io/trino/sql/planner/PlanOptimizers.java

@@ -255,6 +255,7 @@
 import io.trino.sql.planner.optimizations.PlanOptimizer;
 import io.trino.sql.planner.optimizations.PredicatePushDown;
 import io.trino.sql.planner.optimizations.ReplicateJoinAndSemiJoinInDelete;
+import io.trino.sql.planner.optimizations.RewriteOnlyMinMaxCountPlan;
 import io.trino.sql.planner.optimizations.StatsRecordingPlanOptimizer;
 import io.trino.sql.planner.optimizations.TransformQuantifiedComparisonApplyToCorrelatedJoin;
 import io.trino.sql.planner.optimizations.UnaliasSymbolReferences;
@@ -968,6 +969,19 @@ public PlanOptimizers(
                         new RemoveRedundantIdentityProjections())));
         // DO NOT add optimizers that change the plan shape (computations) after this point
 
+        // The following two optimizers should be applied after added intermediate aggregations, and
+        // SHOULD NOT be moved down.
+        builder.add(new RewriteOnlyMinMaxCountPlan(plannerContext, typeAnalyzer));
+        builder.add(new IterativeOptimizer(
+                plannerContext,
+                ruleStats,
+                statsCalculator,
+                costCalculator,
+                ImmutableSet.<Rule<?>>builder()
+                        .addAll(new SimplifyExpressions(plannerContext, typeAnalyzer).rules())
+                        .addAll(new UnwrapCastInComparison(plannerContext, typeAnalyzer).rules())
+                        .build()));
+
         // Remove any remaining sugar
         builder.add(new IterativeOptimizer(
                 plannerContext,