From b11a2417d01fc8dad80bccf3130adc6466734773 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Wed, 3 Nov 2021 21:47:01 +0200 Subject: [PATCH] Add connector metrics to EXPLAIN ANALYZE verbose output It should allow us to get quick feedback on a specific query (without the need to "dig" into the QueryInfo JSON). --- .../HashCollisionPlanNodeStats.java | 11 ++++++--- .../planner/planprinter/PlanNodeStats.java | 23 +++++++++++++++++-- .../planprinter/PlanNodeStatsSummarizer.java | 16 ++++++++++--- .../sql/planner/planprinter/TextRenderer.java | 15 ++++++++++++ .../planprinter/WindowPlanNodeStats.java | 11 ++++++--- .../memory/TestMemoryConnectorTest.java | 16 +++++++++++++ 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/HashCollisionPlanNodeStats.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/HashCollisionPlanNodeStats.java index fcafde5ecac1..9de5b5e926db 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/HashCollisionPlanNodeStats.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/HashCollisionPlanNodeStats.java @@ -15,6 +15,7 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.trino.spi.metrics.Metrics; import io.trino.sql.planner.plan.PlanNodeId; import java.util.Map; @@ -39,7 +40,8 @@ public HashCollisionPlanNodeStats( DataSize planNodeOutputDataSize, DataSize planNodeSpilledDataSize, Map operatorInputStats, - Map operatorHashCollisionsStats) + Map operatorHashCollisionsStats, + Metrics metrics) { super( planNodeId, @@ -50,7 +52,9 @@ public HashCollisionPlanNodeStats( planNodeOutputPositions, planNodeOutputDataSize, planNodeSpilledDataSize, - operatorInputStats); + operatorInputStats, + metrics, + Metrics.EMPTY); this.operatorHashCollisionsStats = requireNonNull(operatorHashCollisionsStats, "operatorHashCollisionsStats is null"); } @@ -104,6 +108,7 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeOutputDataSize(), merged.getPlanNodeSpilledDataSize(), merged.operatorInputStats, - operatorHashCollisionsStats); + operatorHashCollisionsStats, + merged.getMetrics()); } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStats.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStats.java index 38770f43aa49..0e535801c809 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStats.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStats.java @@ -16,6 +16,7 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.spi.Mergeable; +import io.trino.spi.metrics.Metrics; import io.trino.sql.planner.plan.PlanNodeId; import java.util.Map; @@ -42,6 +43,8 @@ public class PlanNodeStats private final long planNodeOutputPositions; private final DataSize planNodeOutputDataSize; private final DataSize planNodeSpilledDataSize; + private final Metrics metrics; + private final Metrics connectorMetrics; protected final Map operatorInputStats; @@ -54,7 +57,9 @@ public class PlanNodeStats long planNodeOutputPositions, DataSize planNodeOutputDataSize, DataSize planNodeSpilledDataSize, - Map operatorInputStats) + Map operatorInputStats, + Metrics metrics, + Metrics connectorMetrics) { this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); @@ -67,6 +72,8 @@ public class PlanNodeStats this.planNodeSpilledDataSize = requireNonNull(planNodeSpilledDataSize, "planNodeSpilledDataSize is null"); this.operatorInputStats = requireNonNull(operatorInputStats, "operatorInputStats is null"); + this.metrics = requireNonNull(metrics, "metrics is null"); + this.connectorMetrics = requireNonNull(connectorMetrics, "connectorMetrics is null"); } private static double computedStdDev(double sumSquared, double sum, long n) @@ -122,6 +129,16 @@ public DataSize getPlanNodeSpilledDataSize() return planNodeSpilledDataSize; } + public Metrics getMetrics() + { + return metrics; + } + + public Metrics getConnectorMetrics() + { + return connectorMetrics; + } + public Map getOperatorInputPositionsAverages() { return operatorInputStats.entrySet().stream() @@ -160,6 +177,8 @@ public PlanNodeStats mergeWith(PlanNodeStats other) planNodeInputPositions, planNodeInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, succinctBytes(this.planNodeSpilledDataSize.toBytes() + other.planNodeSpilledDataSize.toBytes()), - operatorInputStats); + operatorInputStats, + this.metrics.mergeWith(other.metrics), + this.connectorMetrics.mergeWith(other.connectorMetrics)); } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStatsSummarizer.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStatsSummarizer.java index 1710da36dddb..ce2496ad49c3 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStatsSummarizer.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanNodeStatsSummarizer.java @@ -22,6 +22,7 @@ import io.trino.operator.PipelineStats; import io.trino.operator.TaskStats; import io.trino.operator.WindowInfo; +import io.trino.spi.metrics.Metrics; import io.trino.sql.planner.plan.PlanNodeId; import java.util.ArrayList; @@ -82,6 +83,8 @@ private static List getPlanNodeStats(TaskStats taskStats) Map> operatorInputStats = new HashMap<>(); Map> operatorHashCollisionsStats = new HashMap<>(); Map windowNodeStats = new HashMap<>(); + Map metrics = new HashMap<>(); + Map connectorMetrics = new HashMap<>(); for (PipelineStats pipelineStats : taskStats.getPipelines()) { // Due to eventual consistently collected stats, these could be empty @@ -121,6 +124,9 @@ private static List getPlanNodeStats(TaskStats taskStats) operatorStats.getSumSquaredInputPositions())), (map1, map2) -> mergeMaps(map1, map2, OperatorInputStats::merge)); + metrics.merge(planNodeId, operatorStats.getMetrics(), Metrics::mergeWith); + connectorMetrics.merge(planNodeId, operatorStats.getConnectorMetrics(), Metrics::mergeWith); + planNodeInputPositions.merge(planNodeId, operatorStats.getInputPositions(), Long::sum); planNodeInputBytes.merge(planNodeId, operatorStats.getInputDataSize().toBytes(), Long::sum); planNodeSpilledDataSize.merge(planNodeId, operatorStats.getSpilledDataSize().toBytes(), Long::sum); @@ -195,7 +201,8 @@ private static List getPlanNodeStats(TaskStats taskStats) succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), operatorInputStats.get(planNodeId), - operatorHashCollisionsStats.get(planNodeId)); + operatorHashCollisionsStats.get(planNodeId), + metrics.get(planNodeId)); } else if (windowNodeStats.containsKey(planNodeId)) { nodeStats = new WindowPlanNodeStats( @@ -208,7 +215,8 @@ else if (windowNodeStats.containsKey(planNodeId)) { succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), operatorInputStats.get(planNodeId), - windowNodeStats.get(planNodeId)); + windowNodeStats.get(planNodeId), + metrics.get(planNodeId)); } else { nodeStats = new PlanNodeStats( @@ -220,7 +228,9 @@ else if (windowNodeStats.containsKey(planNodeId)) { outputPositions, succinctBytes(planNodeOutputBytes.getOrDefault(planNodeId, 0L)), succinctBytes(planNodeSpilledDataSize.get(planNodeId)), - operatorInputStats.get(planNodeId)); + operatorInputStats.get(planNodeId), + metrics.get(planNodeId), + connectorMetrics.get(planNodeId)); } stats.add(nodeStats); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TextRenderer.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TextRenderer.java index a96b67a6f00f..1800b6804d28 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TextRenderer.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/TextRenderer.java @@ -18,6 +18,8 @@ import io.trino.cost.PlanCostEstimate; import io.trino.cost.PlanNodeStatsAndCostSummary; import io.trino.cost.PlanNodeStatsEstimate; +import io.trino.spi.metrics.Metric; +import io.trino.spi.metrics.Metrics; import io.trino.sql.planner.Symbol; import io.trino.sql.planner.planprinter.NodeRepresentation.TypedSymbol; @@ -27,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; @@ -135,6 +138,8 @@ private String printStats(PlanRepresentation plan, NodeRepresentation node) } output.append("\n"); + printMetrics(output, "connector metrics:", nodeStats.getConnectorMetrics()); + printMetrics(output, "metrics:", nodeStats.getMetrics()); printDistributions(output, nodeStats); printCollisions(output, nodeStats); @@ -145,6 +150,16 @@ private String printStats(PlanRepresentation plan, NodeRepresentation node) return output.toString(); } + private void printMetrics(StringBuilder output, String label, Metrics metrics) + { + if (!verbose || metrics.getMetrics().isEmpty()) { + return; + } + output.append(label).append("\n"); + Map> sortedMap = new TreeMap<>(metrics.getMetrics()); + sortedMap.forEach((name, metric) -> output.append(format(" '%s' = %s\n", name, metric))); + } + private void printDistributions(StringBuilder output, PlanNodeStats stats) { Map inputAverages = stats.getOperatorInputPositionsAverages(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/WindowPlanNodeStats.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/WindowPlanNodeStats.java index d77500301a1e..08169df78c93 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/WindowPlanNodeStats.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/WindowPlanNodeStats.java @@ -15,6 +15,7 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.trino.spi.metrics.Metrics; import io.trino.sql.planner.plan.PlanNodeId; import java.util.Map; @@ -34,7 +35,8 @@ public WindowPlanNodeStats( DataSize planNodeOutputDataSize, DataSize planNodeSpilledDataSize, Map operatorInputStats, - WindowOperatorStats windowOperatorStats) + WindowOperatorStats windowOperatorStats, + Metrics metrics) { super( planNodeId, @@ -45,7 +47,9 @@ public WindowPlanNodeStats( planNodeOutputPositions, planNodeOutputDataSize, planNodeSpilledDataSize, - operatorInputStats); + operatorInputStats, + metrics, + Metrics.EMPTY); this.windowOperatorStats = windowOperatorStats; } @@ -69,6 +73,7 @@ public PlanNodeStats mergeWith(PlanNodeStats other) merged.getPlanNodeOutputDataSize(), merged.getPlanNodeSpilledDataSize(), merged.operatorInputStats, - windowOperatorStats); + windowOperatorStats, + merged.getMetrics()); } } diff --git a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java index 52ff12ab7c55..3d0a20e6cab3 100644 --- a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java +++ b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java @@ -164,6 +164,22 @@ public void testCustomMetricsScanOnly() assertThat(((Count) metrics.getMetrics().get("finished")).getTotal()).isGreaterThan(0); } + @Test + public void testExplainCustomMetricsScanOnly() + { + assertExplainAnalyze( + "EXPLAIN ANALYZE VERBOSE SELECT partkey FROM part", + "'rows' = LongCount\\{total=2000}"); + } + + @Test + public void testExplainCustomMetricsScanFilter() + { + assertExplainAnalyze( + "EXPLAIN ANALYZE VERBOSE SELECT partkey FROM part WHERE partkey % 1000 > 0", + "'rows' = LongCount\\{total=2000}"); + } + private Metrics collectCustomMetrics(String sql) { DistributedQueryRunner runner = (DistributedQueryRunner) getQueryRunner();