From 1e1cefd42028e844e26b8b505f7b6fda0818c3f1 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 2 Sep 2024 18:50:37 +0300 Subject: [PATCH] Datadog demo: update sql exporter metrics --- integrations/datadog/config.yaml | 30 ++++++++++------------------- integrations/datadog/dashboard.json | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/integrations/datadog/config.yaml b/integrations/datadog/config.yaml index af69a56..77a7e40 100644 --- a/integrations/datadog/config.yaml +++ b/integrations/datadog/config.yaml @@ -129,12 +129,11 @@ jobs: R.name as replica_name, R.id as replica_id FROM mz_catalog.mz_cluster_replicas as R - JOIN mz_internal.mz_statement_execution_history_redacted as MSEH + JOIN mz_internal.mz_recent_activity_log_redacted as MSEH ON R.cluster_id = MSEH.cluster_id - JOIN mz_internal.mz_prepared_statement_history_redacted as MPSH - ON MPSH.id = MSEH.prepared_statement_id + WHERE MSEH.began_at > now() - interval '1 min' - AND MPSH.prepared_at > now() - interval '1 min' + AND MSEH.prepared_at > now() - interval '1 min' AND SUBSTRING(R.cluster_id, 1, 1) != 's' GROUP BY MSEH.cluster_name, MSEH.cluster_id, R.name, R.id; - name: "object_freshness" @@ -184,20 +183,11 @@ jobs: values: - "size_bytes" query: | - WITH last_measure AS ( - SELECT - object_id, - MAX(collection_timestamp) timestamp - FROM mz_storage_usage - GROUP BY object_id - ) SELECT - DISTINCT(id) as object_id, - name as object_name, - type as object_type, - size_bytes - FROM mz_storage_usage U, mz_objects O, last_measure L - WHERE O.id = U.object_id - AND O.id = L.object_id - AND U.collection_timestamp = L.timestamp - AND id NOT LIKE 's%'; + DISTINCT(O.id) as object_id, + O.name as object_name, + O.type as object_type, + U.size_bytes + FROM mz_recent_storage_usage U + JOIN mz_objects O ON O.id = U.object_id + WHERE O.id NOT LIKE 's%'; diff --git a/integrations/datadog/dashboard.json b/integrations/datadog/dashboard.json index d4ee04a..1b0c8a2 100644 --- a/integrations/datadog/dashboard.json +++ b/integrations/datadog/dashboard.json @@ -1 +1 @@ -{"title":"Materialize Dashboard","description":null,"widgets":[{"id":3200457131349728,"definition":{"title":"Resources Overview","background_color":"purple","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":291620658209522,"definition":{"title":"CPU Usage per Computing Cluster","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:compute,col:cpu_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","min":"0","max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":3}},{"id":4676270853170894,"definition":{"title":"Memory Usage per Computing Cluster","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:compute,col:memory_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":3}},{"id":7228998847198550,"definition":{"title":"Healthy Computing Clusters","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_cluster_replica_usage{cluster_type:compute,col:replica_status} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":3,"width":12,"height":3}},{"id":841920155325309,"definition":{"title":"Query Latency","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"style":{"palette":"classic","palette_index":2},"alias":"Avg latency","formula":"autosmooth(query1)"},{"style":{"palette":"classic","palette_index":4},"alias":"Max latency","formula":"autosmooth(query2)"},{"style":{"palette":"classic","palette_index":0},"alias":"Min latency","formula":"autosmooth(query3)"}],"queries":[{"data_source":"metrics","name":"query1","query":"avg:materialize.sql_query_latency{col:avg_latency}"},{"data_source":"metrics","name":"query2","query":"avg:materialize.sql_query_latency{col:max_latency}"},{"data_source":"metrics","name":"query3","query":"avg:materialize.sql_query_latency{col:min_latency}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":6,"width":12,"height":3}},{"id":706576452190109,"definition":{"title":"Approximate Lag per Object ","title_size":"16","title_align":"left","type":"toplist","requests":[{"queries":[{"data_source":"metrics","name":"query1","query":"avg:materialize.sql_object_freshness{col:approx_lag_ms} by {object_name}","aggregator":"avg"}],"formulas":[{"formula":"query1","limit":{"count":25,"order":"desc"}}],"response_format":"scalar","conditional_formats":[{"comparator":">","value":5000,"palette":"white_on_yellow"},{"comparator":">","value":0,"palette":"white_on_green"},{"comparator":">","value":60000,"palette":"white_on_red"}]}],"style":{"display":{"type":"stacked","legend":"automatic"}}},"layout":{"x":0,"y":9,"width":6,"height":4}},{"id":2862988017710242,"definition":{"title":"Storage Usage per Object","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_storage_usage{!object_type:source} by {object_type,object_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"datadog16"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":6,"y":9,"width":6,"height":4}}]},"layout":{"x":0,"y":0,"width":12,"height":14}},{"id":6304613009696976,"definition":{"title":"Credit consumption","background_color":"blue","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":4216505741119604,"definition":{"type":"note","content":"This section displays the current credit usage from clusters. Its purpose is to provide a way to visualize and monitor unexpected credit consumption, not showing an accurate amount of credit usage.","background_color":"white","font_size":"14","text_align":"center","vertical_align":"top","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":true},"layout":{"x":0,"y":0,"width":3,"height":3}},{"id":5851028022531786,"definition":{"title":"Consumption rate per hour","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_cluster_replica_usage{col:credits_per_hour}","aggregator":"last"}],"formulas":[{"formula":"query1"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":0,"width":9,"height":3}},{"id":6288227030612956,"definition":{"title":"Per Computing Cluster","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_cluster_replica_usage{!replica_name:linked,col:credits_per_hour} by {cluster_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"semantic"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":0,"y":3,"width":6,"height":4}},{"id":8866266265616072,"definition":{"title":"Per Source and Sink","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_cluster_replica_usage{replica_name:linked,col:credits_per_hour} by {cluster_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"semantic"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":6,"y":3,"width":6,"height":4}}]},"layout":{"x":0,"y":14,"width":12,"height":8}},{"id":4466675117877256,"definition":{"title":"Sources","background_color":"yellow","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":3434985367353964,"definition":{"title":"CPU Usage per Source","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:source,col:cpu_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":4}},{"id":7069604621674458,"definition":{"title":"Memory Usage per Source","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:source,col:memory_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":4}},{"id":3744956911720100,"definition":{"title":"Sources Starting","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:starting,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"custom_text"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":0,"y":4,"width":3,"height":2}},{"id":8446147164226954,"definition":{"title":"Sources Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:running,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_green"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":4,"width":3,"height":2}},{"id":80765431090320,"definition":{"title":"Sources Stalled","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:stalled,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_yellow"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":false},"type":"area"}},"layout":{"x":6,"y":4,"width":3,"height":2}},{"id":8633003794486586,"definition":{"title":"Sources Failing","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:failling,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_red"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":false},"type":"area"}},"layout":{"x":9,"y":4,"width":3,"height":2}},{"id":2272156155006558,"definition":{"title":"Storage Usage ","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_storage_usage{object_type:source} by {object_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"datadog16"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":0,"y":6,"width":12,"height":4}},{"id":4229913734149808,"definition":{"type":"note","content":"After committing the initial snapshot, the source and the downstream objects are ready to start producing output.\n\nFor more context: https://materialize.com/docs/manage/troubleshooting/#has-my-source-ingested-its-initial-snapshot","background_color":"white","font_size":"14","text_align":"left","vertical_align":"top","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":true},"layout":{"x":0,"y":10,"width":3,"height":2}},{"id":6217598342376058,"definition":{"title":"Snapshot committed","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"response_format":"timeseries","queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_source_statistics{col:snapshot_committed} by {source_name}"}],"formulas":[{"formula":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":3,"y":10,"width":9,"height":2}},{"id":5590617379026023,"definition":{"title":"Bytes Received","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_source_statistics{col:bytes_received} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":12,"width":6,"height":4}},{"id":1317023415732612,"definition":{"title":"Messages Received","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_source_statistics{col:messages_received} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":12,"width":6,"height":4}},{"id":7164318732405162,"definition":{"title":"Envelope State Size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{col:envelope_state_bytes} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":16,"width":6,"height":3}},{"id":3902635199640766,"definition":{"title":"Envelope State Count","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{col:envelope_state_count} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":16,"width":6,"height":3}}]},"layout":{"x":0,"y":22,"width":12,"height":20,"is_column_break":true}},{"id":8461660367120526,"definition":{"title":"Sinks","background_color":"gray","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":8544966924804406,"definition":{"title":"CPU Usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:sink*} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":4}},{"id":2070922288633090,"definition":{"title":"Memory Usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:sink*} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":4}},{"id":6346996337026122,"definition":{"title":"Sinks Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:starting,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"custom_text"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":0,"y":4,"width":3,"height":2}},{"id":1788220607919434,"definition":{"title":"Sinks Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:running,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_green"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":4,"width":3,"height":2}},{"id":405464898842380,"definition":{"title":"Sinks Stalled","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:stalled,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_yellow"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":6,"y":4,"width":3,"height":2}},{"id":235027738347796,"definition":{"title":"Sinks Failing","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:failling,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_red"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":9,"y":4,"width":3,"height":2}},{"id":3605514603037456,"definition":{"title":"Bytes Commited","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_sink_statistics{col:bytes_committed}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":6,"width":6,"height":4}},{"id":2634515619757848,"definition":{"title":"Messages Commited","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_sink_statistics{col:messages_committed}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":6,"width":6,"height":4}}]},"layout":{"x":0,"y":42,"width":12,"height":11}}],"template_variables":[],"layout_type":"ordered","notify_list":[],"reflow_type":"fixed"} \ No newline at end of file +{"title":"Materialize Dashboard","description":null,"widgets":[{"id":3200457131349728,"definition":{"title":"Resources Overview","background_color":"purple","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":291620658209522,"definition":{"title":"CPU Usage per Computing Cluster","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:compute,col:cpu_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"scale":"linear","min":"0","max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":3}},{"id":4676270853170894,"definition":{"title":"Memory Usage per Computing Cluster","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:compute,col:memory_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":3}},{"id":7228998847198550,"definition":{"title":"Healthy Computing Clusters","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_cluster_replica_usage{cluster_type:compute,col:replica_status} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":0,"y":3,"width":12,"height":3}},{"id":841920155325309,"definition":{"title":"Query Latency","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"style":{"palette":"classic","palette_index":2},"alias":"Avg latency","formula":"autosmooth(query1)"},{"style":{"palette":"classic","palette_index":4},"alias":"Max latency","formula":"autosmooth(query2)"},{"style":{"palette":"classic","palette_index":0},"alias":"Min latency","formula":"autosmooth(query3)"}],"queries":[{"data_source":"metrics","name":"query1","query":"avg:materialize.sql_query_latency{col:avg_latency}"},{"data_source":"metrics","name":"query2","query":"avg:materialize.sql_query_latency{col:max_latency}"},{"data_source":"metrics","name":"query3","query":"avg:materialize.sql_query_latency{col:min_latency}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":6,"width":12,"height":3}},{"id":706576452190109,"definition":{"title":"Approximate Lag per Object ","title_size":"16","title_align":"left","type":"toplist","requests":[{"queries":[{"data_source":"metrics","name":"query1","query":"avg:materialize.sql_object_freshness{col:approx_lag_ms} by {object_name}","aggregator":"avg"}],"response_format":"scalar","conditional_formats":[{"comparator":">","value":5000,"palette":"white_on_yellow"},{"comparator":">","value":0,"palette":"white_on_green"},{"comparator":">","value":60000,"palette":"white_on_red"}],"formulas":[{"formula":"query1"}],"sort":{"count":25,"order_by":[{"type":"formula","index":0,"order":"desc"}]}}],"style":{"display":{"type":"stacked","legend":"automatic"}}},"layout":{"x":0,"y":9,"width":6,"height":4}},{"id":2862988017710242,"definition":{"title":"Storage Usage per Object","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_storage_usage{!object_type:source} by {object_type,object_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"datadog16"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":6,"y":9,"width":6,"height":4}}]},"layout":{"x":0,"y":0,"width":12,"height":14}},{"id":6304613009696976,"definition":{"title":"Credit consumption","background_color":"blue","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":4216505741119604,"definition":{"type":"note","content":"This section displays the current credit usage from clusters. Its purpose is to provide a way to visualize and monitor unexpected credit consumption, not showing an accurate amount of credit usage.","background_color":"white","font_size":"14","text_align":"center","vertical_align":"top","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":true},"layout":{"x":0,"y":0,"width":3,"height":3}},{"id":5851028022531786,"definition":{"title":"Consumption rate per hour","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_cluster_replica_usage{col:credits_per_hour}","aggregator":"last"}],"formulas":[{"formula":"query1"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":0,"width":9,"height":3}},{"id":6288227030612956,"definition":{"title":"Per Computing Cluster","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_cluster_replica_usage{!replica_name:linked,col:credits_per_hour} by {cluster_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"semantic"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":0,"y":3,"width":6,"height":4}},{"id":8866266265616072,"definition":{"title":"Per Source and Sink","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1","limit":{"order":"desc"}}],"queries":[{"query":"sum:materialize.sql_cluster_replica_usage{replica_name:linked,col:credits_per_hour} by {cluster_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"semantic"}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":6,"y":3,"width":6,"height":4}}]},"layout":{"x":0,"y":14,"width":12,"height":8}},{"id":4466675117877256,"definition":{"title":"Sources","background_color":"yellow","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":3434985367353964,"definition":{"title":"CPU Usage per Source","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:source,col:cpu_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":4}},{"id":7069604621674458,"definition":{"title":"Memory Usage per Source","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"autosmooth(query1)"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:source,col:memory_percent} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":4}},{"id":3744956911720100,"definition":{"title":"Sources Starting","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:starting,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"custom_text"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":0,"y":4,"width":3,"height":2}},{"id":8446147164226954,"definition":{"title":"Sources Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:running,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_green"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":4,"width":3,"height":2}},{"id":80765431090320,"definition":{"title":"Sources Stalled","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:stalled,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_yellow"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":false},"type":"area"}},"layout":{"x":6,"y":4,"width":3,"height":2}},{"id":8633003794486586,"definition":{"title":"Sources Failing","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{source_status_label:failling,col:source_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_red"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":false},"type":"area"}},"layout":{"x":9,"y":4,"width":3,"height":2}},{"id":2272156155006558,"definition":{"title":"Storage Usage ","title_size":"16","title_align":"left","requests":[{"response_format":"scalar","formulas":[{"formula":"query1"}],"queries":[{"query":"sum:materialize.sql_storage_usage{*} by {object_name}","data_source":"metrics","name":"query1","aggregator":"last"}],"style":{"palette":"datadog16"},"sort":{"count":500,"order_by":[{"type":"formula","index":0,"order":"desc"}]}}],"type":"sunburst","legend":{"type":"automatic"}},"layout":{"x":0,"y":6,"width":12,"height":4}},{"id":4229913734149808,"definition":{"type":"note","content":"After committing the initial snapshot, the source and the downstream objects are ready to start producing output.\n\nFor more context: https://materialize.com/docs/manage/troubleshooting/#has-my-source-ingested-its-initial-snapshot","background_color":"white","font_size":"14","text_align":"left","vertical_align":"top","show_tick":false,"tick_pos":"50%","tick_edge":"left","has_padding":true},"layout":{"x":0,"y":10,"width":3,"height":2}},{"id":6217598342376058,"definition":{"title":"Snapshot committed","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"response_format":"timeseries","queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_source_statistics{col:snapshot_committed} by {source_name}"}],"formulas":[{"formula":"query1"}],"style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"bars"}]},"layout":{"x":3,"y":10,"width":9,"height":2}},{"id":5590617379026023,"definition":{"title":"Bytes Received","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_source_statistics{col:bytes_received} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":12,"width":6,"height":4}},{"id":1317023415732612,"definition":{"title":"Messages Received","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_source_statistics{col:messages_received} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":12,"width":6,"height":4}},{"id":7164318732405162,"definition":{"title":"Envelope State Size","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{col:envelope_state_bytes} by {source_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":16,"width":6,"height":3}},{"id":3902635199640766,"definition":{"title":"Envelope State Count","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_source_statistics{col:envelope_state_count} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"semantic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":16,"width":6,"height":3}}]},"layout":{"x":0,"y":22,"width":12,"height":20,"is_column_break":true}},{"id":8461660367120526,"definition":{"title":"Sinks","background_color":"gray","show_title":true,"type":"group","layout_type":"ordered","widgets":[{"id":8544966924804406,"definition":{"title":"CPU Usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:sink*} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":0,"y":0,"width":6,"height":4}},{"id":2070922288633090,"definition":{"title":"Memory Usage","title_size":"16","title_align":"left","show_legend":true,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"name":"query1","data_source":"metrics","query":"avg:materialize.sql_cluster_replica_usage{cluster_type:sink*} by {cluster_name}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}],"yaxis":{"max":"100"}},"layout":{"x":6,"y":0,"width":6,"height":4}},{"id":6346996337026122,"definition":{"title":"Sinks Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:starting,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"custom_text"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":0,"y":4,"width":3,"height":2}},{"id":1788220607919434,"definition":{"title":"Sinks Running","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:running,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_green"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":3,"y":4,"width":3,"height":2}},{"id":405464898842380,"definition":{"title":"Sinks Stalled","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:stalled,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_yellow"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":6,"y":4,"width":3,"height":2}},{"id":235027738347796,"definition":{"title":"Sinks Failing","title_size":"16","title_align":"left","type":"query_value","requests":[{"response_format":"scalar","queries":[{"name":"query1","data_source":"metrics","query":"sum:materialize.sql_sink_statistics{sink_status_label:failling,col:sink_status_counter}","aggregator":"last"}],"formulas":[{"formula":"query1"}],"conditional_formats":[{"comparator":">","value":0,"palette":"black_on_light_red"}]}],"autoscale":true,"precision":2,"timeseries_background":{"yaxis":{"include_zero":true},"type":"area"}},"layout":{"x":9,"y":4,"width":3,"height":2}},{"id":3605514603037456,"definition":{"title":"Bytes Commited","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_sink_statistics{col:bytes_committed}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":0,"y":6,"width":6,"height":4}},{"id":2634515619757848,"definition":{"title":"Messages Commited","title_size":"16","title_align":"left","show_legend":false,"legend_layout":"auto","legend_columns":["avg","min","max","value","sum"],"type":"timeseries","requests":[{"formulas":[{"formula":"query1"}],"queries":[{"data_source":"metrics","name":"query1","query":"sum:materialize.sql_sink_statistics{col:messages_committed}"}],"response_format":"timeseries","style":{"palette":"dog_classic","line_type":"solid","line_width":"normal"},"display_type":"line"}]},"layout":{"x":6,"y":6,"width":6,"height":4}}]},"layout":{"x":0,"y":42,"width":12,"height":11}}],"template_variables":[],"layout_type":"ordered","notify_list":[],"reflow_type":"fixed"}